drivers/net/wireless/intel/iwlwifi/pcie/tx.c

   1 /******************************************************************************
   2  *
   3  * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
   4  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
   5  * Copyright(c) 2016 Intel Deutschland GmbH
   6  *
   7  * Portions of this file are derived from the ipw3945 project, as well
   8  * as portions of the ieee80211 subsystem header files.
   9  *
  10  * This program is free software; you can redistribute it and/or modify it
  11  * under the terms of version 2 of the GNU General Public License as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it will be useful, but WITHOUT
  15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17  * more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along with
  20  * this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
  22  *
  23  * The full GNU General Public License is included in this distribution in the
  24  * file called LICENSE.
  25  *
  26  * Contact Information:
  27  *  Intel Linux Wireless <linuxwifi@intel.com>
  28  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  29  *
  30  *****************************************************************************/
  31 #include <linux/etherdevice.h>
  32 #include <linux/ieee80211.h>
  33 #include <linux/slab.h>
  34 #include <linux/sched.h>
  35 #include <linux/pm_runtime.h>
  36 #include <net/ip6_checksum.h>
  37 #include <net/tso.h>
  38
  39 #include "iwl-debug.h"
  40 #include "iwl-csr.h"
  41 #include "iwl-prph.h"
  42 #include "iwl-io.h"
  43 #include "iwl-scd.h"
  44 #include "iwl-op-mode.h"
  45 #include "internal.h"
  46 /* FIXME: need to abstract out TX command (once we know what it looks like) */
  47 #include "dvm/commands.h"
  48
  49 #define IWL_TX_CRC_SIZE 4
  50 #define IWL_TX_DELIMITER_SIZE 4
  51
  52 /*************** DMA-QUEUE-GENERAL-FUNCTIONS  *****
  53  * DMA services
  54  *
  55  * Theory of operation
  56  *
  57  * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer
  58  * of buffer descriptors, each of which points to one or more data buffers for
  59  * the device to read from or fill.  Driver and device exchange status of each
  60  * queue via "read" and "write" pointers.  Driver keeps minimum of 2 empty
  61  * entries in each circular buffer, to protect against confusing empty and full
  62  * queue states.
  63  *
  64  * The device reads or writes the data in the queues via the device's several
  65  * DMA/FIFO channels.  Each queue is mapped to a single DMA channel.
  66  *
  67  * For Tx queue, there are low mark and high mark limits. If, after queuing
  68  * the packet for Tx, free space become < low mark, Tx queue stopped. When
  69  * reclaiming packets (on 'tx done IRQ), if free space become > high mark,
  70  * Tx queue resumed.
  71  *
  72  ***************************************************/
  73
  74 static int iwl_queue_space(const struct iwl_queue *q)
  75 {
  76         unsigned int max;
  77         unsigned int used;
  78
  79         /*
  80          * To avoid ambiguity between empty and completely full queues, there
  81          * should always be less than TFD_QUEUE_SIZE_MAX elements in the queue.
  82          * If q->n_window is smaller than TFD_QUEUE_SIZE_MAX, there is no need
  83          * to reserve any queue entries for this purpose.
  84          */
  85         if (q->n_window < TFD_QUEUE_SIZE_MAX)
  86                 max = q->n_window;
  87         else
  88                 max = TFD_QUEUE_SIZE_MAX - 1;
  89
  90         /*
  91          * TFD_QUEUE_SIZE_MAX is a power of 2, so the following is equivalent to
  92          * modulo by TFD_QUEUE_SIZE_MAX and is well defined.
  93          */
  94         used = (q->write_ptr - q->read_ptr) & (TFD_QUEUE_SIZE_MAX - 1);
  95
  96         if (WARN_ON(used > max))
  97                 return 0;
  98
  99         return max - used;
 100 }
 101
 102 /*
 103  * iwl_queue_init - Initialize queue's high/low-water and read/write indexes
 104  */
 105 static int iwl_queue_init(struct iwl_queue *q, int slots_num, u32 id)
 106 {
 107         q->n_window = slots_num;
 108         q->id = id;
 109
 110         /* slots_num must be power-of-two size, otherwise
 111          * get_cmd_index is broken. */
 112         if (WARN_ON(!is_power_of_2(slots_num)))
 113                 return -EINVAL;
 114
 115         q->low_mark = q->n_window / 4;
 116         if (q->low_mark < 4)
 117                 q->low_mark = 4;
 118
 119         q->high_mark = q->n_window / 8;
 120         if (q->high_mark < 2)
 121                 q->high_mark = 2;
 122
 123         q->write_ptr = 0;
 124         q->read_ptr = 0;
 125
 126         return 0;
 127 }
 128
 129 static int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
 130                                   struct iwl_dma_ptr *ptr, size_t size)
 131 {
 132         if (WARN_ON(ptr->addr))
 133                 return -EINVAL;
 134
 135         ptr->addr = dma_alloc_coherent(trans->dev, size,
 136                                        &ptr->dma, GFP_KERNEL);
 137         if (!ptr->addr)
 138                 return -ENOMEM;
 139         ptr->size = size;
 140         return 0;
 141 }
 142
 143 static void iwl_pcie_free_dma_ptr(struct iwl_trans *trans,
 144                                   struct iwl_dma_ptr *ptr)
 145 {
 146         if (unlikely(!ptr->addr))
 147                 return;
 148
 149         dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma);
 150         memset(ptr, 0, sizeof(*ptr));
 151 }
 152
 153 static void iwl_pcie_txq_stuck_timer(unsigned long data)
 154 {
 155         struct iwl_txq *txq = (void *)data;
 156         struct iwl_trans_pcie *trans_pcie = txq->trans_pcie;
 157         struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie);
 158
 159         spin_lock(&txq->lock);
 160         /* check if triggered erroneously */
 161         if (txq->q.read_ptr == txq->q.write_ptr) {
 162                 spin_unlock(&txq->lock);
 163                 return;
 164         }
 165         spin_unlock(&txq->lock);
 166
 167         IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->q.id,
 168                 jiffies_to_msecs(txq->wd_timeout));
 169
 170         iwl_trans_pcie_log_scd_error(trans, txq);
 171
 172         iwl_force_nmi(trans);
 173 }
 174
 175 /*
 176  * iwl_pcie_txq_update_byte_cnt_tbl - Set up entry in Tx byte-count array
 177  */
 178 static void iwl_pcie_txq_update_byte_cnt_tbl(struct iwl_trans *trans,
 179                                              struct iwl_txq *txq, u16 byte_cnt)
 180 {
 181         struct iwlagn_scd_bc_tbl *scd_bc_tbl;
 182         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 183         int write_ptr = txq->q.write_ptr;
 184         int txq_id = txq->q.id;
 185         u8 sec_ctl = 0;
 186         u8 sta_id = 0;
 187         u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE;
 188         __le16 bc_ent;
 189         struct iwl_tx_cmd *tx_cmd =
 190                 (void *) txq->entries[txq->q.write_ptr].cmd->payload;
 191
 192         scd_bc_tbl = trans_pcie->scd_bc_tbls.addr;
 193
 194         sta_id = tx_cmd->sta_id;
 195         sec_ctl = tx_cmd->sec_ctl;
 196
 197         switch (sec_ctl & TX_CMD_SEC_MSK) {
 198         case TX_CMD_SEC_CCM:
 199                 len += IEEE80211_CCMP_MIC_LEN;
 200                 break;
 201         case TX_CMD_SEC_TKIP:
 202                 len += IEEE80211_TKIP_ICV_LEN;
 203                 break;
 204         case TX_CMD_SEC_WEP:
 205                 len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN;
 206                 break;
 207         }
 208
 209         if (trans_pcie->bc_table_dword)
 210                 len = DIV_ROUND_UP(len, 4);
 211
 212         if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX))
 213                 return;
 214
 215         bc_ent = cpu_to_le16(len | (sta_id << 12));
 216
 217         scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent;
 218
 219         if (write_ptr < TFD_QUEUE_SIZE_BC_DUP)
 220                 scd_bc_tbl[txq_id].
 221                         tfd_offset[TFD_QUEUE_SIZE_MAX + write_ptr] = bc_ent;
 222 }
 223
 224 static void iwl_pcie_txq_inval_byte_cnt_tbl(struct iwl_trans *trans,
 225                                             struct iwl_txq *txq)
 226 {
 227         struct iwl_trans_pcie *trans_pcie =
 228                 IWL_TRANS_GET_PCIE_TRANS(trans);
 229         struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans_pcie->scd_bc_tbls.addr;
 230         int txq_id = txq->q.id;
 231         int read_ptr = txq->q.read_ptr;
 232         u8 sta_id = 0;
 233         __le16 bc_ent;
 234         struct iwl_tx_cmd *tx_cmd =
 235                 (void *)txq->entries[txq->q.read_ptr].cmd->payload;
 236
 237         WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX);
 238
 239         if (txq_id != trans_pcie->cmd_queue)
 240                 sta_id = tx_cmd->sta_id;
 241
 242         bc_ent = cpu_to_le16(1 | (sta_id << 12));
 243         scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent;
 244
 245         if (read_ptr < TFD_QUEUE_SIZE_BC_DUP)
 246                 scd_bc_tbl[txq_id].
 247                         tfd_offset[TFD_QUEUE_SIZE_MAX + read_ptr] = bc_ent;
 248 }
 249
 250 /*
 251  * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware
 252  */
 253 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 254                                     struct iwl_txq *txq)
 255 {
 256         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 257         u32 reg = 0;
 258         int txq_id = txq->q.id;
 259
 260         lockdep_assert_held(&txq->lock);
 261
 262         /*
 263          * explicitly wake up the NIC if:
 264          * 1. shadow registers aren't enabled
 265          * 2. NIC is woken up for CMD regardless of shadow outside this function
 266          * 3. there is a chance that the NIC is asleep
 267          */
 268         if (!trans->cfg->base_params->shadow_reg_enable &&
 269             txq_id != trans_pcie->cmd_queue &&
 270             test_bit(STATUS_TPOWER_PMI, &trans->status)) {
 271                 /*
 272                  * wake up nic if it's powered down ...
 273                  * uCode will wake up, and interrupt us again, so next
 274                  * time we'll skip this part.
 275                  */
 276                 reg = iwl_read32(trans, CSR_UCODE_DRV_GP1);
 277
 278                 if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) {
 279                         IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n",
 280                                        txq_id, reg);
 281                         iwl_set_bit(trans, CSR_GP_CNTRL,
 282                                     CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
 283                         txq->need_update = true;
 284                         return;
 285                 }
 286         }
 287
 288         /*
 289          * if not in power-save mode, uCode will never sleep when we're
 290          * trying to tx (during RFKILL, we're not trying to tx).
 291          */
 292         IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->q.write_ptr);
 293         if (!txq->block)
 294                 iwl_write32(trans, HBUS_TARG_WRPTR,
 295                             txq->q.write_ptr | (txq_id << 8));
 296 }
 297
 298 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans)
 299 {
 300         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 301         int i;
 302
 303         for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) {
 304                 struct iwl_txq *txq = &trans_pcie->txq[i];
 305
 306                 spin_lock_bh(&txq->lock);
 307                 if (trans_pcie->txq[i].need_update) {
 308                         iwl_pcie_txq_inc_wr_ptr(trans, txq);
 309                         trans_pcie->txq[i].need_update = false;
 310                 }
 311                 spin_unlock_bh(&txq->lock);
 312         }
 313 }
 314
 315 static inline dma_addr_t iwl_pcie_tfd_tb_get_addr(struct iwl_tfd *tfd, u8 idx)
 316 {
 317         struct iwl_tfd_tb *tb = &tfd->tbs[idx];
 318
 319         dma_addr_t addr = get_unaligned_le32(&tb->lo);
 320         if (sizeof(dma_addr_t) > sizeof(u32))
 321                 addr |=
 322                 ((dma_addr_t)(le16_to_cpu(tb->hi_n_len) & 0xF) << 16) << 16;
 323
 324         return addr;
 325 }
 326
 327 static inline void iwl_pcie_tfd_set_tb(struct iwl_tfd *tfd, u8 idx,
 328                                        dma_addr_t addr, u16 len)
 329 {
 330         struct iwl_tfd_tb *tb = &tfd->tbs[idx];
 331         u16 hi_n_len = len << 4;
 332
 333         put_unaligned_le32(addr, &tb->lo);
 334         if (sizeof(dma_addr_t) > sizeof(u32))
 335                 hi_n_len |= ((addr >> 16) >> 16) & 0xF;
 336
 337         tb->hi_n_len = cpu_to_le16(hi_n_len);
 338
 339         tfd->num_tbs = idx + 1;
 340 }
 341
 342 static inline u8 iwl_pcie_tfd_get_num_tbs(struct iwl_tfd *tfd)
 343 {
 344         return tfd->num_tbs & 0x1f;
 345 }
 346
 347 static void iwl_pcie_tfd_unmap(struct iwl_trans *trans,
 348                                struct iwl_cmd_meta *meta,
 349                                struct iwl_tfd *tfd)
 350 {
 351         int i;
 352         int num_tbs;
 353
 354         /* Sanity check on number of chunks */
 355         num_tbs = iwl_pcie_tfd_get_num_tbs(tfd);
 356
 357         if (num_tbs >= IWL_NUM_OF_TBS) {
 358                 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs);
 359                 /* @todo issue fatal error, it is quite serious situation */
 360                 return;
 361         }
 362
 363         /* first TB is never freed - it's the bidirectional DMA data */
 364
 365         for (i = 1; i < num_tbs; i++) {
 366                 if (meta->flags & BIT(i + CMD_TB_BITMAP_POS))
 367                         dma_unmap_page(trans->dev,
 368                                        iwl_pcie_tfd_tb_get_addr(tfd, i),
 369                                        iwl_pcie_tfd_tb_get_len(tfd, i),
 370                                        DMA_TO_DEVICE);
 371                 else
 372                         dma_unmap_single(trans->dev,
 373                                          iwl_pcie_tfd_tb_get_addr(tfd, i),
 374                                          iwl_pcie_tfd_tb_get_len(tfd, i),
 375                                          DMA_TO_DEVICE);
 376         }
 377         tfd->num_tbs = 0;
 378 }
 379
 380 /*
 381  * iwl_pcie_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr]
 382  * @trans - transport private data
 383  * @txq - tx queue
 384  * @dma_dir - the direction of the DMA mapping
 385  *
 386  * Does NOT advance any TFD circular buffer read/write indexes
 387  * Does NOT free the TFD itself (which is within circular buffer)
 388  */
 389 static void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
 390 {
 391         struct iwl_tfd *tfd_tmp = txq->tfds;
 392
 393         /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and
 394          * idx is bounded by n_window
 395          */
 396         int rd_ptr = txq->q.read_ptr;
 397         int idx = get_cmd_index(&txq->q, rd_ptr);
 398
 399         lockdep_assert_held(&txq->lock);
 400
 401         /* We have only q->n_window txq->entries, but we use
 402          * TFD_QUEUE_SIZE_MAX tfds
 403          */
 404         iwl_pcie_tfd_unmap(trans, &txq->entries[idx].meta, &tfd_tmp[rd_ptr]);
 405
 406         /* free SKB */
 407         if (txq->entries) {
 408                 struct sk_buff *skb;
 409
 410                 skb = txq->entries[idx].skb;
 411
 412                 /* Can be called from irqs-disabled context
 413                  * If skb is not NULL, it means that the whole queue is being
 414                  * freed and that the queue is not empty - free the skb
 415                  */
 416                 if (skb) {
 417                         iwl_op_mode_free_skb(trans->op_mode, skb);
 418                         txq->entries[idx].skb = NULL;
 419                 }
 420         }
 421 }
 422
 423 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq,
 424                                   dma_addr_t addr, u16 len, bool reset)
 425 {
 426         struct iwl_queue *q;
 427         struct iwl_tfd *tfd, *tfd_tmp;
 428         u32 num_tbs;
 429
 430         q = &txq->q;
 431         tfd_tmp = txq->tfds;
 432         tfd = &tfd_tmp[q->write_ptr];
 433
 434         if (reset)
 435                 memset(tfd, 0, sizeof(*tfd));
 436
 437         num_tbs = iwl_pcie_tfd_get_num_tbs(tfd);
 438
 439         /* Each TFD can point to a maximum 20 Tx buffers */
 440         if (num_tbs >= IWL_NUM_OF_TBS) {
 441                 IWL_ERR(trans, "Error can not send more than %d chunks\n",
 442                         IWL_NUM_OF_TBS);
 443                 return -EINVAL;
 444         }
 445
 446         if (WARN(addr & ~IWL_TX_DMA_MASK,
 447                  "Unaligned address = %llx\n", (unsigned long long)addr))
 448                 return -EINVAL;
 449
 450         iwl_pcie_tfd_set_tb(tfd, num_tbs, addr, len);
 451
 452         return num_tbs;
 453 }
 454
 455 static int iwl_pcie_txq_alloc(struct iwl_trans *trans,
 456                                struct iwl_txq *txq, int slots_num,
 457                                u32 txq_id)
 458 {
 459         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 460         size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX;
 461         size_t tb0_buf_sz;
 462         int i;
 463
 464         if (WARN_ON(txq->entries || txq->tfds))
 465                 return -EINVAL;
 466
 467         setup_timer(&txq->stuck_timer, iwl_pcie_txq_stuck_timer,
 468                     (unsigned long)txq);
 469         txq->trans_pcie = trans_pcie;
 470
 471         txq->q.n_window = slots_num;
 472
 473         txq->entries = kcalloc(slots_num,
 474                                sizeof(struct iwl_pcie_txq_entry),
 475                                GFP_KERNEL);
 476
 477         if (!txq->entries)
 478                 goto error;
 479
 480         if (txq_id == trans_pcie->cmd_queue)
 481                 for (i = 0; i < slots_num; i++) {
 482                         txq->entries[i].cmd =
 483                                 kmalloc(sizeof(struct iwl_device_cmd),
 484                                         GFP_KERNEL);
 485                         if (!txq->entries[i].cmd)
 486                                 goto error;
 487                 }
 488
 489         /* Circular buffer of transmit frame descriptors (TFDs),
 490          * shared with device */
 491         txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,
 492                                        &txq->q.dma_addr, GFP_KERNEL);
 493         if (!txq->tfds)
 494                 goto error;
 495
 496         BUILD_BUG_ON(IWL_FIRST_TB_SIZE_ALIGN != sizeof(*txq->first_tb_bufs));
 497
 498         tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num;
 499
 500         txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz,
 501                                               &txq->first_tb_dma,
 502                                               GFP_KERNEL);
 503         if (!txq->first_tb_bufs)
 504                 goto err_free_tfds;
 505
 506         txq->q.id = txq_id;
 507
 508         return 0;
 509 err_free_tfds:
 510         dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->q.dma_addr);
 511 error:
 512         if (txq->entries && txq_id == trans_pcie->cmd_queue)
 513                 for (i = 0; i < slots_num; i++)
 514                         kfree(txq->entries[i].cmd);
 515         kfree(txq->entries);
 516         txq->entries = NULL;
 517
 518         return -ENOMEM;
 519
 520 }
 521
 522 static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
 523                               int slots_num, u32 txq_id)
 524 {
 525         int ret;
 526
 527         txq->need_update = false;
 528
 529         /* TFD_QUEUE_SIZE_MAX must be power-of-two size, otherwise
 530          * iwl_queue_inc_wrap and iwl_queue_dec_wrap are broken. */
 531         BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1));
 532
 533         /* Initialize queue's high/low-water marks, and head/tail indexes */
 534         ret = iwl_queue_init(&txq->q, slots_num, txq_id);
 535         if (ret)
 536                 return ret;
 537
 538         spin_lock_init(&txq->lock);
 539         __skb_queue_head_init(&txq->overflow_q);
 540
 541         /*
 542          * Tell nic where to find circular buffer of Tx Frame Descriptors for
 543          * given Tx queue, and enable the DMA channel used for that queue.
 544          * Circular buffer (TFD queue in DRAM) physical base address */
 545         if (trans->cfg->use_tfh)
 546                 iwl_write_direct64(trans,
 547                                    FH_MEM_CBBC_QUEUE(trans, txq_id),
 548                                    txq->q.dma_addr);
 549         else
 550                 iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id),
 551                                    txq->q.dma_addr >> 8);
 552
 553         return 0;
 554 }
 555
 556 static void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie,
 557                                    struct sk_buff *skb)
 558 {
 559         struct page **page_ptr;
 560
 561         page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs);
 562
 563         if (*page_ptr) {
 564                 __free_page(*page_ptr);
 565                 *page_ptr = NULL;
 566         }
 567 }
 568
 569 static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans)
 570 {
 571         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 572
 573         lockdep_assert_held(&trans_pcie->reg_lock);
 574
 575         if (trans_pcie->ref_cmd_in_flight) {
 576                 trans_pcie->ref_cmd_in_flight = false;
 577                 IWL_DEBUG_RPM(trans, "clear ref_cmd_in_flight - unref\n");
 578                 iwl_trans_unref(trans);
 579         }
 580
 581         if (!trans->cfg->base_params->apmg_wake_up_wa)
 582                 return;
 583         if (WARN_ON(!trans_pcie->cmd_hold_nic_awake))
 584                 return;
 585
 586         trans_pcie->cmd_hold_nic_awake = false;
 587         __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
 588                                    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
 589 }
 590
 591 /*
 592  * iwl_pcie_txq_unmap -  Unmap any remaining DMA mappings and free skb's
 593  */
 594 static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
 595 {
 596         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 597         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
 598         struct iwl_queue *q = &txq->q;
 599
 600         spin_lock_bh(&txq->lock);
 601         while (q->write_ptr != q->read_ptr) {
 602                 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
 603                                    txq_id, q->read_ptr);
 604
 605                 if (txq_id != trans_pcie->cmd_queue) {
 606                         struct sk_buff *skb = txq->entries[q->read_ptr].skb;
 607
 608                         if (WARN_ON_ONCE(!skb))
 609                                 continue;
 610
 611                         iwl_pcie_free_tso_page(trans_pcie, skb);
 612                 }
 613                 iwl_pcie_txq_free_tfd(trans, txq);
 614                 q->read_ptr = iwl_queue_inc_wrap(q->read_ptr);
 615
 616                 if (q->read_ptr == q->write_ptr) {
 617                         unsigned long flags;
 618
 619                         spin_lock_irqsave(&trans_pcie->reg_lock, flags);
 620                         if (txq_id != trans_pcie->cmd_queue) {
 621                                 IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n",
 622                                               q->id);
 623                                 iwl_trans_unref(trans);
 624                         } else {
 625                                 iwl_pcie_clear_cmd_in_flight(trans);
 626                         }
 627                         spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
 628                 }
 629         }
 630         txq->active = false;
 631
 632         while (!skb_queue_empty(&txq->overflow_q)) {
 633                 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);
 634
 635                 iwl_op_mode_free_skb(trans->op_mode, skb);
 636         }
 637
 638         spin_unlock_bh(&txq->lock);
 639
 640         /* just in case - this queue may have been stopped */
 641         iwl_wake_queue(trans, txq);
 642 }
 643
 644 /*
 645  * iwl_pcie_txq_free - Deallocate DMA queue.
 646  * @txq: Transmit queue to deallocate.
 647  *
 648  * Empty queue by removing and destroying all BD's.
 649  * Free all buffers.
 650  * 0-fill, but do not free "txq" descriptor structure.
 651  */
 652 static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 653 {
 654         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 655         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
 656         struct device *dev = trans->dev;
 657         int i;
 658
 659         if (WARN_ON(!txq))
 660                 return;
 661
 662         iwl_pcie_txq_unmap(trans, txq_id);
 663
 664         /* De-alloc array of command/tx buffers */
 665         if (txq_id == trans_pcie->cmd_queue)
 666                 for (i = 0; i < txq->q.n_window; i++) {
 667                         kzfree(txq->entries[i].cmd);
 668                         kzfree(txq->entries[i].free_buf);
 669                 }
 670
 671         /* De-alloc circular buffer of TFDs */
 672         if (txq->tfds) {
 673                 dma_free_coherent(dev,
 674                                   sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX,
 675                                   txq->tfds, txq->q.dma_addr);
 676                 txq->q.dma_addr = 0;
 677                 txq->tfds = NULL;
 678
 679                 dma_free_coherent(dev,
 680                                   sizeof(*txq->first_tb_bufs) * txq->q.n_window,
 681                                   txq->first_tb_bufs, txq->first_tb_dma);
 682         }
 683
 684         kfree(txq->entries);
 685         txq->entries = NULL;
 686
 687         del_timer_sync(&txq->stuck_timer);
 688
 689         /* 0-fill queue descriptor structure */
 690         memset(txq, 0, sizeof(*txq));
 691 }
 692
 693 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
 694 {
 695         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 696         int nq = trans->cfg->base_params->num_of_queues;
 697         int chan;
 698         u32 reg_val;
 699         int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) -
 700                                 SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32);
 701
 702         /* make sure all queue are not stopped/used */
 703         memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped));
 704         memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used));
 705
 706         trans_pcie->scd_base_addr =
 707                 iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
 708
 709         WARN_ON(scd_base_addr != 0 &&
 710                 scd_base_addr != trans_pcie->scd_base_addr);
 711
 712         /* reset context data, TX status and translation data */
 713         iwl_trans_write_mem(trans, trans_pcie->scd_base_addr +
 714                                    SCD_CONTEXT_MEM_LOWER_BOUND,
 715                             NULL, clear_dwords);
 716
 717         iwl_write_prph(trans, SCD_DRAM_BASE_ADDR,
 718                        trans_pcie->scd_bc_tbls.dma >> 10);
 719
 720         /* The chain extension of the SCD doesn't work well. This feature is
 721          * enabled by default by the HW, so we need to disable it manually.
 722          */
 723         if (trans->cfg->base_params->scd_chain_ext_wa)
 724                 iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);
 725
 726         iwl_trans_ac_txq_enable(trans, trans_pcie->cmd_queue,
 727                                 trans_pcie->cmd_fifo,
 728                                 trans_pcie->cmd_q_wdg_timeout);
 729
 730         /* Activate all Tx DMA/FIFO channels */
 731         iwl_scd_activate_fifos(trans);
 732
 733         /* Enable DMA channel */
 734         for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++)
 735                 iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan),
 736                                    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
 737                                    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
 738
 739         /* Update FH chicken bits */
 740         reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG);
 741         iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG,
 742                            reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
 743
 744         /* Enable L1-Active */
 745         if (trans->cfg->device_family != IWL_DEVICE_FAMILY_8000)
 746                 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
 747                                     APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
 748 }
 749
 750 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans)
 751 {
 752         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 753         int txq_id;
 754
 755         for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
 756              txq_id++) {
 757                 struct iwl_txq *txq = &trans_pcie->txq[txq_id];
 758                 if (trans->cfg->use_tfh)
 759                         iwl_write_direct64(trans,
 760                                            FH_MEM_CBBC_QUEUE(trans, txq_id),
 761                                            txq->q.dma_addr);
 762                 else
 763                         iwl_write_direct32(trans,
 764                                            FH_MEM_CBBC_QUEUE(trans, txq_id),
 765                                            txq->q.dma_addr >> 8);
 766                 iwl_pcie_txq_unmap(trans, txq_id);
 767                 txq->q.read_ptr = 0;
 768                 txq->q.write_ptr = 0;
 769         }
 770
 771         /* Tell NIC where to find the "keep warm" buffer */
 772         iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
 773                            trans_pcie->kw.dma >> 4);
 774
 775         /*
 776          * Send 0 as the scd_base_addr since the device may have be reset
 777          * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will
 778          * contain garbage.
 779          */
 780         iwl_pcie_tx_start(trans, 0);
 781 }
 782
 783 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans)
 784 {
 785         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 786         unsigned long flags;
 787         int ch, ret;
 788         u32 mask = 0;
 789
 790         spin_lock(&trans_pcie->irq_lock);
 791
 792         if (!iwl_trans_grab_nic_access(trans, &flags))
 793                 goto out;
 794
 795         /* Stop each Tx DMA channel */
 796         for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) {
 797                 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0);
 798                 mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch);
 799         }
 800
 801         /* Wait for DMA channels to be idle */
 802         ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000);
 803         if (ret < 0)
 804                 IWL_ERR(trans,
 805                         "Failing on timeout while stopping DMA channel %d [0x%08x]\n",
 806                         ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG));
 807
 808         iwl_trans_release_nic_access(trans, &flags);
 809
 810 out:
 811         spin_unlock(&trans_pcie->irq_lock);
 812 }
 813
 814 /*
 815  * iwl_pcie_tx_stop - Stop all Tx DMA channels
 816  */
 817 int iwl_pcie_tx_stop(struct iwl_trans *trans)
 818 {
 819         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 820         int txq_id;
 821
 822         /* Turn off all Tx DMA fifos */
 823         iwl_scd_deactivate_fifos(trans);
 824
 825         /* Turn off all Tx DMA channels */
 826         iwl_pcie_tx_stop_fh(trans);
 827
 828         /*
 829          * This function can be called before the op_mode disabled the
 830          * queues. This happens when we have an rfkill interrupt.
 831          * Since we stop Tx altogether - mark the queues as stopped.
 832          */
 833         memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped));
 834         memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used));
 835
 836         /* This can happen: start_hw, stop_device */
 837         if (!trans_pcie->txq)
 838                 return 0;
 839
 840         /* Unmap DMA from host system and free skb's */
 841         for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
 842              txq_id++)
 843                 iwl_pcie_txq_unmap(trans, txq_id);
 844
 845         return 0;
 846 }
 847
 848 /*
 849  * iwl_trans_tx_free - Free TXQ Context
 850  *
 851  * Destroy all TX DMA queues and structures
 852  */
 853 void iwl_pcie_tx_free(struct iwl_trans *trans)
 854 {
 855         int txq_id;
 856         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 857
 858         /* Tx queues */
 859         if (trans_pcie->txq) {
 860                 for (txq_id = 0;
 861                      txq_id < trans->cfg->base_params->num_of_queues; txq_id++)
 862                         iwl_pcie_txq_free(trans, txq_id);
 863         }
 864
 865         kfree(trans_pcie->txq);
 866         trans_pcie->txq = NULL;
 867
 868         iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw);
 869
 870         iwl_pcie_free_dma_ptr(trans, &trans_pcie->scd_bc_tbls);
 871 }
 872
 873 /*
 874  * iwl_pcie_tx_alloc - allocate TX context
 875  * Allocate all Tx DMA structures and initialize them
 876  */
 877 static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 878 {
 879         int ret;
 880         int txq_id, slots_num;
 881         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 882
 883         u16 scd_bc_tbls_size = trans->cfg->base_params->num_of_queues *
 884                         sizeof(struct iwlagn_scd_bc_tbl);
 885
 886         /*It is not allowed to alloc twice, so warn when this happens.
 887          * We cannot rely on the previous allocation, so free and fail */
 888         if (WARN_ON(trans_pcie->txq)) {
 889                 ret = -EINVAL;
 890                 goto error;
 891         }
 892
 893         ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->scd_bc_tbls,
 894                                    scd_bc_tbls_size);
 895         if (ret) {
 896                 IWL_ERR(trans, "Scheduler BC Table allocation failed\n");
 897                 goto error;
 898         }
 899
 900         /* Alloc keep-warm buffer */
 901         ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE);
 902         if (ret) {
 903                 IWL_ERR(trans, "Keep Warm allocation failed\n");
 904                 goto error;
 905         }
 906
 907         trans_pcie->txq = kcalloc(trans->cfg->base_params->num_of_queues,
 908                                   sizeof(struct iwl_txq), GFP_KERNEL);
 909         if (!trans_pcie->txq) {
 910                 IWL_ERR(trans, "Not enough memory for txq\n");
 911                 ret = -ENOMEM;
 912                 goto error;
 913         }
 914
 915         /* Alloc and init all Tx queues, including the command queue (#4/#9) */
 916         for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
 917              txq_id++) {
 918                 slots_num = (txq_id == trans_pcie->cmd_queue) ?
 919                                         TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
 920                 ret = iwl_pcie_txq_alloc(trans, &trans_pcie->txq[txq_id],
 921                                           slots_num, txq_id);
 922                 if (ret) {
 923                         IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
 924                         goto error;
 925                 }
 926         }
 927
 928         return 0;
 929
 930 error:
 931         iwl_pcie_tx_free(trans);
 932
 933         return ret;
 934 }
 935 int iwl_pcie_tx_init(struct iwl_trans *trans)
 936 {
 937         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 938         int ret;
 939         int txq_id, slots_num;
 940         bool alloc = false;
 941
 942         if (!trans_pcie->txq) {
 943                 ret = iwl_pcie_tx_alloc(trans);
 944                 if (ret)
 945                         goto error;
 946                 alloc = true;
 947         }
 948
 949         spin_lock(&trans_pcie->irq_lock);
 950
 951         /* Turn off all Tx DMA fifos */
 952         iwl_scd_deactivate_fifos(trans);
 953
 954         /* Tell NIC where to find the "keep warm" buffer */
 955         iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
 956                            trans_pcie->kw.dma >> 4);
 957
 958         spin_unlock(&trans_pcie->irq_lock);
 959
 960         /* Alloc and init all Tx queues, including the command queue (#4/#9) */
 961         for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
 962              txq_id++) {
 963                 slots_num = (txq_id == trans_pcie->cmd_queue) ?
 964                                         TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
 965                 ret = iwl_pcie_txq_init(trans, &trans_pcie->txq[txq_id],
 966                                          slots_num, txq_id);
 967                 if (ret) {
 968                         IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
 969                         goto error;
 970                 }
 971         }
 972
 973         if (trans->cfg->use_tfh)
 974                 iwl_write_direct32(trans, TFH_TRANSFER_MODE,
 975                                    TFH_TRANSFER_MAX_PENDING_REQ |
 976                                    TFH_CHUNK_SIZE_128 |
 977                                    TFH_CHUNK_SPLIT_MODE);
 978
 979         iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE);
 980         if (trans->cfg->base_params->num_of_queues > 20)
 981                 iwl_set_bits_prph(trans, SCD_GP_CTRL,
 982                                   SCD_GP_CTRL_ENABLE_31_QUEUES);
 983
 984         return 0;
 985 error:
 986         /*Upon error, free only if we allocated something */
 987         if (alloc)
 988                 iwl_pcie_tx_free(trans);
 989         return ret;
 990 }
 991
 992 static inline void iwl_pcie_txq_progress(struct iwl_txq *txq)
 993 {
 994         lockdep_assert_held(&txq->lock);
 995
 996         if (!txq->wd_timeout)
 997                 return;
 998
 999         /*
1000          * station is asleep and we send data - that must
1001          * be uAPSD or PS-Poll. Don't rearm the timer.
1002          */
1003         if (txq->frozen)
1004                 return;
1005
1006         /*
1007          * if empty delete timer, otherwise move timer forward
1008          * since we're making progress on this queue
1009          */
1010         if (txq->q.read_ptr == txq->q.write_ptr)
1011                 del_timer(&txq->stuck_timer);
1012         else
1013                 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
1014 }
1015
1016 /* Frees buffers until index _not_ inclusive */
1017 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
1018                             struct sk_buff_head *skbs)
1019 {
1020         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1021         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
1022         int tfd_num = ssn & (TFD_QUEUE_SIZE_MAX - 1);
1023         struct iwl_queue *q = &txq->q;
1024         int last_to_free;
1025
1026         /* This function is not meant to release cmd queue*/
1027         if (WARN_ON(txq_id == trans_pcie->cmd_queue))
1028                 return;
1029
1030         spin_lock_bh(&txq->lock);
1031
1032         if (!txq->active) {
1033                 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n",
1034                                     txq_id, ssn);
1035                 goto out;
1036         }
1037
1038         if (txq->q.read_ptr == tfd_num)
1039                 goto out;
1040
1041         IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n",
1042                            txq_id, txq->q.read_ptr, tfd_num, ssn);
1043
1044         /*Since we free until index _not_ inclusive, the one before index is
1045          * the last we will free. This one must be used */
1046         last_to_free = iwl_queue_dec_wrap(tfd_num);
1047
1048         if (!iwl_queue_used(q, last_to_free)) {
1049                 IWL_ERR(trans,
1050                         "%s: Read index for DMA queue txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n",
1051                         __func__, txq_id, last_to_free, TFD_QUEUE_SIZE_MAX,
1052                         q->write_ptr, q->read_ptr);
1053                 goto out;
1054         }
1055
1056         if (WARN_ON(!skb_queue_empty(skbs)))
1057                 goto out;
1058
1059         for (;
1060              q->read_ptr != tfd_num;
1061              q->read_ptr = iwl_queue_inc_wrap(q->read_ptr)) {
1062                 struct sk_buff *skb = txq->entries[txq->q.read_ptr].skb;
1063
1064                 if (WARN_ON_ONCE(!skb))
1065                         continue;
1066
1067                 iwl_pcie_free_tso_page(trans_pcie, skb);
1068
1069                 __skb_queue_tail(skbs, skb);
1070
1071                 txq->entries[txq->q.read_ptr].skb = NULL;
1072
1073                 iwl_pcie_txq_inval_byte_cnt_tbl(trans, txq);
1074
1075                 iwl_pcie_txq_free_tfd(trans, txq);
1076         }
1077
1078         iwl_pcie_txq_progress(txq);
1079
1080         if (iwl_queue_space(&txq->q) > txq->q.low_mark &&
1081             test_bit(txq_id, trans_pcie->queue_stopped)) {
1082                 struct sk_buff_head overflow_skbs;
1083
1084                 __skb_queue_head_init(&overflow_skbs);
1085                 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs);
1086
1087                 /*
1088                  * This is tricky: we are in reclaim path which is non
1089                  * re-entrant, so noone will try to take the access the
1090                  * txq data from that path. We stopped tx, so we can't
1091                  * have tx as well. Bottom line, we can unlock and re-lock
1092                  * later.
1093                  */
1094                 spin_unlock_bh(&txq->lock);
1095
1096                 while (!skb_queue_empty(&overflow_skbs)) {
1097                         struct sk_buff *skb = __skb_dequeue(&overflow_skbs);
1098                         struct iwl_device_cmd *dev_cmd_ptr;
1099
1100                         dev_cmd_ptr = *(void **)((u8 *)skb->cb +
1101                                                  trans_pcie->dev_cmd_offs);
1102
1103                         /*
1104                          * Note that we can very well be overflowing again.
1105                          * In that case, iwl_queue_space will be small again
1106                          * and we won't wake mac80211's queue.
1107                          */
1108                         iwl_trans_pcie_tx(trans, skb, dev_cmd_ptr, txq_id);
1109                 }
1110                 spin_lock_bh(&txq->lock);
1111
1112                 if (iwl_queue_space(&txq->q) > txq->q.low_mark)
1113                         iwl_wake_queue(trans, txq);
1114         }
1115
1116         if (q->read_ptr == q->write_ptr) {
1117                 IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", q->id);
1118                 iwl_trans_unref(trans);
1119         }
1120
1121 out:
1122         spin_unlock_bh(&txq->lock);
1123 }
1124
1125 static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
1126                                       const struct iwl_host_cmd *cmd)
1127 {
1128         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1129         int ret;
1130
1131         lockdep_assert_held(&trans_pcie->reg_lock);
1132
1133         if (!(cmd->flags & CMD_SEND_IN_IDLE) &&
1134             !trans_pcie->ref_cmd_in_flight) {
1135                 trans_pcie->ref_cmd_in_flight = true;
1136                 IWL_DEBUG_RPM(trans, "set ref_cmd_in_flight - ref\n");
1137                 iwl_trans_ref(trans);
1138         }
1139
1140         /*
1141          * wake up the NIC to make sure that the firmware will see the host
1142          * command - we will let the NIC sleep once all the host commands
1143          * returned. This needs to be done only on NICs that have
1144          * apmg_wake_up_wa set.
1145          */
1146         if (trans->cfg->base_params->apmg_wake_up_wa &&
1147             !trans_pcie->cmd_hold_nic_awake) {
1148                 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
1149                                          CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1150
1151                 ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
1152                                    CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
1153                                    (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
1154                                     CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP),
1155                                    15000);
1156                 if (ret < 0) {
1157                         __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
1158                                         CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1159                         IWL_ERR(trans, "Failed to wake NIC for hcmd\n");
1160                         return -EIO;
1161                 }
1162                 trans_pcie->cmd_hold_nic_awake = true;
1163         }
1164
1165         return 0;
1166 }
1167
1168 /*
1169  * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd
1170  *
1171  * When FW advances 'R' index, all entries between old and new 'R' index
1172  * need to be reclaimed. As result, some free space forms.  If there is
1173  * enough free space (> low mark), wake the stack that feeds us.
1174  */
1175 static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
1176 {
1177         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1178         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
1179         struct iwl_queue *q = &txq->q;
1180         unsigned long flags;
1181         int nfreed = 0;
1182
1183         lockdep_assert_held(&txq->lock);
1184
1185         if ((idx >= TFD_QUEUE_SIZE_MAX) || (!iwl_queue_used(q, idx))) {
1186                 IWL_ERR(trans,
1187                         "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n",
1188                         __func__, txq_id, idx, TFD_QUEUE_SIZE_MAX,
1189                         q->write_ptr, q->read_ptr);
1190                 return;
1191         }
1192
1193         for (idx = iwl_queue_inc_wrap(idx); q->read_ptr != idx;
1194              q->read_ptr = iwl_queue_inc_wrap(q->read_ptr)) {
1195
1196                 if (nfreed++ > 0) {
1197                         IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n",
1198                                 idx, q->write_ptr, q->read_ptr);
1199                         iwl_force_nmi(trans);
1200                 }
1201         }
1202
1203         if (q->read_ptr == q->write_ptr) {
1204                 spin_lock_irqsave(&trans_pcie->reg_lock, flags);
1205                 iwl_pcie_clear_cmd_in_flight(trans);
1206                 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
1207         }
1208
1209         iwl_pcie_txq_progress(txq);
1210 }
1211
1212 static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid,
1213                                  u16 txq_id)
1214 {
1215         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1216         u32 tbl_dw_addr;
1217         u32 tbl_dw;
1218         u16 scd_q2ratid;
1219
1220         scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK;
1221
1222         tbl_dw_addr = trans_pcie->scd_base_addr +
1223                         SCD_TRANS_TBL_OFFSET_QUEUE(txq_id);
1224
1225         tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr);
1226
1227         if (txq_id & 0x1)
1228                 tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF);
1229         else
1230                 tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000);
1231
1232         iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw);
1233
1234         return 0;
1235 }
1236
1237 /* Receiver address (actually, Rx station's index into station table),
1238  * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */
1239 #define BUILD_RAxTID(sta_id, tid)       (((sta_id) << 4) + (tid))
1240
1241 void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
1242                                const struct iwl_trans_txq_scd_cfg *cfg,
1243                                unsigned int wdg_timeout)
1244 {
1245         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1246         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
1247         int fifo = -1;
1248
1249         if (test_and_set_bit(txq_id, trans_pcie->queue_used))
1250                 WARN_ONCE(1, "queue %d already used - expect issues", txq_id);
1251
1252         txq->wd_timeout = msecs_to_jiffies(wdg_timeout);
1253
1254         if (cfg) {
1255                 fifo = cfg->fifo;
1256
1257                 /* Disable the scheduler prior configuring the cmd queue */
1258                 if (txq_id == trans_pcie->cmd_queue &&
1259                     trans_pcie->scd_set_active)
1260                         iwl_scd_enable_set_active(trans, 0);
1261
1262                 /* Stop this Tx queue before configuring it */
1263                 iwl_scd_txq_set_inactive(trans, txq_id);
1264
1265                 /* Set this queue as a chain-building queue unless it is CMD */
1266                 if (txq_id != trans_pcie->cmd_queue)
1267                         iwl_scd_txq_set_chain(trans, txq_id);
1268
1269                 if (cfg->aggregate) {
1270                         u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid);
1271
1272                         /* Map receiver-address / traffic-ID to this queue */
1273                         iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id);
1274
1275                         /* enable aggregations for the queue */
1276                         iwl_scd_txq_enable_agg(trans, txq_id);
1277                         txq->ampdu = true;
1278                 } else {
1279                         /*
1280                          * disable aggregations for the queue, this will also
1281                          * make the ra_tid mapping configuration irrelevant
1282                          * since it is now a non-AGG queue.
1283                          */
1284                         iwl_scd_txq_disable_agg(trans, txq_id);
1285
1286                         ssn = txq->q.read_ptr;
1287                 }
1288         }
1289
1290         /* Place first TFD at index corresponding to start sequence number.
1291          * Assumes that ssn_idx is valid (!= 0xFFF) */
1292         txq->q.read_ptr = (ssn & 0xff);
1293         txq->q.write_ptr = (ssn & 0xff);
1294         iwl_write_direct32(trans, HBUS_TARG_WRPTR,
1295                            (ssn & 0xff) | (txq_id << 8));
1296
1297         if (cfg) {
1298                 u8 frame_limit = cfg->frame_limit;
1299
1300                 iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn);
1301
1302                 /* Set up Tx window size and frame limit for this queue */
1303                 iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr +
1304                                 SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0);
1305                 iwl_trans_write_mem32(trans,
1306                         trans_pcie->scd_base_addr +
1307                         SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32),
1308                         ((frame_limit << SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1309                                         SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1310                         ((frame_limit << SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1311                                         SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1312
1313                 /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */
1314                 iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id),
1315                                (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1316                                (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) |
1317                                (1 << SCD_QUEUE_STTS_REG_POS_WSL) |
1318                                SCD_QUEUE_STTS_REG_MSK);
1319
1320                 /* enable the scheduler for this queue (only) */
1321                 if (txq_id == trans_pcie->cmd_queue &&
1322                     trans_pcie->scd_set_active)
1323                         iwl_scd_enable_set_active(trans, BIT(txq_id));
1324
1325                 IWL_DEBUG_TX_QUEUES(trans,
1326                                     "Activate queue %d on FIFO %d WrPtr: %d\n",
1327                                     txq_id, fifo, ssn & 0xff);
1328         } else {
1329                 IWL_DEBUG_TX_QUEUES(trans,
1330                                     "Activate queue %d WrPtr: %d\n",
1331                                     txq_id, ssn & 0xff);
1332         }
1333
1334         txq->active = true;
1335 }
1336
1337 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
1338                                         bool shared_mode)
1339 {
1340         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1341         struct iwl_txq *txq = &trans_pcie->txq[txq_id];
1342
1343         txq->ampdu = !shared_mode;
1344 }
1345
1346 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id,
1347                                 bool configure_scd)
1348 {
1349         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1350         u32 stts_addr = trans_pcie->scd_base_addr +
1351                         SCD_TX_STTS_QUEUE_OFFSET(txq_id);
1352         static const u32 zero_val[4] = {};
1353
1354         trans_pcie->txq[txq_id].frozen_expiry_remainder = 0;
1355         trans_pcie->txq[txq_id].frozen = false;
1356
1357         /*
1358          * Upon HW Rfkill - we stop the device, and then stop the queues
1359          * in the op_mode. Just for the sake of the simplicity of the op_mode,
1360          * allow the op_mode to call txq_disable after it already called
1361          * stop_device.
1362          */
1363         if (!test_and_clear_bit(txq_id, trans_pcie->queue_used)) {
1364                 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status),
1365                           "queue %d not used", txq_id);
1366                 return;
1367         }
1368
1369         if (configure_scd) {
1370                 iwl_scd_txq_set_inactive(trans, txq_id);
1371
1372                 iwl_trans_write_mem(trans, stts_addr, (void *)zero_val,
1373                                     ARRAY_SIZE(zero_val));
1374         }
1375
1376         iwl_pcie_txq_unmap(trans, txq_id);
1377         trans_pcie->txq[txq_id].ampdu = false;
1378
1379         IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id);
1380 }
1381
1382 /*************** HOST COMMAND QUEUE FUNCTIONS   *****/
1383
1384 /*
1385  * iwl_pcie_enqueue_hcmd - enqueue a uCode command
1386  * @priv: device private data point
1387  * @cmd: a pointer to the ucode command structure
1388  *
1389  * The function returns < 0 values to indicate the operation
1390  * failed. On success, it returns the index (>= 0) of command in the
1391  * command queue.
1392  */
1393 static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
1394                                  struct iwl_host_cmd *cmd)
1395 {
1396         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1397         struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
1398         struct iwl_queue *q = &txq->q;
1399         struct iwl_device_cmd *out_cmd;
1400         struct iwl_cmd_meta *out_meta;
1401         unsigned long flags;
1402         void *dup_buf = NULL;
1403         dma_addr_t phys_addr;
1404         int idx;
1405         u16 copy_size, cmd_size, tb0_size;
1406         bool had_nocopy = false;
1407         u8 group_id = iwl_cmd_groupid(cmd->id);
1408         int i, ret;
1409         u32 cmd_pos;
1410         const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD];
1411         u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
1412
1413         if (WARN(!trans_pcie->wide_cmd_header &&
1414                  group_id > IWL_ALWAYS_LONG_GROUP,
1415                  "unsupported wide command %#x\n", cmd->id))
1416                 return -EINVAL;
1417
1418         if (group_id != 0) {
1419                 copy_size = sizeof(struct iwl_cmd_header_wide);
1420                 cmd_size = sizeof(struct iwl_cmd_header_wide);
1421         } else {
1422                 copy_size = sizeof(struct iwl_cmd_header);
1423                 cmd_size = sizeof(struct iwl_cmd_header);
1424         }
1425
1426         /* need one for the header if the first is NOCOPY */
1427         BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1);
1428
1429         for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1430                 cmddata[i] = cmd->data[i];
1431                 cmdlen[i] = cmd->len[i];
1432
1433                 if (!cmd->len[i])
1434                         continue;
1435
1436                 /* need at least IWL_FIRST_TB_SIZE copied */
1437                 if (copy_size < IWL_FIRST_TB_SIZE) {
1438                         int copy = IWL_FIRST_TB_SIZE - copy_size;
1439
1440                         if (copy > cmdlen[i])
1441                                 copy = cmdlen[i];
1442                         cmdlen[i] -= copy;
1443                         cmddata[i] += copy;
1444                         copy_size += copy;
1445                 }
1446
1447                 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) {
1448                         had_nocopy = true;
1449                         if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) {
1450                                 idx = -EINVAL;
1451                                 goto free_dup_buf;
1452                         }
1453                 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) {
1454                         /*
1455                          * This is also a chunk that isn't copied
1456                          * to the static buffer so set had_nocopy.
1457                          */
1458                         had_nocopy = true;
1459
1460                         /* only allowed once */
1461                         if (WARN_ON(dup_buf)) {
1462                                 idx = -EINVAL;
1463                                 goto free_dup_buf;
1464                         }
1465
1466                         dup_buf = kmemdup(cmddata[i], cmdlen[i],
1467                                           GFP_ATOMIC);
1468                         if (!dup_buf)
1469                                 return -ENOMEM;
1470                 } else {
1471                         /* NOCOPY must not be followed by normal! */
1472                         if (WARN_ON(had_nocopy)) {
1473                                 idx = -EINVAL;
1474                                 goto free_dup_buf;
1475                         }
1476                         copy_size += cmdlen[i];
1477                 }
1478                 cmd_size += cmd->len[i];
1479         }
1480
1481         /*
1482          * If any of the command structures end up being larger than
1483          * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically
1484          * allocated into separate TFDs, then we will need to
1485          * increase the size of the buffers.
1486          */
1487         if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE,
1488                  "Command %s (%#x) is too large (%d bytes)\n",
1489                  iwl_get_cmd_string(trans, cmd->id),
1490                  cmd->id, copy_size)) {
1491                 idx = -EINVAL;
1492                 goto free_dup_buf;
1493         }
1494
1495         spin_lock_bh(&txq->lock);
1496
1497         if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
1498                 spin_unlock_bh(&txq->lock);
1499
1500                 IWL_ERR(trans, "No space in command queue\n");
1501                 iwl_op_mode_cmd_queue_full(trans->op_mode);
1502                 idx = -ENOSPC;
1503                 goto free_dup_buf;
1504         }
1505
1506         idx = get_cmd_index(q, q->write_ptr);
1507         out_cmd = txq->entries[idx].cmd;
1508         out_meta = &txq->entries[idx].meta;
1509
1510         memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */
1511         if (cmd->flags & CMD_WANT_SKB)
1512                 out_meta->source = cmd;
1513
1514         /* set up the header */
1515         if (group_id != 0) {
1516                 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id);
1517                 out_cmd->hdr_wide.group_id = group_id;
1518                 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id);
1519                 out_cmd->hdr_wide.length =
1520                         cpu_to_le16(cmd_size -
1521                                     sizeof(struct iwl_cmd_header_wide));
1522                 out_cmd->hdr_wide.reserved = 0;
1523                 out_cmd->hdr_wide.sequence =
1524                         cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) |
1525                                                  INDEX_TO_SEQ(q->write_ptr));
1526
1527                 cmd_pos = sizeof(struct iwl_cmd_header_wide);
1528                 copy_size = sizeof(struct iwl_cmd_header_wide);
1529         } else {
1530                 out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id);
1531                 out_cmd->hdr.sequence =
1532                         cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) |
1533                                                  INDEX_TO_SEQ(q->write_ptr));
1534                 out_cmd->hdr.group_id = 0;
1535
1536                 cmd_pos = sizeof(struct iwl_cmd_header);
1537                 copy_size = sizeof(struct iwl_cmd_header);
1538         }
1539
1540         /* and copy the data that needs to be copied */
1541         for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1542                 int copy;
1543
1544                 if (!cmd->len[i])
1545                         continue;
1546
1547                 /* copy everything if not nocopy/dup */
1548                 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
1549                                            IWL_HCMD_DFL_DUP))) {
1550                         copy = cmd->len[i];
1551
1552                         memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
1553                         cmd_pos += copy;
1554                         copy_size += copy;
1555                         continue;
1556                 }
1557
1558                 /*
1559                  * Otherwise we need at least IWL_FIRST_TB_SIZE copied
1560                  * in total (for bi-directional DMA), but copy up to what
1561                  * we can fit into the payload for debug dump purposes.
1562                  */
1563                 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]);
1564
1565                 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
1566                 cmd_pos += copy;
1567
1568                 /* However, treat copy_size the proper way, we need it below */
1569                 if (copy_size < IWL_FIRST_TB_SIZE) {
1570                         copy = IWL_FIRST_TB_SIZE - copy_size;
1571
1572                         if (copy > cmd->len[i])
1573                                 copy = cmd->len[i];
1574                         copy_size += copy;
1575                 }
1576         }
1577
1578         IWL_DEBUG_HC(trans,
1579                      "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n",
1580                      iwl_get_cmd_string(trans, cmd->id),
1581                      group_id, out_cmd->hdr.cmd,
1582                      le16_to_cpu(out_cmd->hdr.sequence),
1583                      cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue);
1584
1585         /* start the TFD with the minimum copy bytes */
1586         tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
1587         memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size);
1588         iwl_pcie_txq_build_tfd(trans, txq,
1589                                iwl_pcie_get_first_tb_dma(txq, idx),
1590                                tb0_size, true);
1591
1592         /* map first command fragment, if any remains */
1593         if (copy_size > tb0_size) {
1594                 phys_addr = dma_map_single(trans->dev,
1595                                            ((u8 *)&out_cmd->hdr) + tb0_size,
1596                                            copy_size - tb0_size,
1597                                            DMA_TO_DEVICE);
1598                 if (dma_mapping_error(trans->dev, phys_addr)) {
1599                         iwl_pcie_tfd_unmap(trans, out_meta,
1600                                            &txq->tfds[q->write_ptr]);
1601                         idx = -ENOMEM;
1602                         goto out;
1603                 }
1604
1605                 iwl_pcie_txq_build_tfd(trans, txq, phys_addr,
1606                                        copy_size - tb0_size, false);
1607         }
1608
1609         /* map the remaining (adjusted) nocopy/dup fragments */
1610         for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
1611                 const void *data = cmddata[i];
1612
1613                 if (!cmdlen[i])
1614                         continue;
1615                 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
1616                                            IWL_HCMD_DFL_DUP)))
1617                         continue;
1618                 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP)
1619                         data = dup_buf;
1620                 phys_addr = dma_map_single(trans->dev, (void *)data,
1621                                            cmdlen[i], DMA_TO_DEVICE);
1622                 if (dma_mapping_error(trans->dev, phys_addr)) {
1623                         iwl_pcie_tfd_unmap(trans, out_meta,
1624                                            &txq->tfds[q->write_ptr]);
1625                         idx = -ENOMEM;
1626                         goto out;
1627                 }
1628
1629                 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false);
1630         }
1631
1632         BUILD_BUG_ON(IWL_NUM_OF_TBS + CMD_TB_BITMAP_POS >
1633                      sizeof(out_meta->flags) * BITS_PER_BYTE);
1634         out_meta->flags = cmd->flags;
1635         if (WARN_ON_ONCE(txq->entries[idx].free_buf))
1636                 kzfree(txq->entries[idx].free_buf);
1637         txq->entries[idx].free_buf = dup_buf;
1638
1639         trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide);
1640
1641         /* start timer if queue currently empty */
1642         if (q->read_ptr == q->write_ptr && txq->wd_timeout)
1643                 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
1644
1645         spin_lock_irqsave(&trans_pcie->reg_lock, flags);
1646         ret = iwl_pcie_set_cmd_in_flight(trans, cmd);
1647         if (ret < 0) {
1648                 idx = ret;
1649                 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
1650                 goto out;
1651         }
1652
1653         /* Increment and update queue's write index */
1654         q->write_ptr = iwl_queue_inc_wrap(q->write_ptr);
1655         iwl_pcie_txq_inc_wr_ptr(trans, txq);
1656
1657         spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
1658
1659  out:
1660         spin_unlock_bh(&txq->lock);
1661  free_dup_buf:
1662         if (idx < 0)
1663                 kfree(dup_buf);
1664         return idx;
1665 }
1666
1667 /*
1668  * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them
1669  * @rxb: Rx buffer to reclaim
1670  */
1671 void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
1672                             struct iwl_rx_cmd_buffer *rxb)
1673 {
1674         struct iwl_rx_packet *pkt = rxb_addr(rxb);
1675         u16 sequence = le16_to_cpu(pkt->hdr.sequence);
1676         u8 group_id = iwl_cmd_groupid(pkt->hdr.group_id);
1677         u32 cmd_id;
1678         int txq_id = SEQ_TO_QUEUE(sequence);
1679         int index = SEQ_TO_INDEX(sequence);
1680         int cmd_index;
1681         struct iwl_device_cmd *cmd;
1682         struct iwl_cmd_meta *meta;
1683         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1684         struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
1685
1686         /* If a Tx command is being handled and it isn't in the actual
1687          * command queue then there a command routing bug has been introduced
1688          * in the queue management code. */
1689         if (WARN(txq_id != trans_pcie->cmd_queue,
1690                  "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n",
1691                  txq_id, trans_pcie->cmd_queue, sequence,
1692                  trans_pcie->txq[trans_pcie->cmd_queue].q.read_ptr,
1693                  trans_pcie->txq[trans_pcie->cmd_queue].q.write_ptr)) {
1694                 iwl_print_hex_error(trans, pkt, 32);
1695                 return;
1696         }
1697
1698         spin_lock_bh(&txq->lock);
1699
1700         cmd_index = get_cmd_index(&txq->q, index);
1701         cmd = txq->entries[cmd_index].cmd;
1702         meta = &txq->entries[cmd_index].meta;
1703         cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0);
1704
1705         iwl_pcie_tfd_unmap(trans, meta, &txq->tfds[index]);
1706
1707         /* Input error checking is done when commands are added to queue. */
1708         if (meta->flags & CMD_WANT_SKB) {
1709                 struct page *p = rxb_steal_page(rxb);
1710
1711                 meta->source->resp_pkt = pkt;
1712                 meta->source->_rx_page_addr = (unsigned long)page_address(p);
1713                 meta->source->_rx_page_order = trans_pcie->rx_page_order;
1714         }
1715
1716         if (meta->flags & CMD_WANT_ASYNC_CALLBACK)
1717                 iwl_op_mode_async_cb(trans->op_mode, cmd);
1718
1719         iwl_pcie_cmdq_reclaim(trans, txq_id, index);
1720
1721         if (!(meta->flags & CMD_ASYNC)) {
1722                 if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) {
1723                         IWL_WARN(trans,
1724                                  "HCMD_ACTIVE already clear for command %s\n",
1725                                  iwl_get_cmd_string(trans, cmd_id));
1726                 }
1727                 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1728                 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
1729                                iwl_get_cmd_string(trans, cmd_id));
1730                 wake_up(&trans_pcie->wait_command_queue);
1731         }
1732
1733         if (meta->flags & CMD_MAKE_TRANS_IDLE) {
1734                 IWL_DEBUG_INFO(trans, "complete %s - mark trans as idle\n",
1735                                iwl_get_cmd_string(trans, cmd->hdr.cmd));
1736                 set_bit(STATUS_TRANS_IDLE, &trans->status);
1737                 wake_up(&trans_pcie->d0i3_waitq);
1738         }
1739
1740         if (meta->flags & CMD_WAKE_UP_TRANS) {
1741                 IWL_DEBUG_INFO(trans, "complete %s - clear trans idle flag\n",
1742                                iwl_get_cmd_string(trans, cmd->hdr.cmd));
1743                 clear_bit(STATUS_TRANS_IDLE, &trans->status);
1744                 wake_up(&trans_pcie->d0i3_waitq);
1745         }
1746
1747         meta->flags = 0;
1748
1749         spin_unlock_bh(&txq->lock);
1750 }
1751
1752 #define HOST_COMPLETE_TIMEOUT   (2 * HZ)
1753
1754 static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans,
1755                                     struct iwl_host_cmd *cmd)
1756 {
1757         int ret;
1758
1759         /* An asynchronous command can not expect an SKB to be set. */
1760         if (WARN_ON(cmd->flags & CMD_WANT_SKB))
1761                 return -EINVAL;
1762
1763         ret = iwl_pcie_enqueue_hcmd(trans, cmd);
1764         if (ret < 0) {
1765                 IWL_ERR(trans,
1766                         "Error sending %s: enqueue_hcmd failed: %d\n",
1767                         iwl_get_cmd_string(trans, cmd->id), ret);
1768                 return ret;
1769         }
1770         return 0;
1771 }
1772
1773 static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans,
1774                                    struct iwl_host_cmd *cmd)
1775 {
1776         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1777         int cmd_idx;
1778         int ret;
1779
1780         IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n",
1781                        iwl_get_cmd_string(trans, cmd->id));
1782
1783         if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE,
1784                                   &trans->status),
1785                  "Command %s: a command is already active!\n",
1786                  iwl_get_cmd_string(trans, cmd->id)))
1787                 return -EIO;
1788
1789         IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n",
1790                        iwl_get_cmd_string(trans, cmd->id));
1791
1792         if (pm_runtime_suspended(&trans_pcie->pci_dev->dev)) {
1793                 ret = wait_event_timeout(trans_pcie->d0i3_waitq,
1794                                  pm_runtime_active(&trans_pcie->pci_dev->dev),
1795                                  msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT));
1796                 if (!ret) {
1797                         IWL_ERR(trans, "Timeout exiting D0i3 before hcmd\n");
1798                         return -ETIMEDOUT;
1799                 }
1800         }
1801
1802         cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd);
1803         if (cmd_idx < 0) {
1804                 ret = cmd_idx;
1805                 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1806                 IWL_ERR(trans,
1807                         "Error sending %s: enqueue_hcmd failed: %d\n",
1808                         iwl_get_cmd_string(trans, cmd->id), ret);
1809                 return ret;
1810         }
1811
1812         ret = wait_event_timeout(trans_pcie->wait_command_queue,
1813                                  !test_bit(STATUS_SYNC_HCMD_ACTIVE,
1814                                            &trans->status),
1815                                  HOST_COMPLETE_TIMEOUT);
1816         if (!ret) {
1817                 struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
1818                 struct iwl_queue *q = &txq->q;
1819
1820                 IWL_ERR(trans, "Error sending %s: time out after %dms.\n",
1821                         iwl_get_cmd_string(trans, cmd->id),
1822                         jiffies_to_msecs(HOST_COMPLETE_TIMEOUT));
1823
1824                 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n",
1825                         q->read_ptr, q->write_ptr);
1826
1827                 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1828                 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
1829                                iwl_get_cmd_string(trans, cmd->id));
1830                 ret = -ETIMEDOUT;
1831
1832                 iwl_force_nmi(trans);
1833                 iwl_trans_fw_error(trans);
1834
1835                 goto cancel;
1836         }
1837
1838         if (test_bit(STATUS_FW_ERROR, &trans->status)) {
1839                 IWL_ERR(trans, "FW error in SYNC CMD %s\n",
1840                         iwl_get_cmd_string(trans, cmd->id));
1841                 dump_stack();
1842                 ret = -EIO;
1843                 goto cancel;
1844         }
1845
1846         if (!(cmd->flags & CMD_SEND_IN_RFKILL) &&
1847             test_bit(STATUS_RFKILL, &trans->status)) {
1848                 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n");
1849                 ret = -ERFKILL;
1850                 goto cancel;
1851         }
1852
1853         if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) {
1854                 IWL_ERR(trans, "Error: Response NULL in '%s'\n",
1855                         iwl_get_cmd_string(trans, cmd->id));
1856                 ret = -EIO;
1857                 goto cancel;
1858         }
1859
1860         return 0;
1861
1862 cancel:
1863         if (cmd->flags & CMD_WANT_SKB) {
1864                 /*
1865                  * Cancel the CMD_WANT_SKB flag for the cmd in the
1866                  * TX cmd queue. Otherwise in case the cmd comes
1867                  * in later, it will possibly set an invalid
1868                  * address (cmd->meta.source).
1869                  */
1870                 trans_pcie->txq[trans_pcie->cmd_queue].
1871                         entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB;
1872         }
1873
1874         if (cmd->resp_pkt) {
1875                 iwl_free_resp(cmd);
1876                 cmd->resp_pkt = NULL;
1877         }
1878
1879         return ret;
1880 }
1881
1882 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
1883 {
1884         if (!(cmd->flags & CMD_SEND_IN_RFKILL) &&
1885             test_bit(STATUS_RFKILL, &trans->status)) {
1886                 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n",
1887                                   cmd->id);
1888                 return -ERFKILL;
1889         }
1890
1891         if (cmd->flags & CMD_ASYNC)
1892                 return iwl_pcie_send_hcmd_async(trans, cmd);
1893
1894         /* We still can fail on RFKILL that can be asserted while we wait */
1895         return iwl_pcie_send_hcmd_sync(trans, cmd);
1896 }
1897
1898 static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb,
1899                              struct iwl_txq *txq, u8 hdr_len,
1900                              struct iwl_cmd_meta *out_meta,
1901                              struct iwl_device_cmd *dev_cmd, u16 tb1_len)
1902 {
1903         struct iwl_queue *q = &txq->q;
1904         u16 tb2_len;
1905         int i;
1906
1907         /*
1908          * Set up TFD's third entry to point directly to remainder
1909          * of skb's head, if any
1910          */
1911         tb2_len = skb_headlen(skb) - hdr_len;
1912
1913         if (tb2_len > 0) {
1914                 dma_addr_t tb2_phys = dma_map_single(trans->dev,
1915                                                      skb->data + hdr_len,
1916                                                      tb2_len, DMA_TO_DEVICE);
1917                 if (unlikely(dma_mapping_error(trans->dev, tb2_phys))) {
1918                         iwl_pcie_tfd_unmap(trans, out_meta,
1919                                            &txq->tfds[q->write_ptr]);
1920                         return -EINVAL;
1921                 }
1922                 iwl_pcie_txq_build_tfd(trans, txq, tb2_phys, tb2_len, false);
1923         }
1924
1925         /* set up the remaining entries to point to the data */
1926         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1927                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1928                 dma_addr_t tb_phys;
1929                 int tb_idx;
1930
1931                 if (!skb_frag_size(frag))
1932                         continue;
1933
1934                 tb_phys = skb_frag_dma_map(trans->dev, frag, 0,
1935                                            skb_frag_size(frag), DMA_TO_DEVICE);
1936
1937                 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
1938                         iwl_pcie_tfd_unmap(trans, out_meta,
1939                                            &txq->tfds[q->write_ptr]);
1940                         return -EINVAL;
1941                 }
1942                 tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys,
1943                                                 skb_frag_size(frag), false);
1944
1945                 out_meta->flags |= BIT(tb_idx + CMD_TB_BITMAP_POS);
1946         }
1947
1948         trace_iwlwifi_dev_tx(trans->dev, skb,
1949                              &txq->tfds[txq->q.write_ptr],
1950                              sizeof(struct iwl_tfd),
1951                              &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len,
1952                              skb->data + hdr_len, tb2_len);
1953         trace_iwlwifi_dev_tx_data(trans->dev, skb,
1954                                   hdr_len, skb->len - hdr_len);
1955         return 0;
1956 }
1957
1958 #ifdef CONFIG_INET
1959 static struct iwl_tso_hdr_page *
1960 get_page_hdr(struct iwl_trans *trans, size_t len)
1961 {
1962         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1963         struct iwl_tso_hdr_page *p = this_cpu_ptr(trans_pcie->tso_hdr_page);
1964
1965         if (!p->page)
1966                 goto alloc;
1967
1968         /* enough room on this page */
1969         if (p->pos + len < (u8 *)page_address(p->page) + PAGE_SIZE)
1970                 return p;
1971
1972         /* We don't have enough room on this page, get a new one. */
1973         __free_page(p->page);
1974
1975 alloc:
1976         p->page = alloc_page(GFP_ATOMIC);
1977         if (!p->page)
1978                 return NULL;
1979         p->pos = page_address(p->page);
1980         return p;
1981 }
1982
1983 static void iwl_compute_pseudo_hdr_csum(void *iph, struct tcphdr *tcph,
1984                                         bool ipv6, unsigned int len)
1985 {
1986         if (ipv6) {
1987                 struct ipv6hdr *iphv6 = iph;
1988
1989                 tcph->check = ~csum_ipv6_magic(&iphv6->saddr, &iphv6->daddr,
1990                                                len + tcph->doff * 4,
1991                                                IPPROTO_TCP, 0);
1992         } else {
1993                 struct iphdr *iphv4 = iph;
1994
1995                 ip_send_check(iphv4);
1996                 tcph->check = ~csum_tcpudp_magic(iphv4->saddr, iphv4->daddr,
1997                                                  len + tcph->doff * 4,
1998                                                  IPPROTO_TCP, 0);
1999         }
2000 }
2001
2002 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
2003                                    struct iwl_txq *txq, u8 hdr_len,
2004                                    struct iwl_cmd_meta *out_meta,
2005                                    struct iwl_device_cmd *dev_cmd, u16 tb1_len)
2006 {
2007         struct iwl_trans_pcie *trans_pcie = txq->trans_pcie;
2008         struct ieee80211_hdr *hdr = (void *)skb->data;
2009         unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
2010         unsigned int mss = skb_shinfo(skb)->gso_size;
2011         struct iwl_queue *q = &txq->q;
2012         u16 length, iv_len, amsdu_pad;
2013         u8 *start_hdr;
2014         struct iwl_tso_hdr_page *hdr_page;
2015         struct page **page_ptr;
2016         int ret;
2017         struct tso_t tso;
2018
2019         /* if the packet is protected, then it must be CCMP or GCMP */
2020         BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN);
2021         iv_len = ieee80211_has_protected(hdr->frame_control) ?
2022                 IEEE80211_CCMP_HDR_LEN : 0;
2023
2024         trace_iwlwifi_dev_tx(trans->dev, skb,
2025                              &txq->tfds[txq->q.write_ptr],
2026                              sizeof(struct iwl_tfd),
2027                              &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len,
2028                              NULL, 0);
2029
2030         ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb);
2031         snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb);
2032         total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len;
2033         amsdu_pad = 0;
2034
2035         /* total amount of header we may need for this A-MSDU */
2036         hdr_room = DIV_ROUND_UP(total_len, mss) *
2037                 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len;
2038
2039         /* Our device supports 9 segments at most, it will fit in 1 page */
2040         hdr_page = get_page_hdr(trans, hdr_room);
2041         if (!hdr_page)
2042                 return -ENOMEM;
2043
2044         get_page(hdr_page->page);
2045         start_hdr = hdr_page->pos;
2046         page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs);
2047         *page_ptr = hdr_page->page;
2048         memcpy(hdr_page->pos, skb->data + hdr_len, iv_len);
2049         hdr_page->pos += iv_len;
2050
2051         /*
2052          * Pull the ieee80211 header + IV to be able to use TSO core,
2053          * we will restore it for the tx_status flow.
2054          */
2055         skb_pull(skb, hdr_len + iv_len);
2056
2057         tso_start(skb, &tso);
2058
2059         while (total_len) {
2060                 /* this is the data left for this subframe */
2061                 unsigned int data_left =
2062                         min_t(unsigned int, mss, total_len);
2063                 struct sk_buff *csum_skb = NULL;
2064                 unsigned int hdr_tb_len;
2065                 dma_addr_t hdr_tb_phys;
2066                 struct tcphdr *tcph;
2067                 u8 *iph;
2068
2069                 total_len -= data_left;
2070
2071                 memset(hdr_page->pos, 0, amsdu_pad);
2072                 hdr_page->pos += amsdu_pad;
2073                 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen +
2074                                   data_left)) & 0x3;
2075                 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr));
2076                 hdr_page->pos += ETH_ALEN;
2077                 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr));
2078                 hdr_page->pos += ETH_ALEN;
2079
2080                 length = snap_ip_tcp_hdrlen + data_left;
2081                 *((__be16 *)hdr_page->pos) = cpu_to_be16(length);
2082                 hdr_page->pos += sizeof(length);
2083
2084                 /*
2085                  * This will copy the SNAP as well which will be considered
2086                  * as MAC header.
2087                  */
2088                 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len);
2089                 iph = hdr_page->pos + 8;
2090                 tcph = (void *)(iph + ip_hdrlen);
2091
2092                 /* For testing on current hardware only */
2093                 if (trans_pcie->sw_csum_tx) {
2094                         csum_skb = alloc_skb(data_left + tcp_hdrlen(skb),
2095                                              GFP_ATOMIC);
2096                         if (!csum_skb) {
2097                                 ret = -ENOMEM;
2098                                 goto out_unmap;
2099                         }
2100
2101                         iwl_compute_pseudo_hdr_csum(iph, tcph,
2102                                                     skb->protocol ==
2103                                                         htons(ETH_P_IPV6),
2104                                                     data_left);
2105
2106                         memcpy(skb_put(csum_skb, tcp_hdrlen(skb)),
2107                                tcph, tcp_hdrlen(skb));
2108                         skb_set_transport_header(csum_skb, 0);
2109                         csum_skb->csum_start =
2110                                 (unsigned char *)tcp_hdr(csum_skb) -
2111                                                  csum_skb->head;
2112                 }
2113
2114                 hdr_page->pos += snap_ip_tcp_hdrlen;
2115
2116                 hdr_tb_len = hdr_page->pos - start_hdr;
2117                 hdr_tb_phys = dma_map_single(trans->dev, start_hdr,
2118                                              hdr_tb_len, DMA_TO_DEVICE);
2119                 if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) {
2120                         dev_kfree_skb(csum_skb);
2121                         ret = -EINVAL;
2122                         goto out_unmap;
2123                 }
2124                 iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys,
2125                                        hdr_tb_len, false);
2126                 trace_iwlwifi_dev_tx_tso_chunk(trans->dev, start_hdr,
2127                                                hdr_tb_len);
2128
2129                 /* prepare the start_hdr for the next subframe */
2130                 start_hdr = hdr_page->pos;
2131
2132                 /* put the payload */
2133                 while (data_left) {
2134                         unsigned int size = min_t(unsigned int, tso.size,
2135                                                   data_left);
2136                         dma_addr_t tb_phys;
2137
2138                         if (trans_pcie->sw_csum_tx)
2139                                 memcpy(skb_put(csum_skb, size), tso.data, size);
2140
2141                         tb_phys = dma_map_single(trans->dev, tso.data,
2142                                                  size, DMA_TO_DEVICE);
2143                         if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
2144                                 dev_kfree_skb(csum_skb);
2145                                 ret = -EINVAL;
2146                                 goto out_unmap;
2147                         }
2148
2149                         iwl_pcie_txq_build_tfd(trans, txq, tb_phys,
2150                                                size, false);
2151                         trace_iwlwifi_dev_tx_tso_chunk(trans->dev, tso.data,
2152                                                        size);
2153
2154                         data_left -= size;
2155                         tso_build_data(skb, &tso, size);
2156                 }
2157
2158                 /* For testing on early hardware only */
2159                 if (trans_pcie->sw_csum_tx) {
2160                         __wsum csum;
2161
2162                         csum = skb_checksum(csum_skb,
2163                                             skb_checksum_start_offset(csum_skb),
2164                                             csum_skb->len -
2165                                             skb_checksum_start_offset(csum_skb),
2166                                             0);
2167                         dev_kfree_skb(csum_skb);
2168                         dma_sync_single_for_cpu(trans->dev, hdr_tb_phys,
2169                                                 hdr_tb_len, DMA_TO_DEVICE);
2170                         tcph->check = csum_fold(csum);
2171                         dma_sync_single_for_device(trans->dev, hdr_tb_phys,
2172                                                    hdr_tb_len, DMA_TO_DEVICE);
2173                 }
2174         }
2175
2176         /* re -add the WiFi header and IV */
2177         skb_push(skb, hdr_len + iv_len);
2178
2179         return 0;
2180
2181 out_unmap:
2182         iwl_pcie_tfd_unmap(trans, out_meta, &txq->tfds[q->write_ptr]);
2183         return ret;
2184 }
2185 #else /* CONFIG_INET */
2186 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
2187                                    struct iwl_txq *txq, u8 hdr_len,
2188                                    struct iwl_cmd_meta *out_meta,
2189                                    struct iwl_device_cmd *dev_cmd, u16 tb1_len)
2190 {
2191         /* No A-MSDU without CONFIG_INET */
2192         WARN_ON(1);
2193
2194         return -1;
2195 }
2196 #endif /* CONFIG_INET */
2197
2198 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
2199                       struct iwl_device_cmd *dev_cmd, int txq_id)
2200 {
2201         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2202         struct ieee80211_hdr *hdr;
2203         struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload;
2204         struct iwl_cmd_meta *out_meta;
2205         struct iwl_txq *txq;
2206         struct iwl_queue *q;
2207         dma_addr_t tb0_phys, tb1_phys, scratch_phys;
2208         void *tb1_addr;
2209         u16 len, tb1_len;
2210         bool wait_write_ptr;
2211         __le16 fc;
2212         u8 hdr_len;
2213         u16 wifi_seq;
2214         bool amsdu;
2215
2216         txq = &trans_pcie->txq[txq_id];
2217         q = &txq->q;
2218
2219         if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used),
2220                       "TX on unused queue %d\n", txq_id))
2221                 return -EINVAL;
2222
2223         if (unlikely(trans_pcie->sw_csum_tx &&
2224                      skb->ip_summed == CHECKSUM_PARTIAL)) {
2225                 int offs = skb_checksum_start_offset(skb);
2226                 int csum_offs = offs + skb->csum_offset;
2227                 __wsum csum;
2228
2229                 if (skb_ensure_writable(skb, csum_offs + sizeof(__sum16)))
2230                         return -1;
2231
2232                 csum = skb_checksum(skb, offs, skb->len - offs, 0);
2233                 *(__sum16 *)(skb->data + csum_offs) = csum_fold(csum);
2234
2235                 skb->ip_summed = CHECKSUM_UNNECESSARY;
2236         }
2237
2238         if (skb_is_nonlinear(skb) &&
2239             skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS &&
2240             __skb_linearize(skb))
2241                 return -ENOMEM;
2242
2243         /* mac80211 always puts the full header into the SKB's head,
2244          * so there's no need to check if it's readable there
2245          */
2246         hdr = (struct ieee80211_hdr *)skb->data;
2247         fc = hdr->frame_control;
2248         hdr_len = ieee80211_hdrlen(fc);
2249
2250         spin_lock(&txq->lock);
2251
2252         if (iwl_queue_space(q) < q->high_mark) {
2253                 iwl_stop_queue(trans, txq);
2254
2255                 /* don't put the packet on the ring, if there is no room */
2256                 if (unlikely(iwl_queue_space(q) < 3)) {
2257                         struct iwl_device_cmd **dev_cmd_ptr;
2258
2259                         dev_cmd_ptr = (void *)((u8 *)skb->cb +
2260                                                trans_pcie->dev_cmd_offs);
2261
2262                         *dev_cmd_ptr = dev_cmd;
2263                         __skb_queue_tail(&txq->overflow_q, skb);
2264
2265                         spin_unlock(&txq->lock);
2266                         return 0;
2267                 }
2268         }
2269
2270         /* In AGG mode, the index in the ring must correspond to the WiFi
2271          * sequence number. This is a HW requirements to help the SCD to parse
2272          * the BA.
2273          * Check here that the packets are in the right place on the ring.
2274          */
2275         wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
2276         WARN_ONCE(txq->ampdu &&
2277                   (wifi_seq & 0xff) != q->write_ptr,
2278                   "Q: %d WiFi Seq %d tfdNum %d",
2279                   txq_id, wifi_seq, q->write_ptr);
2280
2281         /* Set up driver data for this TFD */
2282         txq->entries[q->write_ptr].skb = skb;
2283         txq->entries[q->write_ptr].cmd = dev_cmd;
2284
2285         dev_cmd->hdr.sequence =
2286                 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
2287                             INDEX_TO_SEQ(q->write_ptr)));
2288
2289         tb0_phys = iwl_pcie_get_first_tb_dma(txq, q->write_ptr);
2290         scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) +
2291                        offsetof(struct iwl_tx_cmd, scratch);
2292
2293         tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
2294         tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);
2295
2296         /* Set up first empty entry in queue's array of Tx/cmd buffers */
2297         out_meta = &txq->entries[q->write_ptr].meta;
2298         out_meta->flags = 0;
2299
2300         /*
2301          * The second TB (tb1) points to the remainder of the TX command
2302          * and the 802.11 header - dword aligned size
2303          * (This calculation modifies the TX command, so do it before the
2304          * setup of the first TB)
2305          */
2306         len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) +
2307               hdr_len - IWL_FIRST_TB_SIZE;
2308         /* do not align A-MSDU to dword as the subframe header aligns it */
2309         amsdu = ieee80211_is_data_qos(fc) &&
2310                 (*ieee80211_get_qos_ctl(hdr) &
2311                  IEEE80211_QOS_CTL_A_MSDU_PRESENT);
2312         if (trans_pcie->sw_csum_tx || !amsdu) {
2313                 tb1_len = ALIGN(len, 4);
2314                 /* Tell NIC about any 2-byte padding after MAC header */
2315                 if (tb1_len != len)
2316                         tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK;
2317         } else {
2318                 tb1_len = len;
2319         }
2320
2321         /* The first TB points to bi-directional DMA data */
2322         memcpy(&txq->first_tb_bufs[q->write_ptr], &dev_cmd->hdr,
2323                IWL_FIRST_TB_SIZE);
2324         iwl_pcie_txq_build_tfd(trans, txq, tb0_phys,
2325                                IWL_FIRST_TB_SIZE, true);
2326
2327         /* there must be data left over for TB1 or this code must be changed */
2328         BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE);
2329
2330         /* map the data for TB1 */
2331         tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE;
2332         tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE);
2333         if (unlikely(dma_mapping_error(trans->dev, tb1_phys)))
2334                 goto out_err;
2335         iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false);
2336
2337         if (amsdu) {
2338                 if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len,
2339                                                      out_meta, dev_cmd,
2340                                                      tb1_len)))
2341                         goto out_err;
2342         } else if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len,
2343                                        out_meta, dev_cmd, tb1_len))) {
2344                 goto out_err;
2345         }
2346
2347         /* Set up entry for this TFD in Tx byte-count array */
2348         iwl_pcie_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len));
2349
2350         wait_write_ptr = ieee80211_has_morefrags(fc);
2351
2352         /* start timer if queue currently empty */
2353         if (q->read_ptr == q->write_ptr) {
2354                 if (txq->wd_timeout) {
2355                         /*
2356                          * If the TXQ is active, then set the timer, if not,
2357                          * set the timer in remainder so that the timer will
2358                          * be armed with the right value when the station will
2359                          * wake up.
2360                          */
2361                         if (!txq->frozen)
2362                                 mod_timer(&txq->stuck_timer,
2363                                           jiffies + txq->wd_timeout);
2364                         else
2365                                 txq->frozen_expiry_remainder = txq->wd_timeout;
2366                 }
2367                 IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", q->id);
2368                 iwl_trans_ref(trans);
2369         }
2370
2371         /* Tell device the write index *just past* this latest filled TFD */
2372         q->write_ptr = iwl_queue_inc_wrap(q->write_ptr);
2373         if (!wait_write_ptr)
2374                 iwl_pcie_txq_inc_wr_ptr(trans, txq);
2375
2376         /*
2377          * At this point the frame is "transmitted" successfully
2378          * and we will get a TX status notification eventually.
2379          */
2380         spin_unlock(&txq->lock);
2381         return 0;
2382 out_err:
2383         spin_unlock(&txq->lock);
2384         return -1;
2385 }