drivers/net/vmxnet3/vmxnet3_drv.c

   1 /*
   2  * Linux driver for VMware's vmxnet3 ethernet NIC.
   3  *
   4  * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation; version 2 of the License and no later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  13  * NON INFRINGEMENT. See the GNU General Public License for more
  14  * details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * The full GNU General Public License is included in this distribution in
  21  * the file called "COPYING".
  22  *
  23  * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
  24  *
  25  */
  26
  27 #include <linux/module.h>
  28 #include <net/ip6_checksum.h>
  29
  30 #include "vmxnet3_int.h"
  31
  32 char vmxnet3_driver_name[] = "vmxnet3";
  33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
  34
  35 /*
  36  * PCI Device ID Table
  37  * Last entry must be all 0s
  38  */
  39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
  40         {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
  41         {0}
  42 };
  43
  44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
  45
  46 static int enable_mq = 1;
  47
  48 static void
  49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
  50
  51 /*
  52  *    Enable/Disable the given intr
  53  */
  54 static void
  55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
  56 {
  57         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
  58 }
  59
  60
  61 static void
  62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
  63 {
  64         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
  65 }
  66
  67
  68 /*
  69  *    Enable/Disable all intrs used by the device
  70  */
  71 static void
  72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
  73 {
  74         int i;
  75
  76         for (i = 0; i < adapter->intr.num_intrs; i++)
  77                 vmxnet3_enable_intr(adapter, i);
  78         adapter->shared->devRead.intrConf.intrCtrl &=
  79                                         cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
  80 }
  81
  82
  83 static void
  84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
  85 {
  86         int i;
  87
  88         adapter->shared->devRead.intrConf.intrCtrl |=
  89                                         cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
  90         for (i = 0; i < adapter->intr.num_intrs; i++)
  91                 vmxnet3_disable_intr(adapter, i);
  92 }
  93
  94
  95 static void
  96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
  97 {
  98         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
  99 }
 100
 101
 102 static bool
 103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 104 {
 105         return tq->stopped;
 106 }
 107
 108
 109 static void
 110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 111 {
 112         tq->stopped = false;
 113         netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
 114 }
 115
 116
 117 static void
 118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 119 {
 120         tq->stopped = false;
 121         netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
 122 }
 123
 124
 125 static void
 126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 127 {
 128         tq->stopped = true;
 129         tq->num_stop++;
 130         netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
 131 }
 132
 133
 134 /*
 135  * Check the link state. This may start or stop the tx queue.
 136  */
 137 static void
 138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
 139 {
 140         u32 ret;
 141         int i;
 142         unsigned long flags;
 143
 144         spin_lock_irqsave(&adapter->cmd_lock, flags);
 145         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
 146         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
 147         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 148
 149         adapter->link_speed = ret >> 16;
 150         if (ret & 1) { /* Link is up. */
 151                 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
 152                             adapter->link_speed);
 153                 netif_carrier_on(adapter->netdev);
 154
 155                 if (affectTxQueue) {
 156                         for (i = 0; i < adapter->num_tx_queues; i++)
 157                                 vmxnet3_tq_start(&adapter->tx_queue[i],
 158                                                  adapter);
 159                 }
 160         } else {
 161                 netdev_info(adapter->netdev, "NIC Link is Down\n");
 162                 netif_carrier_off(adapter->netdev);
 163
 164                 if (affectTxQueue) {
 165                         for (i = 0; i < adapter->num_tx_queues; i++)
 166                                 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
 167                 }
 168         }
 169 }
 170
 171 static void
 172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
 173 {
 174         int i;
 175         unsigned long flags;
 176         u32 events = le32_to_cpu(adapter->shared->ecr);
 177         if (!events)
 178                 return;
 179
 180         vmxnet3_ack_events(adapter, events);
 181
 182         /* Check if link state has changed */
 183         if (events & VMXNET3_ECR_LINK)
 184                 vmxnet3_check_link(adapter, true);
 185
 186         /* Check if there is an error on xmit/recv queues */
 187         if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
 188                 spin_lock_irqsave(&adapter->cmd_lock, flags);
 189                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 190                                        VMXNET3_CMD_GET_QUEUE_STATUS);
 191                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 192
 193                 for (i = 0; i < adapter->num_tx_queues; i++)
 194                         if (adapter->tqd_start[i].status.stopped)
 195                                 dev_err(&adapter->netdev->dev,
 196                                         "%s: tq[%d] error 0x%x\n",
 197                                         adapter->netdev->name, i, le32_to_cpu(
 198                                         adapter->tqd_start[i].status.error));
 199                 for (i = 0; i < adapter->num_rx_queues; i++)
 200                         if (adapter->rqd_start[i].status.stopped)
 201                                 dev_err(&adapter->netdev->dev,
 202                                         "%s: rq[%d] error 0x%x\n",
 203                                         adapter->netdev->name, i,
 204                                         adapter->rqd_start[i].status.error);
 205
 206                 schedule_work(&adapter->work);
 207         }
 208 }
 209
 210 #ifdef __BIG_ENDIAN_BITFIELD
 211 /*
 212  * The device expects the bitfields in shared structures to be written in
 213  * little endian. When CPU is big endian, the following routines are used to
 214  * correctly read and write into ABI.
 215  * The general technique used here is : double word bitfields are defined in
 216  * opposite order for big endian architecture. Then before reading them in
 217  * driver the complete double word is translated using le32_to_cpu. Similarly
 218  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
 219  * double words into required format.
 220  * In order to avoid touching bits in shared structure more than once, temporary
 221  * descriptors are used. These are passed as srcDesc to following functions.
 222  */
 223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
 224                                 struct Vmxnet3_RxDesc *dstDesc)
 225 {
 226         u32 *src = (u32 *)srcDesc + 2;
 227         u32 *dst = (u32 *)dstDesc + 2;
 228         dstDesc->addr = le64_to_cpu(srcDesc->addr);
 229         *dst = le32_to_cpu(*src);
 230         dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
 231 }
 232
 233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
 234                                struct Vmxnet3_TxDesc *dstDesc)
 235 {
 236         int i;
 237         u32 *src = (u32 *)(srcDesc + 1);
 238         u32 *dst = (u32 *)(dstDesc + 1);
 239
 240         /* Working backwards so that the gen bit is set at the end. */
 241         for (i = 2; i > 0; i--) {
 242                 src--;
 243                 dst--;
 244                 *dst = cpu_to_le32(*src);
 245         }
 246 }
 247
 248
 249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
 250                                 struct Vmxnet3_RxCompDesc *dstDesc)
 251 {
 252         int i = 0;
 253         u32 *src = (u32 *)srcDesc;
 254         u32 *dst = (u32 *)dstDesc;
 255         for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
 256                 *dst = le32_to_cpu(*src);
 257                 src++;
 258                 dst++;
 259         }
 260 }
 261
 262
 263 /* Used to read bitfield values from double words. */
 264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
 265 {
 266         u32 temp = le32_to_cpu(*bitfield);
 267         u32 mask = ((1 << size) - 1) << pos;
 268         temp &= mask;
 269         temp >>= pos;
 270         return temp;
 271 }
 272
 273
 274
 275 #endif  /* __BIG_ENDIAN_BITFIELD */
 276
 277 #ifdef __BIG_ENDIAN_BITFIELD
 278
 279 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
 280                         txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
 281                         VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
 282 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
 283                         txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
 284                         VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
 285 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
 286                         VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
 287                         VMXNET3_TCD_GEN_SIZE)
 288 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
 289                         VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
 290 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
 291                         (dstrcd) = (tmp); \
 292                         vmxnet3_RxCompToCPU((rcd), (tmp)); \
 293                 } while (0)
 294 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
 295                         (dstrxd) = (tmp); \
 296                         vmxnet3_RxDescToCPU((rxd), (tmp)); \
 297                 } while (0)
 298
 299 #else
 300
 301 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
 302 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
 303 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
 304 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
 305 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
 306 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
 307
 308 #endif /* __BIG_ENDIAN_BITFIELD  */
 309
 310
 311 static void
 312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
 313                      struct pci_dev *pdev)
 314 {
 315         if (tbi->map_type == VMXNET3_MAP_SINGLE)
 316                 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
 317                                  PCI_DMA_TODEVICE);
 318         else if (tbi->map_type == VMXNET3_MAP_PAGE)
 319                 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
 320                                PCI_DMA_TODEVICE);
 321         else
 322                 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
 323
 324         tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
 325 }
 326
 327
 328 static int
 329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
 330                   struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
 331 {
 332         struct sk_buff *skb;
 333         int entries = 0;
 334
 335         /* no out of order completion */
 336         BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
 337         BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
 338
 339         skb = tq->buf_info[eop_idx].skb;
 340         BUG_ON(skb == NULL);
 341         tq->buf_info[eop_idx].skb = NULL;
 342
 343         VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
 344
 345         while (tq->tx_ring.next2comp != eop_idx) {
 346                 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
 347                                      pdev);
 348
 349                 /* update next2comp w/o tx_lock. Since we are marking more,
 350                  * instead of less, tx ring entries avail, the worst case is
 351                  * that the tx routine incorrectly re-queues a pkt due to
 352                  * insufficient tx ring entries.
 353                  */
 354                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 355                 entries++;
 356         }
 357
 358         dev_kfree_skb_any(skb);
 359         return entries;
 360 }
 361
 362
 363 static int
 364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
 365                         struct vmxnet3_adapter *adapter)
 366 {
 367         int completed = 0;
 368         union Vmxnet3_GenericDesc *gdesc;
 369
 370         gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 371         while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
 372                 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
 373                                                &gdesc->tcd), tq, adapter->pdev,
 374                                                adapter);
 375
 376                 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
 377                 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 378         }
 379
 380         if (completed) {
 381                 spin_lock(&tq->tx_lock);
 382                 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
 383                              vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
 384                              VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
 385                              netif_carrier_ok(adapter->netdev))) {
 386                         vmxnet3_tq_wake(tq, adapter);
 387                 }
 388                 spin_unlock(&tq->tx_lock);
 389         }
 390         return completed;
 391 }
 392
 393
 394 static void
 395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
 396                    struct vmxnet3_adapter *adapter)
 397 {
 398         int i;
 399
 400         while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
 401                 struct vmxnet3_tx_buf_info *tbi;
 402
 403                 tbi = tq->buf_info + tq->tx_ring.next2comp;
 404
 405                 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
 406                 if (tbi->skb) {
 407                         dev_kfree_skb_any(tbi->skb);
 408                         tbi->skb = NULL;
 409                 }
 410                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 411         }
 412
 413         /* sanity check, verify all buffers are indeed unmapped and freed */
 414         for (i = 0; i < tq->tx_ring.size; i++) {
 415                 BUG_ON(tq->buf_info[i].skb != NULL ||
 416                        tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
 417         }
 418
 419         tq->tx_ring.gen = VMXNET3_INIT_GEN;
 420         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 421
 422         tq->comp_ring.gen = VMXNET3_INIT_GEN;
 423         tq->comp_ring.next2proc = 0;
 424 }
 425
 426
 427 static void
 428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
 429                    struct vmxnet3_adapter *adapter)
 430 {
 431         if (tq->tx_ring.base) {
 432                 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
 433                                   sizeof(struct Vmxnet3_TxDesc),
 434                                   tq->tx_ring.base, tq->tx_ring.basePA);
 435                 tq->tx_ring.base = NULL;
 436         }
 437         if (tq->data_ring.base) {
 438                 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
 439                                   sizeof(struct Vmxnet3_TxDataDesc),
 440                                   tq->data_ring.base, tq->data_ring.basePA);
 441                 tq->data_ring.base = NULL;
 442         }
 443         if (tq->comp_ring.base) {
 444                 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
 445                                   sizeof(struct Vmxnet3_TxCompDesc),
 446                                   tq->comp_ring.base, tq->comp_ring.basePA);
 447                 tq->comp_ring.base = NULL;
 448         }
 449         if (tq->buf_info) {
 450                 dma_free_coherent(&adapter->pdev->dev,
 451                                   tq->tx_ring.size * sizeof(tq->buf_info[0]),
 452                                   tq->buf_info, tq->buf_info_pa);
 453                 tq->buf_info = NULL;
 454         }
 455 }
 456
 457
 458 /* Destroy all tx queues */
 459 void
 460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
 461 {
 462         int i;
 463
 464         for (i = 0; i < adapter->num_tx_queues; i++)
 465                 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
 466 }
 467
 468
 469 static void
 470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
 471                 struct vmxnet3_adapter *adapter)
 472 {
 473         int i;
 474
 475         /* reset the tx ring contents to 0 and reset the tx ring states */
 476         memset(tq->tx_ring.base, 0, tq->tx_ring.size *
 477                sizeof(struct Vmxnet3_TxDesc));
 478         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 479         tq->tx_ring.gen = VMXNET3_INIT_GEN;
 480
 481         memset(tq->data_ring.base, 0, tq->data_ring.size *
 482                sizeof(struct Vmxnet3_TxDataDesc));
 483
 484         /* reset the tx comp ring contents to 0 and reset comp ring states */
 485         memset(tq->comp_ring.base, 0, tq->comp_ring.size *
 486                sizeof(struct Vmxnet3_TxCompDesc));
 487         tq->comp_ring.next2proc = 0;
 488         tq->comp_ring.gen = VMXNET3_INIT_GEN;
 489
 490         /* reset the bookkeeping data */
 491         memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
 492         for (i = 0; i < tq->tx_ring.size; i++)
 493                 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
 494
 495         /* stats are not reset */
 496 }
 497
 498
 499 static int
 500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
 501                   struct vmxnet3_adapter *adapter)
 502 {
 503         size_t sz;
 504
 505         BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
 506                tq->comp_ring.base || tq->buf_info);
 507
 508         tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 509                         tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
 510                         &tq->tx_ring.basePA, GFP_KERNEL);
 511         if (!tq->tx_ring.base) {
 512                 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
 513                 goto err;
 514         }
 515
 516         tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 517                         tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
 518                         &tq->data_ring.basePA, GFP_KERNEL);
 519         if (!tq->data_ring.base) {
 520                 netdev_err(adapter->netdev, "failed to allocate data ring\n");
 521                 goto err;
 522         }
 523
 524         tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 525                         tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
 526                         &tq->comp_ring.basePA, GFP_KERNEL);
 527         if (!tq->comp_ring.base) {
 528                 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
 529                 goto err;
 530         }
 531
 532         sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
 533         tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
 534                                            &tq->buf_info_pa, GFP_KERNEL);
 535         if (!tq->buf_info)
 536                 goto err;
 537
 538         return 0;
 539
 540 err:
 541         vmxnet3_tq_destroy(tq, adapter);
 542         return -ENOMEM;
 543 }
 544
 545 static void
 546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
 547 {
 548         int i;
 549
 550         for (i = 0; i < adapter->num_tx_queues; i++)
 551                 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
 552 }
 553
 554 /*
 555  *    starting from ring->next2fill, allocate rx buffers for the given ring
 556  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
 557  *    are allocated or allocation fails
 558  */
 559
 560 static int
 561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
 562                         int num_to_alloc, struct vmxnet3_adapter *adapter)
 563 {
 564         int num_allocated = 0;
 565         struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
 566         struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
 567         u32 val;
 568
 569         while (num_allocated <= num_to_alloc) {
 570                 struct vmxnet3_rx_buf_info *rbi;
 571                 union Vmxnet3_GenericDesc *gd;
 572
 573                 rbi = rbi_base + ring->next2fill;
 574                 gd = ring->base + ring->next2fill;
 575
 576                 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
 577                         if (rbi->skb == NULL) {
 578                                 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
 579                                                                        rbi->len,
 580                                                                        GFP_KERNEL);
 581                                 if (unlikely(rbi->skb == NULL)) {
 582                                         rq->stats.rx_buf_alloc_failure++;
 583                                         break;
 584                                 }
 585
 586                                 rbi->dma_addr = dma_map_single(
 587                                                 &adapter->pdev->dev,
 588                                                 rbi->skb->data, rbi->len,
 589                                                 PCI_DMA_FROMDEVICE);
 590                         } else {
 591                                 /* rx buffer skipped by the device */
 592                         }
 593                         val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
 594                 } else {
 595                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
 596                                rbi->len  != PAGE_SIZE);
 597
 598                         if (rbi->page == NULL) {
 599                                 rbi->page = alloc_page(GFP_ATOMIC);
 600                                 if (unlikely(rbi->page == NULL)) {
 601                                         rq->stats.rx_buf_alloc_failure++;
 602                                         break;
 603                                 }
 604                                 rbi->dma_addr = dma_map_page(
 605                                                 &adapter->pdev->dev,
 606                                                 rbi->page, 0, PAGE_SIZE,
 607                                                 PCI_DMA_FROMDEVICE);
 608                         } else {
 609                                 /* rx buffers skipped by the device */
 610                         }
 611                         val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
 612                 }
 613
 614                 BUG_ON(rbi->dma_addr == 0);
 615                 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
 616                 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
 617                                            | val | rbi->len);
 618
 619                 /* Fill the last buffer but dont mark it ready, or else the
 620                  * device will think that the queue is full */
 621                 if (num_allocated == num_to_alloc)
 622                         break;
 623
 624                 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
 625                 num_allocated++;
 626                 vmxnet3_cmd_ring_adv_next2fill(ring);
 627         }
 628
 629         netdev_dbg(adapter->netdev,
 630                 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
 631                 num_allocated, ring->next2fill, ring->next2comp);
 632
 633         /* so that the device can distinguish a full ring and an empty ring */
 634         BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
 635
 636         return num_allocated;
 637 }
 638
 639
 640 static void
 641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
 642                     struct vmxnet3_rx_buf_info *rbi)
 643 {
 644         struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
 645                 skb_shinfo(skb)->nr_frags;
 646
 647         BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
 648
 649         __skb_frag_set_page(frag, rbi->page);
 650         frag->page_offset = 0;
 651         skb_frag_size_set(frag, rcd->len);
 652         skb->data_len += rcd->len;
 653         skb->truesize += PAGE_SIZE;
 654         skb_shinfo(skb)->nr_frags++;
 655 }
 656
 657
 658 static void
 659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
 660                 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
 661                 struct vmxnet3_adapter *adapter)
 662 {
 663         u32 dw2, len;
 664         unsigned long buf_offset;
 665         int i;
 666         union Vmxnet3_GenericDesc *gdesc;
 667         struct vmxnet3_tx_buf_info *tbi = NULL;
 668
 669         BUG_ON(ctx->copy_size > skb_headlen(skb));
 670
 671         /* use the previous gen bit for the SOP desc */
 672         dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 673
 674         ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
 675         gdesc = ctx->sop_txd; /* both loops below can be skipped */
 676
 677         /* no need to map the buffer if headers are copied */
 678         if (ctx->copy_size) {
 679                 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
 680                                         tq->tx_ring.next2fill *
 681                                         sizeof(struct Vmxnet3_TxDataDesc));
 682                 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
 683                 ctx->sop_txd->dword[3] = 0;
 684
 685                 tbi = tq->buf_info + tq->tx_ring.next2fill;
 686                 tbi->map_type = VMXNET3_MAP_NONE;
 687
 688                 netdev_dbg(adapter->netdev,
 689                         "txd[%u]: 0x%Lx 0x%x 0x%x\n",
 690                         tq->tx_ring.next2fill,
 691                         le64_to_cpu(ctx->sop_txd->txd.addr),
 692                         ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
 693                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 694
 695                 /* use the right gen for non-SOP desc */
 696                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 697         }
 698
 699         /* linear part can use multiple tx desc if it's big */
 700         len = skb_headlen(skb) - ctx->copy_size;
 701         buf_offset = ctx->copy_size;
 702         while (len) {
 703                 u32 buf_size;
 704
 705                 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
 706                         buf_size = len;
 707                         dw2 |= len;
 708                 } else {
 709                         buf_size = VMXNET3_MAX_TX_BUF_SIZE;
 710                         /* spec says that for TxDesc.len, 0 == 2^14 */
 711                 }
 712
 713                 tbi = tq->buf_info + tq->tx_ring.next2fill;
 714                 tbi->map_type = VMXNET3_MAP_SINGLE;
 715                 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
 716                                 skb->data + buf_offset, buf_size,
 717                                 PCI_DMA_TODEVICE);
 718
 719                 tbi->len = buf_size;
 720
 721                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 722                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 723
 724                 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
 725                 gdesc->dword[2] = cpu_to_le32(dw2);
 726                 gdesc->dword[3] = 0;
 727
 728                 netdev_dbg(adapter->netdev,
 729                         "txd[%u]: 0x%Lx 0x%x 0x%x\n",
 730                         tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
 731                         le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
 732                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 733                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 734
 735                 len -= buf_size;
 736                 buf_offset += buf_size;
 737         }
 738
 739         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 740                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 741                 u32 buf_size;
 742
 743                 buf_offset = 0;
 744                 len = skb_frag_size(frag);
 745                 while (len) {
 746                         tbi = tq->buf_info + tq->tx_ring.next2fill;
 747                         if (len < VMXNET3_MAX_TX_BUF_SIZE) {
 748                                 buf_size = len;
 749                                 dw2 |= len;
 750                         } else {
 751                                 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
 752                                 /* spec says that for TxDesc.len, 0 == 2^14 */
 753                         }
 754                         tbi->map_type = VMXNET3_MAP_PAGE;
 755                         tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
 756                                                          buf_offset, buf_size,
 757                                                          DMA_TO_DEVICE);
 758
 759                         tbi->len = buf_size;
 760
 761                         gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 762                         BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 763
 764                         gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
 765                         gdesc->dword[2] = cpu_to_le32(dw2);
 766                         gdesc->dword[3] = 0;
 767
 768                         netdev_dbg(adapter->netdev,
 769                                 "txd[%u]: 0x%llu %u %u\n",
 770                                 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
 771                                 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
 772                         vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 773                         dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 774
 775                         len -= buf_size;
 776                         buf_offset += buf_size;
 777                 }
 778         }
 779
 780         ctx->eop_txd = gdesc;
 781
 782         /* set the last buf_info for the pkt */
 783         tbi->skb = skb;
 784         tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
 785 }
 786
 787
 788 /* Init all tx queues */
 789 static void
 790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
 791 {
 792         int i;
 793
 794         for (i = 0; i < adapter->num_tx_queues; i++)
 795                 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
 796 }
 797
 798
 799 /*
 800  *    parse and copy relevant protocol headers:
 801  *      For a tso pkt, relevant headers are L2/3/4 including options
 802  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
 803  *      if it's a TCP/UDP pkt
 804  *
 805  * Returns:
 806  *    -1:  error happens during parsing
 807  *     0:  protocol headers parsed, but too big to be copied
 808  *     1:  protocol headers parsed and copied
 809  *
 810  * Other effects:
 811  *    1. related *ctx fields are updated.
 812  *    2. ctx->copy_size is # of bytes copied
 813  *    3. the portion copied is guaranteed to be in the linear part
 814  *
 815  */
 816 static int
 817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 818                            struct vmxnet3_tx_ctx *ctx,
 819                            struct vmxnet3_adapter *adapter)
 820 {
 821         struct Vmxnet3_TxDataDesc *tdd;
 822
 823         if (ctx->mss) { /* TSO */
 824                 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
 825                 ctx->l4_hdr_size = tcp_hdrlen(skb);
 826                 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
 827         } else {
 828                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 829                         ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
 830
 831                         if (ctx->ipv4) {
 832                                 const struct iphdr *iph = ip_hdr(skb);
 833
 834                                 if (iph->protocol == IPPROTO_TCP)
 835                                         ctx->l4_hdr_size = tcp_hdrlen(skb);
 836                                 else if (iph->protocol == IPPROTO_UDP)
 837                                         ctx->l4_hdr_size = sizeof(struct udphdr);
 838                                 else
 839                                         ctx->l4_hdr_size = 0;
 840                         } else {
 841                                 /* for simplicity, don't copy L4 headers */
 842                                 ctx->l4_hdr_size = 0;
 843                         }
 844                         ctx->copy_size = min(ctx->eth_ip_hdr_size +
 845                                          ctx->l4_hdr_size, skb->len);
 846                 } else {
 847                         ctx->eth_ip_hdr_size = 0;
 848                         ctx->l4_hdr_size = 0;
 849                         /* copy as much as allowed */
 850                         ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
 851                                              , skb_headlen(skb));
 852                 }
 853
 854                 /* make sure headers are accessible directly */
 855                 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
 856                         goto err;
 857         }
 858
 859         if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
 860                 tq->stats.oversized_hdr++;
 861                 ctx->copy_size = 0;
 862                 return 0;
 863         }
 864
 865         tdd = tq->data_ring.base + tq->tx_ring.next2fill;
 866
 867         memcpy(tdd->data, skb->data, ctx->copy_size);
 868         netdev_dbg(adapter->netdev,
 869                 "copy %u bytes to dataRing[%u]\n",
 870                 ctx->copy_size, tq->tx_ring.next2fill);
 871         return 1;
 872
 873 err:
 874         return -1;
 875 }
 876
 877
 878 static void
 879 vmxnet3_prepare_tso(struct sk_buff *skb,
 880                     struct vmxnet3_tx_ctx *ctx)
 881 {
 882         struct tcphdr *tcph = tcp_hdr(skb);
 883
 884         if (ctx->ipv4) {
 885                 struct iphdr *iph = ip_hdr(skb);
 886
 887                 iph->check = 0;
 888                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
 889                                                  IPPROTO_TCP, 0);
 890         } else {
 891                 struct ipv6hdr *iph = ipv6_hdr(skb);
 892
 893                 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
 894                                                IPPROTO_TCP, 0);
 895         }
 896 }
 897
 898 static int txd_estimate(const struct sk_buff *skb)
 899 {
 900         int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
 901         int i;
 902
 903         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 904                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 905
 906                 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
 907         }
 908         return count;
 909 }
 910
 911 /*
 912  * Transmits a pkt thru a given tq
 913  * Returns:
 914  *    NETDEV_TX_OK:      descriptors are setup successfully
 915  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
 916  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
 917  *
 918  * Side-effects:
 919  *    1. tx ring may be changed
 920  *    2. tq stats may be updated accordingly
 921  *    3. shared->txNumDeferred may be updated
 922  */
 923
 924 static int
 925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 926                 struct vmxnet3_adapter *adapter, struct net_device *netdev)
 927 {
 928         int ret;
 929         u32 count;
 930         unsigned long flags;
 931         struct vmxnet3_tx_ctx ctx;
 932         union Vmxnet3_GenericDesc *gdesc;
 933 #ifdef __BIG_ENDIAN_BITFIELD
 934         /* Use temporary descriptor to avoid touching bits multiple times */
 935         union Vmxnet3_GenericDesc tempTxDesc;
 936 #endif
 937
 938         count = txd_estimate(skb);
 939
 940         ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
 941
 942         ctx.mss = skb_shinfo(skb)->gso_size;
 943         if (ctx.mss) {
 944                 if (skb_header_cloned(skb)) {
 945                         if (unlikely(pskb_expand_head(skb, 0, 0,
 946                                                       GFP_ATOMIC) != 0)) {
 947                                 tq->stats.drop_tso++;
 948                                 goto drop_pkt;
 949                         }
 950                         tq->stats.copy_skb_header++;
 951                 }
 952                 vmxnet3_prepare_tso(skb, &ctx);
 953         } else {
 954                 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
 955
 956                         /* non-tso pkts must not use more than
 957                          * VMXNET3_MAX_TXD_PER_PKT entries
 958                          */
 959                         if (skb_linearize(skb) != 0) {
 960                                 tq->stats.drop_too_many_frags++;
 961                                 goto drop_pkt;
 962                         }
 963                         tq->stats.linearized++;
 964
 965                         /* recalculate the # of descriptors to use */
 966                         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
 967                 }
 968         }
 969
 970         spin_lock_irqsave(&tq->tx_lock, flags);
 971
 972         if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
 973                 tq->stats.tx_ring_full++;
 974                 netdev_dbg(adapter->netdev,
 975                         "tx queue stopped on %s, next2comp %u"
 976                         " next2fill %u\n", adapter->netdev->name,
 977                         tq->tx_ring.next2comp, tq->tx_ring.next2fill);
 978
 979                 vmxnet3_tq_stop(tq, adapter);
 980                 spin_unlock_irqrestore(&tq->tx_lock, flags);
 981                 return NETDEV_TX_BUSY;
 982         }
 983
 984
 985         ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
 986         if (ret >= 0) {
 987                 BUG_ON(ret <= 0 && ctx.copy_size != 0);
 988                 /* hdrs parsed, check against other limits */
 989                 if (ctx.mss) {
 990                         if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
 991                                      VMXNET3_MAX_TX_BUF_SIZE)) {
 992                                 goto hdr_too_big;
 993                         }
 994                 } else {
 995                         if (skb->ip_summed == CHECKSUM_PARTIAL) {
 996                                 if (unlikely(ctx.eth_ip_hdr_size +
 997                                              skb->csum_offset >
 998                                              VMXNET3_MAX_CSUM_OFFSET)) {
 999                                         goto hdr_too_big;
1000                                 }
1001                         }
1002                 }
1003         } else {
1004                 tq->stats.drop_hdr_inspect_err++;
1005                 goto unlock_drop_pkt;
1006         }
1007
1008         /* fill tx descs related to addr & len */
1009         vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1010
1011         /* setup the EOP desc */
1012         ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1013
1014         /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016         gdesc = &tempTxDesc;
1017         gdesc->dword[2] = ctx.sop_txd->dword[2];
1018         gdesc->dword[3] = ctx.sop_txd->dword[3];
1019 #else
1020         gdesc = ctx.sop_txd;
1021 #endif
1022         if (ctx.mss) {
1023                 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024                 gdesc->txd.om = VMXNET3_OM_TSO;
1025                 gdesc->txd.msscof = ctx.mss;
1026                 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027                              gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1028         } else {
1029                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030                         gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031                         gdesc->txd.om = VMXNET3_OM_CSUM;
1032                         gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1033                                             skb->csum_offset;
1034                 } else {
1035                         gdesc->txd.om = 0;
1036                         gdesc->txd.msscof = 0;
1037                 }
1038                 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1039         }
1040
1041         if (vlan_tx_tag_present(skb)) {
1042                 gdesc->txd.ti = 1;
1043                 gdesc->txd.tci = vlan_tx_tag_get(skb);
1044         }
1045
1046         /* finally flips the GEN bit of the SOP desc. */
1047         gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1048                                                   VMXNET3_TXD_GEN);
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050         /* Finished updating in bitfields of Tx Desc, so write them in original
1051          * place.
1052          */
1053         vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054                            (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055         gdesc = ctx.sop_txd;
1056 #endif
1057         netdev_dbg(adapter->netdev,
1058                 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1059                 (u32)(ctx.sop_txd -
1060                 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061                 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1062
1063         spin_unlock_irqrestore(&tq->tx_lock, flags);
1064
1065         if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066                                         le32_to_cpu(tq->shared->txThreshold)) {
1067                 tq->shared->txNumDeferred = 0;
1068                 VMXNET3_WRITE_BAR0_REG(adapter,
1069                                        VMXNET3_REG_TXPROD + tq->qid * 8,
1070                                        tq->tx_ring.next2fill);
1071         }
1072
1073         return NETDEV_TX_OK;
1074
1075 hdr_too_big:
1076         tq->stats.drop_oversized_hdr++;
1077 unlock_drop_pkt:
1078         spin_unlock_irqrestore(&tq->tx_lock, flags);
1079 drop_pkt:
1080         tq->stats.drop_total++;
1081         dev_kfree_skb_any(skb);
1082         return NETDEV_TX_OK;
1083 }
1084
1085
1086 static netdev_tx_t
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1088 {
1089         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1090
1091         BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092         return vmxnet3_tq_xmit(skb,
1093                                &adapter->tx_queue[skb->queue_mapping],
1094                                adapter, netdev);
1095 }
1096
1097
1098 static void
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100                 struct sk_buff *skb,
1101                 union Vmxnet3_GenericDesc *gdesc)
1102 {
1103         if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104                 /* typical case: TCP/UDP over IP and both csums are correct */
1105                 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106                                                         VMXNET3_RCD_CSUM_OK) {
1107                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1108                         BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109                         BUG_ON(!(gdesc->rcd.v4  || gdesc->rcd.v6));
1110                         BUG_ON(gdesc->rcd.frg);
1111                 } else {
1112                         if (gdesc->rcd.csum) {
1113                                 skb->csum = htons(gdesc->rcd.csum);
1114                                 skb->ip_summed = CHECKSUM_PARTIAL;
1115                         } else {
1116                                 skb_checksum_none_assert(skb);
1117                         }
1118                 }
1119         } else {
1120                 skb_checksum_none_assert(skb);
1121         }
1122 }
1123
1124
1125 static void
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127                  struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
1128 {
1129         rq->stats.drop_err++;
1130         if (!rcd->fcs)
1131                 rq->stats.drop_fcs++;
1132
1133         rq->stats.drop_total++;
1134
1135         /*
1136          * We do not unmap and chain the rx buffer to the skb.
1137          * We basically pretend this buffer is not used and will be recycled
1138          * by vmxnet3_rq_alloc_rx_buf()
1139          */
1140
1141         /*
1142          * ctx->skb may be NULL if this is the first and the only one
1143          * desc for the pkt
1144          */
1145         if (ctx->skb)
1146                 dev_kfree_skb_irq(ctx->skb);
1147
1148         ctx->skb = NULL;
1149 }
1150
1151
1152 static int
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154                        struct vmxnet3_adapter *adapter, int quota)
1155 {
1156         static const u32 rxprod_reg[2] = {
1157                 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1158         };
1159         u32 num_rxd = 0;
1160         bool skip_page_frags = false;
1161         struct Vmxnet3_RxCompDesc *rcd;
1162         struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164         struct Vmxnet3_RxDesc rxCmdDesc;
1165         struct Vmxnet3_RxCompDesc rxComp;
1166 #endif
1167         vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1168                           &rxComp);
1169         while (rcd->gen == rq->comp_ring.gen) {
1170                 struct vmxnet3_rx_buf_info *rbi;
1171                 struct sk_buff *skb, *new_skb = NULL;
1172                 struct page *new_page = NULL;
1173                 int num_to_alloc;
1174                 struct Vmxnet3_RxDesc *rxd;
1175                 u32 idx, ring_idx;
1176                 struct vmxnet3_cmd_ring *ring = NULL;
1177                 if (num_rxd >= quota) {
1178                         /* we may stop even before we see the EOP desc of
1179                          * the current pkt
1180                          */
1181                         break;
1182                 }
1183                 num_rxd++;
1184                 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1185                 idx = rcd->rxdIdx;
1186                 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187                 ring = rq->rx_ring + ring_idx;
1188                 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1189                                   &rxCmdDesc);
1190                 rbi = rq->buf_info[ring_idx] + idx;
1191
1192                 BUG_ON(rxd->addr != rbi->dma_addr ||
1193                        rxd->len != rbi->len);
1194
1195                 if (unlikely(rcd->eop && rcd->err)) {
1196                         vmxnet3_rx_error(rq, rcd, ctx, adapter);
1197                         goto rcd_done;
1198                 }
1199
1200                 if (rcd->sop) { /* first buf of the pkt */
1201                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202                                rcd->rqID != rq->qid);
1203
1204                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205                         BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1206
1207                         if (unlikely(rcd->len == 0)) {
1208                                 /* Pretend the rx buffer is skipped. */
1209                                 BUG_ON(!(rcd->sop && rcd->eop));
1210                                 netdev_dbg(adapter->netdev,
1211                                         "rxRing[%u][%u] 0 length\n",
1212                                         ring_idx, idx);
1213                                 goto rcd_done;
1214                         }
1215
1216                         skip_page_frags = false;
1217                         ctx->skb = rbi->skb;
1218                         new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1219                                                             rbi->len);
1220                         if (new_skb == NULL) {
1221                                 /* Skb allocation failed, do not handover this
1222                                  * skb to stack. Reuse it. Drop the existing pkt
1223                                  */
1224                                 rq->stats.rx_buf_alloc_failure++;
1225                                 ctx->skb = NULL;
1226                                 rq->stats.drop_total++;
1227                                 skip_page_frags = true;
1228                                 goto rcd_done;
1229                         }
1230
1231                         dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1232                                          rbi->len,
1233                                          PCI_DMA_FROMDEVICE);
1234
1235 #ifdef VMXNET3_RSS
1236                         if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237                             (adapter->netdev->features & NETIF_F_RXHASH))
1238                                 skb_set_hash(ctx->skb,
1239                                              le32_to_cpu(rcd->rssHash),
1240                                              PKT_HASH_TYPE_L3);
1241 #endif
1242                         skb_put(ctx->skb, rcd->len);
1243
1244                         /* Immediate refill */
1245                         rbi->skb = new_skb;
1246                         rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1247                                                        rbi->skb->data, rbi->len,
1248                                                        PCI_DMA_FROMDEVICE);
1249                         rxd->addr = cpu_to_le64(rbi->dma_addr);
1250                         rxd->len = rbi->len;
1251
1252                 } else {
1253                         BUG_ON(ctx->skb == NULL && !skip_page_frags);
1254
1255                         /* non SOP buffer must be type 1 in most cases */
1256                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1257                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1258
1259                         /* If an sop buffer was dropped, skip all
1260                          * following non-sop fragments. They will be reused.
1261                          */
1262                         if (skip_page_frags)
1263                                 goto rcd_done;
1264
1265                         new_page = alloc_page(GFP_ATOMIC);
1266                         if (unlikely(new_page == NULL)) {
1267                                 /* Replacement page frag could not be allocated.
1268                                  * Reuse this page. Drop the pkt and free the
1269                                  * skb which contained this page as a frag. Skip
1270                                  * processing all the following non-sop frags.
1271                                  */
1272                                 rq->stats.rx_buf_alloc_failure++;
1273                                 dev_kfree_skb(ctx->skb);
1274                                 ctx->skb = NULL;
1275                                 skip_page_frags = true;
1276                                 goto rcd_done;
1277                         }
1278
1279                         if (rcd->len) {
1280                                 dma_unmap_page(&adapter->pdev->dev,
1281                                                rbi->dma_addr, rbi->len,
1282                                                PCI_DMA_FROMDEVICE);
1283
1284                                 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1285                         }
1286
1287                         /* Immediate refill */
1288                         rbi->page = new_page;
1289                         rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1290                                                      rbi->page,
1291                                                      0, PAGE_SIZE,
1292                                                      PCI_DMA_FROMDEVICE);
1293                         rxd->addr = cpu_to_le64(rbi->dma_addr);
1294                         rxd->len = rbi->len;
1295                 }
1296
1297
1298                 skb = ctx->skb;
1299                 if (rcd->eop) {
1300                         skb->len += skb->data_len;
1301
1302                         vmxnet3_rx_csum(adapter, skb,
1303                                         (union Vmxnet3_GenericDesc *)rcd);
1304                         skb->protocol = eth_type_trans(skb, adapter->netdev);
1305
1306                         if (unlikely(rcd->ts))
1307                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1308
1309                         if (adapter->netdev->features & NETIF_F_LRO)
1310                                 netif_receive_skb(skb);
1311                         else
1312                                 napi_gro_receive(&rq->napi, skb);
1313
1314                         ctx->skb = NULL;
1315                 }
1316
1317 rcd_done:
1318                 /* device may have skipped some rx descs */
1319                 ring->next2comp = idx;
1320                 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1321                 ring = rq->rx_ring + ring_idx;
1322                 while (num_to_alloc) {
1323                         vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1324                                           &rxCmdDesc);
1325                         BUG_ON(!rxd->addr);
1326
1327                         /* Recv desc is ready to be used by the device */
1328                         rxd->gen = ring->gen;
1329                         vmxnet3_cmd_ring_adv_next2fill(ring);
1330                         num_to_alloc--;
1331                 }
1332
1333                 /* if needed, update the register */
1334                 if (unlikely(rq->shared->updateRxProd)) {
1335                         VMXNET3_WRITE_BAR0_REG(adapter,
1336                                                rxprod_reg[ring_idx] + rq->qid * 8,
1337                                                ring->next2fill);
1338                 }
1339
1340                 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1341                 vmxnet3_getRxComp(rcd,
1342                                   &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1343         }
1344
1345         return num_rxd;
1346 }
1347
1348
1349 static void
1350 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1351                    struct vmxnet3_adapter *adapter)
1352 {
1353         u32 i, ring_idx;
1354         struct Vmxnet3_RxDesc *rxd;
1355
1356         for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1357                 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1358 #ifdef __BIG_ENDIAN_BITFIELD
1359                         struct Vmxnet3_RxDesc rxDesc;
1360 #endif
1361                         vmxnet3_getRxDesc(rxd,
1362                                 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1363
1364                         if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1365                                         rq->buf_info[ring_idx][i].skb) {
1366                                 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1367                                                  rxd->len, PCI_DMA_FROMDEVICE);
1368                                 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1369                                 rq->buf_info[ring_idx][i].skb = NULL;
1370                         } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1371                                         rq->buf_info[ring_idx][i].page) {
1372                                 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1373                                                rxd->len, PCI_DMA_FROMDEVICE);
1374                                 put_page(rq->buf_info[ring_idx][i].page);
1375                                 rq->buf_info[ring_idx][i].page = NULL;
1376                         }
1377                 }
1378
1379                 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1380                 rq->rx_ring[ring_idx].next2fill =
1381                                         rq->rx_ring[ring_idx].next2comp = 0;
1382         }
1383
1384         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1385         rq->comp_ring.next2proc = 0;
1386 }
1387
1388
1389 static void
1390 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1391 {
1392         int i;
1393
1394         for (i = 0; i < adapter->num_rx_queues; i++)
1395                 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1396 }
1397
1398
1399 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1400                                struct vmxnet3_adapter *adapter)
1401 {
1402         int i;
1403         int j;
1404
1405         /* all rx buffers must have already been freed */
1406         for (i = 0; i < 2; i++) {
1407                 if (rq->buf_info[i]) {
1408                         for (j = 0; j < rq->rx_ring[i].size; j++)
1409                                 BUG_ON(rq->buf_info[i][j].page != NULL);
1410                 }
1411         }
1412
1413
1414         for (i = 0; i < 2; i++) {
1415                 if (rq->rx_ring[i].base) {
1416                         dma_free_coherent(&adapter->pdev->dev,
1417                                           rq->rx_ring[i].size
1418                                           * sizeof(struct Vmxnet3_RxDesc),
1419                                           rq->rx_ring[i].base,
1420                                           rq->rx_ring[i].basePA);
1421                         rq->rx_ring[i].base = NULL;
1422                 }
1423                 rq->buf_info[i] = NULL;
1424         }
1425
1426         if (rq->comp_ring.base) {
1427                 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1428                                   * sizeof(struct Vmxnet3_RxCompDesc),
1429                                   rq->comp_ring.base, rq->comp_ring.basePA);
1430                 rq->comp_ring.base = NULL;
1431         }
1432
1433         if (rq->buf_info[0]) {
1434                 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1435                         (rq->rx_ring[0].size + rq->rx_ring[1].size);
1436                 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1437                                   rq->buf_info_pa);
1438         }
1439 }
1440
1441
1442 static int
1443 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1444                 struct vmxnet3_adapter  *adapter)
1445 {
1446         int i;
1447
1448         /* initialize buf_info */
1449         for (i = 0; i < rq->rx_ring[0].size; i++) {
1450
1451                 /* 1st buf for a pkt is skbuff */
1452                 if (i % adapter->rx_buf_per_pkt == 0) {
1453                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1454                         rq->buf_info[0][i].len = adapter->skb_buf_size;
1455                 } else { /* subsequent bufs for a pkt is frag */
1456                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1457                         rq->buf_info[0][i].len = PAGE_SIZE;
1458                 }
1459         }
1460         for (i = 0; i < rq->rx_ring[1].size; i++) {
1461                 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1462                 rq->buf_info[1][i].len = PAGE_SIZE;
1463         }
1464
1465         /* reset internal state and allocate buffers for both rings */
1466         for (i = 0; i < 2; i++) {
1467                 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1468
1469                 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1470                        sizeof(struct Vmxnet3_RxDesc));
1471                 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1472         }
1473         if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1474                                     adapter) == 0) {
1475                 /* at least has 1 rx buffer for the 1st ring */
1476                 return -ENOMEM;
1477         }
1478         vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1479
1480         /* reset the comp ring */
1481         rq->comp_ring.next2proc = 0;
1482         memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1483                sizeof(struct Vmxnet3_RxCompDesc));
1484         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1485
1486         /* reset rxctx */
1487         rq->rx_ctx.skb = NULL;
1488
1489         /* stats are not reset */
1490         return 0;
1491 }
1492
1493
1494 static int
1495 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1496 {
1497         int i, err = 0;
1498
1499         for (i = 0; i < adapter->num_rx_queues; i++) {
1500                 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1501                 if (unlikely(err)) {
1502                         dev_err(&adapter->netdev->dev, "%s: failed to "
1503                                 "initialize rx queue%i\n",
1504                                 adapter->netdev->name, i);
1505                         break;
1506                 }
1507         }
1508         return err;
1509
1510 }
1511
1512
1513 static int
1514 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1515 {
1516         int i;
1517         size_t sz;
1518         struct vmxnet3_rx_buf_info *bi;
1519
1520         for (i = 0; i < 2; i++) {
1521
1522                 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1523                 rq->rx_ring[i].base = dma_alloc_coherent(
1524                                                 &adapter->pdev->dev, sz,
1525                                                 &rq->rx_ring[i].basePA,
1526                                                 GFP_KERNEL);
1527                 if (!rq->rx_ring[i].base) {
1528                         netdev_err(adapter->netdev,
1529                                    "failed to allocate rx ring %d\n", i);
1530                         goto err;
1531                 }
1532         }
1533
1534         sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1535         rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1536                                                 &rq->comp_ring.basePA,
1537                                                 GFP_KERNEL);
1538         if (!rq->comp_ring.base) {
1539                 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1540                 goto err;
1541         }
1542
1543         sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1544                                                    rq->rx_ring[1].size);
1545         bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1546                                  GFP_KERNEL);
1547         if (!bi)
1548                 goto err;
1549
1550         rq->buf_info[0] = bi;
1551         rq->buf_info[1] = bi + rq->rx_ring[0].size;
1552
1553         return 0;
1554
1555 err:
1556         vmxnet3_rq_destroy(rq, adapter);
1557         return -ENOMEM;
1558 }
1559
1560
1561 static int
1562 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1563 {
1564         int i, err = 0;
1565
1566         for (i = 0; i < adapter->num_rx_queues; i++) {
1567                 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1568                 if (unlikely(err)) {
1569                         dev_err(&adapter->netdev->dev,
1570                                 "%s: failed to create rx queue%i\n",
1571                                 adapter->netdev->name, i);
1572                         goto err_out;
1573                 }
1574         }
1575         return err;
1576 err_out:
1577         vmxnet3_rq_destroy_all(adapter);
1578         return err;
1579
1580 }
1581
1582 /* Multiple queue aware polling function for tx and rx */
1583
1584 static int
1585 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1586 {
1587         int rcd_done = 0, i;
1588         if (unlikely(adapter->shared->ecr))
1589                 vmxnet3_process_events(adapter);
1590         for (i = 0; i < adapter->num_tx_queues; i++)
1591                 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1592
1593         for (i = 0; i < adapter->num_rx_queues; i++)
1594                 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1595                                                    adapter, budget);
1596         return rcd_done;
1597 }
1598
1599
1600 static int
1601 vmxnet3_poll(struct napi_struct *napi, int budget)
1602 {
1603         struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1604                                           struct vmxnet3_rx_queue, napi);
1605         int rxd_done;
1606
1607         rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1608
1609         if (rxd_done < budget) {
1610                 napi_complete(napi);
1611                 vmxnet3_enable_all_intrs(rx_queue->adapter);
1612         }
1613         return rxd_done;
1614 }
1615
1616 /*
1617  * NAPI polling function for MSI-X mode with multiple Rx queues
1618  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1619  */
1620
1621 static int
1622 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1623 {
1624         struct vmxnet3_rx_queue *rq = container_of(napi,
1625                                                 struct vmxnet3_rx_queue, napi);
1626         struct vmxnet3_adapter *adapter = rq->adapter;
1627         int rxd_done;
1628
1629         /* When sharing interrupt with corresponding tx queue, process
1630          * tx completions in that queue as well
1631          */
1632         if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1633                 struct vmxnet3_tx_queue *tq =
1634                                 &adapter->tx_queue[rq - adapter->rx_queue];
1635                 vmxnet3_tq_tx_complete(tq, adapter);
1636         }
1637
1638         rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1639
1640         if (rxd_done < budget) {
1641                 napi_complete(napi);
1642                 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1643         }
1644         return rxd_done;
1645 }
1646
1647
1648 #ifdef CONFIG_PCI_MSI
1649
1650 /*
1651  * Handle completion interrupts on tx queues
1652  * Returns whether or not the intr is handled
1653  */
1654
1655 static irqreturn_t
1656 vmxnet3_msix_tx(int irq, void *data)
1657 {
1658         struct vmxnet3_tx_queue *tq = data;
1659         struct vmxnet3_adapter *adapter = tq->adapter;
1660
1661         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1662                 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1663
1664         /* Handle the case where only one irq is allocate for all tx queues */
1665         if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1666                 int i;
1667                 for (i = 0; i < adapter->num_tx_queues; i++) {
1668                         struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1669                         vmxnet3_tq_tx_complete(txq, adapter);
1670                 }
1671         } else {
1672                 vmxnet3_tq_tx_complete(tq, adapter);
1673         }
1674         vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1675
1676         return IRQ_HANDLED;
1677 }
1678
1679
1680 /*
1681  * Handle completion interrupts on rx queues. Returns whether or not the
1682  * intr is handled
1683  */
1684
1685 static irqreturn_t
1686 vmxnet3_msix_rx(int irq, void *data)
1687 {
1688         struct vmxnet3_rx_queue *rq = data;
1689         struct vmxnet3_adapter *adapter = rq->adapter;
1690
1691         /* disable intr if needed */
1692         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1693                 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1694         napi_schedule(&rq->napi);
1695
1696         return IRQ_HANDLED;
1697 }
1698
1699 /*
1700  *----------------------------------------------------------------------------
1701  *
1702  * vmxnet3_msix_event --
1703  *
1704  *    vmxnet3 msix event intr handler
1705  *
1706  * Result:
1707  *    whether or not the intr is handled
1708  *
1709  *----------------------------------------------------------------------------
1710  */
1711
1712 static irqreturn_t
1713 vmxnet3_msix_event(int irq, void *data)
1714 {
1715         struct net_device *dev = data;
1716         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1717
1718         /* disable intr if needed */
1719         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1720                 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1721
1722         if (adapter->shared->ecr)
1723                 vmxnet3_process_events(adapter);
1724
1725         vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1726
1727         return IRQ_HANDLED;
1728 }
1729
1730 #endif /* CONFIG_PCI_MSI  */
1731
1732
1733 /* Interrupt handler for vmxnet3  */
1734 static irqreturn_t
1735 vmxnet3_intr(int irq, void *dev_id)
1736 {
1737         struct net_device *dev = dev_id;
1738         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1739
1740         if (adapter->intr.type == VMXNET3_IT_INTX) {
1741                 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1742                 if (unlikely(icr == 0))
1743                         /* not ours */
1744                         return IRQ_NONE;
1745         }
1746
1747
1748         /* disable intr if needed */
1749         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1750                 vmxnet3_disable_all_intrs(adapter);
1751
1752         napi_schedule(&adapter->rx_queue[0].napi);
1753
1754         return IRQ_HANDLED;
1755 }
1756
1757 #ifdef CONFIG_NET_POLL_CONTROLLER
1758
1759 /* netpoll callback. */
1760 static void
1761 vmxnet3_netpoll(struct net_device *netdev)
1762 {
1763         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1764
1765         switch (adapter->intr.type) {
1766 #ifdef CONFIG_PCI_MSI
1767         case VMXNET3_IT_MSIX: {
1768                 int i;
1769                 for (i = 0; i < adapter->num_rx_queues; i++)
1770                         vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
1771                 break;
1772         }
1773 #endif
1774         case VMXNET3_IT_MSI:
1775         default:
1776                 vmxnet3_intr(0, adapter->netdev);
1777                 break;
1778         }
1779
1780 }
1781 #endif  /* CONFIG_NET_POLL_CONTROLLER */
1782
1783 static int
1784 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1785 {
1786         struct vmxnet3_intr *intr = &adapter->intr;
1787         int err = 0, i;
1788         int vector = 0;
1789
1790 #ifdef CONFIG_PCI_MSI
1791         if (adapter->intr.type == VMXNET3_IT_MSIX) {
1792                 for (i = 0; i < adapter->num_tx_queues; i++) {
1793                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1794                                 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1795                                         adapter->netdev->name, vector);
1796                                 err = request_irq(
1797                                               intr->msix_entries[vector].vector,
1798                                               vmxnet3_msix_tx, 0,
1799                                               adapter->tx_queue[i].name,
1800                                               &adapter->tx_queue[i]);
1801                         } else {
1802                                 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1803                                         adapter->netdev->name, vector);
1804                         }
1805                         if (err) {
1806                                 dev_err(&adapter->netdev->dev,
1807                                         "Failed to request irq for MSIX, %s, "
1808                                         "error %d\n",
1809                                         adapter->tx_queue[i].name, err);
1810                                 return err;
1811                         }
1812
1813                         /* Handle the case where only 1 MSIx was allocated for
1814                          * all tx queues */
1815                         if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1816                                 for (; i < adapter->num_tx_queues; i++)
1817                                         adapter->tx_queue[i].comp_ring.intr_idx
1818                                                                 = vector;
1819                                 vector++;
1820                                 break;
1821                         } else {
1822                                 adapter->tx_queue[i].comp_ring.intr_idx
1823                                                                 = vector++;
1824                         }
1825                 }
1826                 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1827                         vector = 0;
1828
1829                 for (i = 0; i < adapter->num_rx_queues; i++) {
1830                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1831                                 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1832                                         adapter->netdev->name, vector);
1833                         else
1834                                 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1835                                         adapter->netdev->name, vector);
1836                         err = request_irq(intr->msix_entries[vector].vector,
1837                                           vmxnet3_msix_rx, 0,
1838                                           adapter->rx_queue[i].name,
1839                                           &(adapter->rx_queue[i]));
1840                         if (err) {
1841                                 netdev_err(adapter->netdev,
1842                                            "Failed to request irq for MSIX, "
1843                                            "%s, error %d\n",
1844                                            adapter->rx_queue[i].name, err);
1845                                 return err;
1846                         }
1847
1848                         adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1849                 }
1850
1851                 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1852                         adapter->netdev->name, vector);
1853                 err = request_irq(intr->msix_entries[vector].vector,
1854                                   vmxnet3_msix_event, 0,
1855                                   intr->event_msi_vector_name, adapter->netdev);
1856                 intr->event_intr_idx = vector;
1857
1858         } else if (intr->type == VMXNET3_IT_MSI) {
1859                 adapter->num_rx_queues = 1;
1860                 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1861                                   adapter->netdev->name, adapter->netdev);
1862         } else {
1863 #endif
1864                 adapter->num_rx_queues = 1;
1865                 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1866                                   IRQF_SHARED, adapter->netdev->name,
1867                                   adapter->netdev);
1868 #ifdef CONFIG_PCI_MSI
1869         }
1870 #endif
1871         intr->num_intrs = vector + 1;
1872         if (err) {
1873                 netdev_err(adapter->netdev,
1874                            "Failed to request irq (intr type:%d), error %d\n",
1875                            intr->type, err);
1876         } else {
1877                 /* Number of rx queues will not change after this */
1878                 for (i = 0; i < adapter->num_rx_queues; i++) {
1879                         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1880                         rq->qid = i;
1881                         rq->qid2 = i + adapter->num_rx_queues;
1882                 }
1883
1884
1885
1886                 /* init our intr settings */
1887                 for (i = 0; i < intr->num_intrs; i++)
1888                         intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1889                 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1890                         adapter->intr.event_intr_idx = 0;
1891                         for (i = 0; i < adapter->num_tx_queues; i++)
1892                                 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1893                         adapter->rx_queue[0].comp_ring.intr_idx = 0;
1894                 }
1895
1896                 netdev_info(adapter->netdev,
1897                             "intr type %u, mode %u, %u vectors allocated\n",
1898                             intr->type, intr->mask_mode, intr->num_intrs);
1899         }
1900
1901         return err;
1902 }
1903
1904
1905 static void
1906 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1907 {
1908         struct vmxnet3_intr *intr = &adapter->intr;
1909         BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1910
1911         switch (intr->type) {
1912 #ifdef CONFIG_PCI_MSI
1913         case VMXNET3_IT_MSIX:
1914         {
1915                 int i, vector = 0;
1916
1917                 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1918                         for (i = 0; i < adapter->num_tx_queues; i++) {
1919                                 free_irq(intr->msix_entries[vector++].vector,
1920                                          &(adapter->tx_queue[i]));
1921                                 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1922                                         break;
1923                         }
1924                 }
1925
1926                 for (i = 0; i < adapter->num_rx_queues; i++) {
1927                         free_irq(intr->msix_entries[vector++].vector,
1928                                  &(adapter->rx_queue[i]));
1929                 }
1930
1931                 free_irq(intr->msix_entries[vector].vector,
1932                          adapter->netdev);
1933                 BUG_ON(vector >= intr->num_intrs);
1934                 break;
1935         }
1936 #endif
1937         case VMXNET3_IT_MSI:
1938                 free_irq(adapter->pdev->irq, adapter->netdev);
1939                 break;
1940         case VMXNET3_IT_INTX:
1941                 free_irq(adapter->pdev->irq, adapter->netdev);
1942                 break;
1943         default:
1944                 BUG();
1945         }
1946 }
1947
1948
1949 static void
1950 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1951 {
1952         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1953         u16 vid;
1954
1955         /* allow untagged pkts */
1956         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1957
1958         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1959                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1960 }
1961
1962
1963 static int
1964 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1965 {
1966         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1967
1968         if (!(netdev->flags & IFF_PROMISC)) {
1969                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1970                 unsigned long flags;
1971
1972                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1973                 spin_lock_irqsave(&adapter->cmd_lock, flags);
1974                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1975                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1976                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1977         }
1978
1979         set_bit(vid, adapter->active_vlans);
1980
1981         return 0;
1982 }
1983
1984
1985 static int
1986 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1987 {
1988         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1989
1990         if (!(netdev->flags & IFF_PROMISC)) {
1991                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1992                 unsigned long flags;
1993
1994                 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1995                 spin_lock_irqsave(&adapter->cmd_lock, flags);
1996                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1997                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1998                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1999         }
2000
2001         clear_bit(vid, adapter->active_vlans);
2002
2003         return 0;
2004 }
2005
2006
2007 static u8 *
2008 vmxnet3_copy_mc(struct net_device *netdev)
2009 {
2010         u8 *buf = NULL;
2011         u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2012
2013         /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2014         if (sz <= 0xffff) {
2015                 /* We may be called with BH disabled */
2016                 buf = kmalloc(sz, GFP_ATOMIC);
2017                 if (buf) {
2018                         struct netdev_hw_addr *ha;
2019                         int i = 0;
2020
2021                         netdev_for_each_mc_addr(ha, netdev)
2022                                 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2023                                        ETH_ALEN);
2024                 }
2025         }
2026         return buf;
2027 }
2028
2029
2030 static void
2031 vmxnet3_set_mc(struct net_device *netdev)
2032 {
2033         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2034         unsigned long flags;
2035         struct Vmxnet3_RxFilterConf *rxConf =
2036                                         &adapter->shared->devRead.rxFilterConf;
2037         u8 *new_table = NULL;
2038         dma_addr_t new_table_pa = 0;
2039         u32 new_mode = VMXNET3_RXM_UCAST;
2040
2041         if (netdev->flags & IFF_PROMISC) {
2042                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2043                 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2044
2045                 new_mode |= VMXNET3_RXM_PROMISC;
2046         } else {
2047                 vmxnet3_restore_vlan(adapter);
2048         }
2049
2050         if (netdev->flags & IFF_BROADCAST)
2051                 new_mode |= VMXNET3_RXM_BCAST;
2052
2053         if (netdev->flags & IFF_ALLMULTI)
2054                 new_mode |= VMXNET3_RXM_ALL_MULTI;
2055         else
2056                 if (!netdev_mc_empty(netdev)) {
2057                         new_table = vmxnet3_copy_mc(netdev);
2058                         if (new_table) {
2059                                 new_mode |= VMXNET3_RXM_MCAST;
2060                                 rxConf->mfTableLen = cpu_to_le16(
2061                                         netdev_mc_count(netdev) * ETH_ALEN);
2062                                 new_table_pa = dma_map_single(
2063                                                         &adapter->pdev->dev,
2064                                                         new_table,
2065                                                         rxConf->mfTableLen,
2066                                                         PCI_DMA_TODEVICE);
2067                                 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2068                         } else {
2069                                 netdev_info(netdev, "failed to copy mcast list"
2070                                             ", setting ALL_MULTI\n");
2071                                 new_mode |= VMXNET3_RXM_ALL_MULTI;
2072                         }
2073                 }
2074
2075
2076         if (!(new_mode & VMXNET3_RXM_MCAST)) {
2077                 rxConf->mfTableLen = 0;
2078                 rxConf->mfTablePA = 0;
2079         }
2080
2081         spin_lock_irqsave(&adapter->cmd_lock, flags);
2082         if (new_mode != rxConf->rxMode) {
2083                 rxConf->rxMode = cpu_to_le32(new_mode);
2084                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2085                                        VMXNET3_CMD_UPDATE_RX_MODE);
2086                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2087                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2088         }
2089
2090         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2091                                VMXNET3_CMD_UPDATE_MAC_FILTERS);
2092         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2093
2094         if (new_table) {
2095                 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2096                                  rxConf->mfTableLen, PCI_DMA_TODEVICE);
2097                 kfree(new_table);
2098         }
2099 }
2100
2101 void
2102 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2103 {
2104         int i;
2105
2106         for (i = 0; i < adapter->num_rx_queues; i++)
2107                 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2108 }
2109
2110
2111 /*
2112  *   Set up driver_shared based on settings in adapter.
2113  */
2114
2115 static void
2116 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2117 {
2118         struct Vmxnet3_DriverShared *shared = adapter->shared;
2119         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2120         struct Vmxnet3_TxQueueConf *tqc;
2121         struct Vmxnet3_RxQueueConf *rqc;
2122         int i;
2123
2124         memset(shared, 0, sizeof(*shared));
2125
2126         /* driver settings */
2127         shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2128         devRead->misc.driverInfo.version = cpu_to_le32(
2129                                                 VMXNET3_DRIVER_VERSION_NUM);
2130         devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2131                                 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2132         devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2133         *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2134                                 *((u32 *)&devRead->misc.driverInfo.gos));
2135         devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2136         devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2137
2138         devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2139         devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2140
2141         /* set up feature flags */
2142         if (adapter->netdev->features & NETIF_F_RXCSUM)
2143                 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2144
2145         if (adapter->netdev->features & NETIF_F_LRO) {
2146                 devRead->misc.uptFeatures |= UPT1_F_LRO;
2147                 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2148         }
2149         if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2150                 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2151
2152         devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2153         devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2154         devRead->misc.queueDescLen = cpu_to_le32(
2155                 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2156                 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2157
2158         /* tx queue settings */
2159         devRead->misc.numTxQueues =  adapter->num_tx_queues;
2160         for (i = 0; i < adapter->num_tx_queues; i++) {
2161                 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2162                 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2163                 tqc = &adapter->tqd_start[i].conf;
2164                 tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
2165                 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2166                 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2167                 tqc->ddPA           = cpu_to_le64(tq->buf_info_pa);
2168                 tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
2169                 tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
2170                 tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
2171                 tqc->ddLen          = cpu_to_le32(
2172                                         sizeof(struct vmxnet3_tx_buf_info) *
2173                                         tqc->txRingSize);
2174                 tqc->intrIdx        = tq->comp_ring.intr_idx;
2175         }
2176
2177         /* rx queue settings */
2178         devRead->misc.numRxQueues = adapter->num_rx_queues;
2179         for (i = 0; i < adapter->num_rx_queues; i++) {
2180                 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2181                 rqc = &adapter->rqd_start[i].conf;
2182                 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2183                 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2184                 rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
2185                 rqc->ddPA            = cpu_to_le64(rq->buf_info_pa);
2186                 rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
2187                 rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
2188                 rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
2189                 rqc->ddLen           = cpu_to_le32(
2190                                         sizeof(struct vmxnet3_rx_buf_info) *
2191                                         (rqc->rxRingSize[0] +
2192                                          rqc->rxRingSize[1]));
2193                 rqc->intrIdx         = rq->comp_ring.intr_idx;
2194         }
2195
2196 #ifdef VMXNET3_RSS
2197         memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2198
2199         if (adapter->rss) {
2200                 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2201                 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2202                         0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2203                         0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2204                         0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2205                         0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2206                         0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2207                 };
2208
2209                 devRead->misc.uptFeatures |= UPT1_F_RSS;
2210                 devRead->misc.numRxQueues = adapter->num_rx_queues;
2211                 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2212                                     UPT1_RSS_HASH_TYPE_IPV4 |
2213                                     UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2214                                     UPT1_RSS_HASH_TYPE_IPV6;
2215                 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2216                 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2217                 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2218                 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2219
2220                 for (i = 0; i < rssConf->indTableSize; i++)
2221                         rssConf->indTable[i] = ethtool_rxfh_indir_default(
2222                                 i, adapter->num_rx_queues);
2223
2224                 devRead->rssConfDesc.confVer = 1;
2225                 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2226                 devRead->rssConfDesc.confPA =
2227                         cpu_to_le64(adapter->rss_conf_pa);
2228         }
2229
2230 #endif /* VMXNET3_RSS */
2231
2232         /* intr settings */
2233         devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2234                                      VMXNET3_IMM_AUTO;
2235         devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2236         for (i = 0; i < adapter->intr.num_intrs; i++)
2237                 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2238
2239         devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2240         devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2241
2242         /* rx filter settings */
2243         devRead->rxFilterConf.rxMode = 0;
2244         vmxnet3_restore_vlan(adapter);
2245         vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2246
2247         /* the rest are already zeroed */
2248 }
2249
2250
2251 int
2252 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2253 {
2254         int err, i;
2255         u32 ret;
2256         unsigned long flags;
2257
2258         netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2259                 " ring sizes %u %u %u\n", adapter->netdev->name,
2260                 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2261                 adapter->tx_queue[0].tx_ring.size,
2262                 adapter->rx_queue[0].rx_ring[0].size,
2263                 adapter->rx_queue[0].rx_ring[1].size);
2264
2265         vmxnet3_tq_init_all(adapter);
2266         err = vmxnet3_rq_init_all(adapter);
2267         if (err) {
2268                 netdev_err(adapter->netdev,
2269                            "Failed to init rx queue error %d\n", err);
2270                 goto rq_err;
2271         }
2272
2273         err = vmxnet3_request_irqs(adapter);
2274         if (err) {
2275                 netdev_err(adapter->netdev,
2276                            "Failed to setup irq for error %d\n", err);
2277                 goto irq_err;
2278         }
2279
2280         vmxnet3_setup_driver_shared(adapter);
2281
2282         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2283                                adapter->shared_pa));
2284         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2285                                adapter->shared_pa));
2286         spin_lock_irqsave(&adapter->cmd_lock, flags);
2287         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2288                                VMXNET3_CMD_ACTIVATE_DEV);
2289         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2290         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2291
2292         if (ret != 0) {
2293                 netdev_err(adapter->netdev,
2294                            "Failed to activate dev: error %u\n", ret);
2295                 err = -EINVAL;
2296                 goto activate_err;
2297         }
2298
2299         for (i = 0; i < adapter->num_rx_queues; i++) {
2300                 VMXNET3_WRITE_BAR0_REG(adapter,
2301                                 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2302                                 adapter->rx_queue[i].rx_ring[0].next2fill);
2303                 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2304                                 (i * VMXNET3_REG_ALIGN)),
2305                                 adapter->rx_queue[i].rx_ring[1].next2fill);
2306         }
2307
2308         /* Apply the rx filter settins last. */
2309         vmxnet3_set_mc(adapter->netdev);
2310
2311         /*
2312          * Check link state when first activating device. It will start the
2313          * tx queue if the link is up.
2314          */
2315         vmxnet3_check_link(adapter, true);
2316         for (i = 0; i < adapter->num_rx_queues; i++)
2317                 napi_enable(&adapter->rx_queue[i].napi);
2318         vmxnet3_enable_all_intrs(adapter);
2319         clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2320         return 0;
2321
2322 activate_err:
2323         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2324         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2325         vmxnet3_free_irqs(adapter);
2326 irq_err:
2327 rq_err:
2328         /* free up buffers we allocated */
2329         vmxnet3_rq_cleanup_all(adapter);
2330         return err;
2331 }
2332
2333
2334 void
2335 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2336 {
2337         unsigned long flags;
2338         spin_lock_irqsave(&adapter->cmd_lock, flags);
2339         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2340         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2341 }
2342
2343
2344 int
2345 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2346 {
2347         int i;
2348         unsigned long flags;
2349         if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2350                 return 0;
2351
2352
2353         spin_lock_irqsave(&adapter->cmd_lock, flags);
2354         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2355                                VMXNET3_CMD_QUIESCE_DEV);
2356         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2357         vmxnet3_disable_all_intrs(adapter);
2358
2359         for (i = 0; i < adapter->num_rx_queues; i++)
2360                 napi_disable(&adapter->rx_queue[i].napi);
2361         netif_tx_disable(adapter->netdev);
2362         adapter->link_speed = 0;
2363         netif_carrier_off(adapter->netdev);
2364
2365         vmxnet3_tq_cleanup_all(adapter);
2366         vmxnet3_rq_cleanup_all(adapter);
2367         vmxnet3_free_irqs(adapter);
2368         return 0;
2369 }
2370
2371
2372 static void
2373 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2374 {
2375         u32 tmp;
2376
2377         tmp = *(u32 *)mac;
2378         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2379
2380         tmp = (mac[5] << 8) | mac[4];
2381         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2382 }
2383
2384
2385 static int
2386 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2387 {
2388         struct sockaddr *addr = p;
2389         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2390
2391         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2392         vmxnet3_write_mac_addr(adapter, addr->sa_data);
2393
2394         return 0;
2395 }
2396
2397
2398 /* ==================== initialization and cleanup routines ============ */
2399
2400 static int
2401 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2402 {
2403         int err;
2404         unsigned long mmio_start, mmio_len;
2405         struct pci_dev *pdev = adapter->pdev;
2406
2407         err = pci_enable_device(pdev);
2408         if (err) {
2409                 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2410                 return err;
2411         }
2412
2413         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2414                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2415                         dev_err(&pdev->dev,
2416                                 "pci_set_consistent_dma_mask failed\n");
2417                         err = -EIO;
2418                         goto err_set_mask;
2419                 }
2420                 *dma64 = true;
2421         } else {
2422                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2423                         dev_err(&pdev->dev,
2424                                 "pci_set_dma_mask failed\n");
2425                         err = -EIO;
2426                         goto err_set_mask;
2427                 }
2428                 *dma64 = false;
2429         }
2430
2431         err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2432                                            vmxnet3_driver_name);
2433         if (err) {
2434                 dev_err(&pdev->dev,
2435                         "Failed to request region for adapter: error %d\n", err);
2436                 goto err_set_mask;
2437         }
2438
2439         pci_set_master(pdev);
2440
2441         mmio_start = pci_resource_start(pdev, 0);
2442         mmio_len = pci_resource_len(pdev, 0);
2443         adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2444         if (!adapter->hw_addr0) {
2445                 dev_err(&pdev->dev, "Failed to map bar0\n");
2446                 err = -EIO;
2447                 goto err_ioremap;
2448         }
2449
2450         mmio_start = pci_resource_start(pdev, 1);
2451         mmio_len = pci_resource_len(pdev, 1);
2452         adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2453         if (!adapter->hw_addr1) {
2454                 dev_err(&pdev->dev, "Failed to map bar1\n");
2455                 err = -EIO;
2456                 goto err_bar1;
2457         }
2458         return 0;
2459
2460 err_bar1:
2461         iounmap(adapter->hw_addr0);
2462 err_ioremap:
2463         pci_release_selected_regions(pdev, (1 << 2) - 1);
2464 err_set_mask:
2465         pci_disable_device(pdev);
2466         return err;
2467 }
2468
2469
2470 static void
2471 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2472 {
2473         BUG_ON(!adapter->pdev);
2474
2475         iounmap(adapter->hw_addr0);
2476         iounmap(adapter->hw_addr1);
2477         pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2478         pci_disable_device(adapter->pdev);
2479 }
2480
2481
2482 static void
2483 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2484 {
2485         size_t sz, i, ring0_size, ring1_size, comp_size;
2486         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2487
2488
2489         if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2490                                     VMXNET3_MAX_ETH_HDR_SIZE) {
2491                 adapter->skb_buf_size = adapter->netdev->mtu +
2492                                         VMXNET3_MAX_ETH_HDR_SIZE;
2493                 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2494                         adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2495
2496                 adapter->rx_buf_per_pkt = 1;
2497         } else {
2498                 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2499                 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2500                                             VMXNET3_MAX_ETH_HDR_SIZE;
2501                 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2502         }
2503
2504         /*
2505          * for simplicity, force the ring0 size to be a multiple of
2506          * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2507          */
2508         sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2509         ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2510         ring0_size = (ring0_size + sz - 1) / sz * sz;
2511         ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2512                            sz * sz);
2513         ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2514         comp_size = ring0_size + ring1_size;
2515
2516         for (i = 0; i < adapter->num_rx_queues; i++) {
2517                 rq = &adapter->rx_queue[i];
2518                 rq->rx_ring[0].size = ring0_size;
2519                 rq->rx_ring[1].size = ring1_size;
2520                 rq->comp_ring.size = comp_size;
2521         }
2522 }
2523
2524
2525 int
2526 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2527                       u32 rx_ring_size, u32 rx_ring2_size)
2528 {
2529         int err = 0, i;
2530
2531         for (i = 0; i < adapter->num_tx_queues; i++) {
2532                 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2533                 tq->tx_ring.size   = tx_ring_size;
2534                 tq->data_ring.size = tx_ring_size;
2535                 tq->comp_ring.size = tx_ring_size;
2536                 tq->shared = &adapter->tqd_start[i].ctrl;
2537                 tq->stopped = true;
2538                 tq->adapter = adapter;
2539                 tq->qid = i;
2540                 err = vmxnet3_tq_create(tq, adapter);
2541                 /*
2542                  * Too late to change num_tx_queues. We cannot do away with
2543                  * lesser number of queues than what we asked for
2544                  */
2545                 if (err)
2546                         goto queue_err;
2547         }
2548
2549         adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2550         adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2551         vmxnet3_adjust_rx_ring_size(adapter);
2552         for (i = 0; i < adapter->num_rx_queues; i++) {
2553                 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2554                 /* qid and qid2 for rx queues will be assigned later when num
2555                  * of rx queues is finalized after allocating intrs */
2556                 rq->shared = &adapter->rqd_start[i].ctrl;
2557                 rq->adapter = adapter;
2558                 err = vmxnet3_rq_create(rq, adapter);
2559                 if (err) {
2560                         if (i == 0) {
2561                                 netdev_err(adapter->netdev,
2562                                            "Could not allocate any rx queues. "
2563                                            "Aborting.\n");
2564                                 goto queue_err;
2565                         } else {
2566                                 netdev_info(adapter->netdev,
2567                                             "Number of rx queues changed "
2568                                             "to : %d.\n", i);
2569                                 adapter->num_rx_queues = i;
2570                                 err = 0;
2571                                 break;
2572                         }
2573                 }
2574         }
2575         return err;
2576 queue_err:
2577         vmxnet3_tq_destroy_all(adapter);
2578         return err;
2579 }
2580
2581 static int
2582 vmxnet3_open(struct net_device *netdev)
2583 {
2584         struct vmxnet3_adapter *adapter;
2585         int err, i;
2586
2587         adapter = netdev_priv(netdev);
2588
2589         for (i = 0; i < adapter->num_tx_queues; i++)
2590                 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2591
2592         err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2593                                     VMXNET3_DEF_RX_RING_SIZE,
2594                                     VMXNET3_DEF_RX_RING_SIZE);
2595         if (err)
2596                 goto queue_err;
2597
2598         err = vmxnet3_activate_dev(adapter);
2599         if (err)
2600                 goto activate_err;
2601
2602         return 0;
2603
2604 activate_err:
2605         vmxnet3_rq_destroy_all(adapter);
2606         vmxnet3_tq_destroy_all(adapter);
2607 queue_err:
2608         return err;
2609 }
2610
2611
2612 static int
2613 vmxnet3_close(struct net_device *netdev)
2614 {
2615         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2616
2617         /*
2618          * Reset_work may be in the middle of resetting the device, wait for its
2619          * completion.
2620          */
2621         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2622                 msleep(1);
2623
2624         vmxnet3_quiesce_dev(adapter);
2625
2626         vmxnet3_rq_destroy_all(adapter);
2627         vmxnet3_tq_destroy_all(adapter);
2628
2629         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2630
2631
2632         return 0;
2633 }
2634
2635
2636 void
2637 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2638 {
2639         int i;
2640
2641         /*
2642          * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2643          * vmxnet3_close() will deadlock.
2644          */
2645         BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2646
2647         /* we need to enable NAPI, otherwise dev_close will deadlock */
2648         for (i = 0; i < adapter->num_rx_queues; i++)
2649                 napi_enable(&adapter->rx_queue[i].napi);
2650         dev_close(adapter->netdev);
2651 }
2652
2653
2654 static int
2655 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2656 {
2657         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2658         int err = 0;
2659
2660         if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2661                 return -EINVAL;
2662
2663         netdev->mtu = new_mtu;
2664
2665         /*
2666          * Reset_work may be in the middle of resetting the device, wait for its
2667          * completion.
2668          */
2669         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2670                 msleep(1);
2671
2672         if (netif_running(netdev)) {
2673                 vmxnet3_quiesce_dev(adapter);
2674                 vmxnet3_reset_dev(adapter);
2675
2676                 /* we need to re-create the rx queue based on the new mtu */
2677                 vmxnet3_rq_destroy_all(adapter);
2678                 vmxnet3_adjust_rx_ring_size(adapter);
2679                 err = vmxnet3_rq_create_all(adapter);
2680                 if (err) {
2681                         netdev_err(netdev,
2682                                    "failed to re-create rx queues, "
2683                                    " error %d. Closing it.\n", err);
2684                         goto out;
2685                 }
2686
2687                 err = vmxnet3_activate_dev(adapter);
2688                 if (err) {
2689                         netdev_err(netdev,
2690                                    "failed to re-activate, error %d. "
2691                                    "Closing it\n", err);
2692                         goto out;
2693                 }
2694         }
2695
2696 out:
2697         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2698         if (err)
2699                 vmxnet3_force_close(adapter);
2700
2701         return err;
2702 }
2703
2704
2705 static void
2706 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2707 {
2708         struct net_device *netdev = adapter->netdev;
2709
2710         netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2711                 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2712                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2713                 NETIF_F_LRO;
2714         if (dma64)
2715                 netdev->hw_features |= NETIF_F_HIGHDMA;
2716         netdev->vlan_features = netdev->hw_features &
2717                                 ~(NETIF_F_HW_VLAN_CTAG_TX |
2718                                   NETIF_F_HW_VLAN_CTAG_RX);
2719         netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2720 }
2721
2722
2723 static void
2724 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2725 {
2726         u32 tmp;
2727
2728         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2729         *(u32 *)mac = tmp;
2730
2731         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2732         mac[4] = tmp & 0xff;
2733         mac[5] = (tmp >> 8) & 0xff;
2734 }
2735
2736 #ifdef CONFIG_PCI_MSI
2737
2738 /*
2739  * Enable MSIx vectors.
2740  * Returns :
2741  *      VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2742  *       were enabled.
2743  *      number of vectors which were enabled otherwise (this number is greater
2744  *       than VMXNET3_LINUX_MIN_MSIX_VECT)
2745  */
2746
2747 static int
2748 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
2749 {
2750         int ret = pci_enable_msix_range(adapter->pdev,
2751                                         adapter->intr.msix_entries, nvec, nvec);
2752
2753         if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
2754                 dev_err(&adapter->netdev->dev,
2755                         "Failed to enable %d MSI-X, trying %d\n",
2756                         nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
2757
2758                 ret = pci_enable_msix_range(adapter->pdev,
2759                                             adapter->intr.msix_entries,
2760                                             VMXNET3_LINUX_MIN_MSIX_VECT,
2761                                             VMXNET3_LINUX_MIN_MSIX_VECT);
2762         }
2763
2764         if (ret < 0) {
2765                 dev_err(&adapter->netdev->dev,
2766                         "Failed to enable MSI-X, error: %d\n", ret);
2767         }
2768
2769         return ret;
2770 }
2771
2772
2773 #endif /* CONFIG_PCI_MSI */
2774
2775 static void
2776 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2777 {
2778         u32 cfg;
2779         unsigned long flags;
2780
2781         /* intr settings */
2782         spin_lock_irqsave(&adapter->cmd_lock, flags);
2783         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2784                                VMXNET3_CMD_GET_CONF_INTR);
2785         cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2786         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2787         adapter->intr.type = cfg & 0x3;
2788         adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2789
2790         if (adapter->intr.type == VMXNET3_IT_AUTO) {
2791                 adapter->intr.type = VMXNET3_IT_MSIX;
2792         }
2793
2794 #ifdef CONFIG_PCI_MSI
2795         if (adapter->intr.type == VMXNET3_IT_MSIX) {
2796                 int i, nvec;
2797
2798                 nvec  = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
2799                         1 : adapter->num_tx_queues;
2800                 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
2801                         0 : adapter->num_rx_queues;
2802                 nvec += 1;      /* for link event */
2803                 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
2804                        nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
2805
2806                 for (i = 0; i < nvec; i++)
2807                         adapter->intr.msix_entries[i].entry = i;
2808
2809                 nvec = vmxnet3_acquire_msix_vectors(adapter, nvec);
2810                 if (nvec < 0)
2811                         goto msix_err;
2812
2813                 /* If we cannot allocate one MSIx vector per queue
2814                  * then limit the number of rx queues to 1
2815                  */
2816                 if (nvec == VMXNET3_LINUX_MIN_MSIX_VECT) {
2817                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2818                             || adapter->num_rx_queues != 1) {
2819                                 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2820                                 netdev_err(adapter->netdev,
2821                                            "Number of rx queues : 1\n");
2822                                 adapter->num_rx_queues = 1;
2823                         }
2824                 }
2825
2826                 adapter->intr.num_intrs = nvec;
2827                 return;
2828
2829 msix_err:
2830                 /* If we cannot allocate MSIx vectors use only one rx queue */
2831                 dev_info(&adapter->pdev->dev,
2832                          "Failed to enable MSI-X, error %d. "
2833                          "Limiting #rx queues to 1, try MSI.\n", nvec);
2834
2835                 adapter->intr.type = VMXNET3_IT_MSI;
2836         }
2837
2838         if (adapter->intr.type == VMXNET3_IT_MSI) {
2839                 if (!pci_enable_msi(adapter->pdev)) {
2840                         adapter->num_rx_queues = 1;
2841                         adapter->intr.num_intrs = 1;
2842                         return;
2843                 }
2844         }
2845 #endif /* CONFIG_PCI_MSI */
2846
2847         adapter->num_rx_queues = 1;
2848         dev_info(&adapter->netdev->dev,
2849                  "Using INTx interrupt, #Rx queues: 1.\n");
2850         adapter->intr.type = VMXNET3_IT_INTX;
2851
2852         /* INT-X related setting */
2853         adapter->intr.num_intrs = 1;
2854 }
2855
2856
2857 static void
2858 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2859 {
2860         if (adapter->intr.type == VMXNET3_IT_MSIX)
2861                 pci_disable_msix(adapter->pdev);
2862         else if (adapter->intr.type == VMXNET3_IT_MSI)
2863                 pci_disable_msi(adapter->pdev);
2864         else
2865                 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2866 }
2867
2868
2869 static void
2870 vmxnet3_tx_timeout(struct net_device *netdev)
2871 {
2872         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2873         adapter->tx_timeout_count++;
2874
2875         netdev_err(adapter->netdev, "tx hang\n");
2876         schedule_work(&adapter->work);
2877         netif_wake_queue(adapter->netdev);
2878 }
2879
2880
2881 static void
2882 vmxnet3_reset_work(struct work_struct *data)
2883 {
2884         struct vmxnet3_adapter *adapter;
2885
2886         adapter = container_of(data, struct vmxnet3_adapter, work);
2887
2888         /* if another thread is resetting the device, no need to proceed */
2889         if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2890                 return;
2891
2892         /* if the device is closed, we must leave it alone */
2893         rtnl_lock();
2894         if (netif_running(adapter->netdev)) {
2895                 netdev_notice(adapter->netdev, "resetting\n");
2896                 vmxnet3_quiesce_dev(adapter);
2897                 vmxnet3_reset_dev(adapter);
2898                 vmxnet3_activate_dev(adapter);
2899         } else {
2900                 netdev_info(adapter->netdev, "already closed\n");
2901         }
2902         rtnl_unlock();
2903
2904         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2905 }
2906
2907
2908 static int
2909 vmxnet3_probe_device(struct pci_dev *pdev,
2910                      const struct pci_device_id *id)
2911 {
2912         static const struct net_device_ops vmxnet3_netdev_ops = {
2913                 .ndo_open = vmxnet3_open,
2914                 .ndo_stop = vmxnet3_close,
2915                 .ndo_start_xmit = vmxnet3_xmit_frame,
2916                 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2917                 .ndo_change_mtu = vmxnet3_change_mtu,
2918                 .ndo_set_features = vmxnet3_set_features,
2919                 .ndo_get_stats64 = vmxnet3_get_stats64,
2920                 .ndo_tx_timeout = vmxnet3_tx_timeout,
2921                 .ndo_set_rx_mode = vmxnet3_set_mc,
2922                 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2923                 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2924 #ifdef CONFIG_NET_POLL_CONTROLLER
2925                 .ndo_poll_controller = vmxnet3_netpoll,
2926 #endif
2927         };
2928         int err;
2929         bool dma64 = false; /* stupid gcc */
2930         u32 ver;
2931         struct net_device *netdev;
2932         struct vmxnet3_adapter *adapter;
2933         u8 mac[ETH_ALEN];
2934         int size;
2935         int num_tx_queues;
2936         int num_rx_queues;
2937
2938         if (!pci_msi_enabled())
2939                 enable_mq = 0;
2940
2941 #ifdef VMXNET3_RSS
2942         if (enable_mq)
2943                 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2944                                     (int)num_online_cpus());
2945         else
2946 #endif
2947                 num_rx_queues = 1;
2948         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2949
2950         if (enable_mq)
2951                 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2952                                     (int)num_online_cpus());
2953         else
2954                 num_tx_queues = 1;
2955
2956         num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2957         netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2958                                    max(num_tx_queues, num_rx_queues));
2959         dev_info(&pdev->dev,
2960                  "# of Tx queues : %d, # of Rx queues : %d\n",
2961                  num_tx_queues, num_rx_queues);
2962
2963         if (!netdev)
2964                 return -ENOMEM;
2965
2966         pci_set_drvdata(pdev, netdev);
2967         adapter = netdev_priv(netdev);
2968         adapter->netdev = netdev;
2969         adapter->pdev = pdev;
2970
2971         spin_lock_init(&adapter->cmd_lock);
2972         adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2973                                              sizeof(struct vmxnet3_adapter),
2974                                              PCI_DMA_TODEVICE);
2975         adapter->shared = dma_alloc_coherent(
2976                                 &adapter->pdev->dev,
2977                                 sizeof(struct Vmxnet3_DriverShared),
2978                                 &adapter->shared_pa, GFP_KERNEL);
2979         if (!adapter->shared) {
2980                 dev_err(&pdev->dev, "Failed to allocate memory\n");
2981                 err = -ENOMEM;
2982                 goto err_alloc_shared;
2983         }
2984
2985         adapter->num_rx_queues = num_rx_queues;
2986         adapter->num_tx_queues = num_tx_queues;
2987         adapter->rx_buf_per_pkt = 1;
2988
2989         size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2990         size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2991         adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
2992                                                 &adapter->queue_desc_pa,
2993                                                 GFP_KERNEL);
2994
2995         if (!adapter->tqd_start) {
2996                 dev_err(&pdev->dev, "Failed to allocate memory\n");
2997                 err = -ENOMEM;
2998                 goto err_alloc_queue_desc;
2999         }
3000         adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3001                                                             adapter->num_tx_queues);
3002
3003         adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3004                                               sizeof(struct Vmxnet3_PMConf),
3005                                               &adapter->pm_conf_pa,
3006                                               GFP_KERNEL);
3007         if (adapter->pm_conf == NULL) {
3008                 err = -ENOMEM;
3009                 goto err_alloc_pm;
3010         }
3011
3012 #ifdef VMXNET3_RSS
3013
3014         adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3015                                                sizeof(struct UPT1_RSSConf),
3016                                                &adapter->rss_conf_pa,
3017                                                GFP_KERNEL);
3018         if (adapter->rss_conf == NULL) {
3019                 err = -ENOMEM;
3020                 goto err_alloc_rss;
3021         }
3022 #endif /* VMXNET3_RSS */
3023
3024         err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3025         if (err < 0)
3026                 goto err_alloc_pci;
3027
3028         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3029         if (ver & 1) {
3030                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3031         } else {
3032                 dev_err(&pdev->dev,
3033                         "Incompatible h/w version (0x%x) for adapter\n", ver);
3034                 err = -EBUSY;
3035                 goto err_ver;
3036         }
3037
3038         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3039         if (ver & 1) {
3040                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3041         } else {
3042                 dev_err(&pdev->dev,
3043                         "Incompatible upt version (0x%x) for adapter\n", ver);
3044                 err = -EBUSY;
3045                 goto err_ver;
3046         }
3047
3048         SET_NETDEV_DEV(netdev, &pdev->dev);
3049         vmxnet3_declare_features(adapter, dma64);
3050
3051         if (adapter->num_tx_queues == adapter->num_rx_queues)
3052                 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3053         else
3054                 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3055
3056         vmxnet3_alloc_intr_resources(adapter);
3057
3058 #ifdef VMXNET3_RSS
3059         if (adapter->num_rx_queues > 1 &&
3060             adapter->intr.type == VMXNET3_IT_MSIX) {
3061                 adapter->rss = true;
3062                 netdev->hw_features |= NETIF_F_RXHASH;
3063                 netdev->features |= NETIF_F_RXHASH;
3064                 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3065         } else {
3066                 adapter->rss = false;
3067         }
3068 #endif
3069
3070         vmxnet3_read_mac_addr(adapter, mac);
3071         memcpy(netdev->dev_addr,  mac, netdev->addr_len);
3072
3073         netdev->netdev_ops = &vmxnet3_netdev_ops;
3074         vmxnet3_set_ethtool_ops(netdev);
3075         netdev->watchdog_timeo = 5 * HZ;
3076
3077         INIT_WORK(&adapter->work, vmxnet3_reset_work);
3078         set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3079
3080         if (adapter->intr.type == VMXNET3_IT_MSIX) {
3081                 int i;
3082                 for (i = 0; i < adapter->num_rx_queues; i++) {
3083                         netif_napi_add(adapter->netdev,
3084                                        &adapter->rx_queue[i].napi,
3085                                        vmxnet3_poll_rx_only, 64);
3086                 }
3087         } else {
3088                 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3089                                vmxnet3_poll, 64);
3090         }
3091
3092         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3093         netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3094
3095         netif_carrier_off(netdev);
3096         err = register_netdev(netdev);
3097
3098         if (err) {
3099                 dev_err(&pdev->dev, "Failed to register adapter\n");
3100                 goto err_register;
3101         }
3102
3103         vmxnet3_check_link(adapter, false);
3104         return 0;
3105
3106 err_register:
3107         vmxnet3_free_intr_resources(adapter);
3108 err_ver:
3109         vmxnet3_free_pci_resources(adapter);
3110 err_alloc_pci:
3111 #ifdef VMXNET3_RSS
3112         dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3113                           adapter->rss_conf, adapter->rss_conf_pa);
3114 err_alloc_rss:
3115 #endif
3116         dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3117                           adapter->pm_conf, adapter->pm_conf_pa);
3118 err_alloc_pm:
3119         dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3120                           adapter->queue_desc_pa);
3121 err_alloc_queue_desc:
3122         dma_free_coherent(&adapter->pdev->dev,
3123                           sizeof(struct Vmxnet3_DriverShared),
3124                           adapter->shared, adapter->shared_pa);
3125 err_alloc_shared:
3126         dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3127                          sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3128         free_netdev(netdev);
3129         return err;
3130 }
3131
3132
3133 static void
3134 vmxnet3_remove_device(struct pci_dev *pdev)
3135 {
3136         struct net_device *netdev = pci_get_drvdata(pdev);
3137         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3138         int size = 0;
3139         int num_rx_queues;
3140
3141 #ifdef VMXNET3_RSS
3142         if (enable_mq)
3143                 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3144                                     (int)num_online_cpus());
3145         else
3146 #endif
3147                 num_rx_queues = 1;
3148         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3149
3150         cancel_work_sync(&adapter->work);
3151
3152         unregister_netdev(netdev);
3153
3154         vmxnet3_free_intr_resources(adapter);
3155         vmxnet3_free_pci_resources(adapter);
3156 #ifdef VMXNET3_RSS
3157         dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3158                           adapter->rss_conf, adapter->rss_conf_pa);
3159 #endif
3160         dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3161                           adapter->pm_conf, adapter->pm_conf_pa);
3162
3163         size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3164         size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3165         dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3166                           adapter->queue_desc_pa);
3167         dma_free_coherent(&adapter->pdev->dev,
3168                           sizeof(struct Vmxnet3_DriverShared),
3169                           adapter->shared, adapter->shared_pa);
3170         dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3171                          sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3172         free_netdev(netdev);
3173 }
3174
3175
3176 #ifdef CONFIG_PM
3177
3178 static int
3179 vmxnet3_suspend(struct device *device)
3180 {
3181         struct pci_dev *pdev = to_pci_dev(device);
3182         struct net_device *netdev = pci_get_drvdata(pdev);
3183         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3184         struct Vmxnet3_PMConf *pmConf;
3185         struct ethhdr *ehdr;
3186         struct arphdr *ahdr;
3187         u8 *arpreq;
3188         struct in_device *in_dev;
3189         struct in_ifaddr *ifa;
3190         unsigned long flags;
3191         int i = 0;
3192
3193         if (!netif_running(netdev))
3194                 return 0;
3195
3196         for (i = 0; i < adapter->num_rx_queues; i++)
3197                 napi_disable(&adapter->rx_queue[i].napi);
3198
3199         vmxnet3_disable_all_intrs(adapter);
3200         vmxnet3_free_irqs(adapter);
3201         vmxnet3_free_intr_resources(adapter);
3202
3203         netif_device_detach(netdev);
3204         netif_tx_stop_all_queues(netdev);
3205
3206         /* Create wake-up filters. */
3207         pmConf = adapter->pm_conf;
3208         memset(pmConf, 0, sizeof(*pmConf));
3209
3210         if (adapter->wol & WAKE_UCAST) {
3211                 pmConf->filters[i].patternSize = ETH_ALEN;
3212                 pmConf->filters[i].maskSize = 1;
3213                 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3214                 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3215
3216                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3217                 i++;
3218         }
3219
3220         if (adapter->wol & WAKE_ARP) {
3221                 in_dev = in_dev_get(netdev);
3222                 if (!in_dev)
3223                         goto skip_arp;
3224
3225                 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3226                 if (!ifa)
3227                         goto skip_arp;
3228
3229                 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3230                         sizeof(struct arphdr) +         /* ARP header */
3231                         2 * ETH_ALEN +          /* 2 Ethernet addresses*/
3232                         2 * sizeof(u32);        /*2 IPv4 addresses */
3233                 pmConf->filters[i].maskSize =
3234                         (pmConf->filters[i].patternSize - 1) / 8 + 1;
3235
3236                 /* ETH_P_ARP in Ethernet header. */
3237                 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3238                 ehdr->h_proto = htons(ETH_P_ARP);
3239
3240                 /* ARPOP_REQUEST in ARP header. */
3241                 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3242                 ahdr->ar_op = htons(ARPOP_REQUEST);
3243                 arpreq = (u8 *)(ahdr + 1);
3244
3245                 /* The Unicast IPv4 address in 'tip' field. */
3246                 arpreq += 2 * ETH_ALEN + sizeof(u32);
3247                 *(u32 *)arpreq = ifa->ifa_address;
3248
3249                 /* The mask for the relevant bits. */
3250                 pmConf->filters[i].mask[0] = 0x00;
3251                 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3252                 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3253                 pmConf->filters[i].mask[3] = 0x00;
3254                 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3255                 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3256                 in_dev_put(in_dev);
3257
3258                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3259                 i++;
3260         }
3261
3262 skip_arp:
3263         if (adapter->wol & WAKE_MAGIC)
3264                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3265
3266         pmConf->numFilters = i;
3267
3268         adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3269         adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3270                                                                   *pmConf));
3271         adapter->shared->devRead.pmConfDesc.confPA =
3272                 cpu_to_le64(adapter->pm_conf_pa);
3273
3274         spin_lock_irqsave(&adapter->cmd_lock, flags);
3275         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3276                                VMXNET3_CMD_UPDATE_PMCFG);
3277         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3278
3279         pci_save_state(pdev);
3280         pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3281                         adapter->wol);
3282         pci_disable_device(pdev);
3283         pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3284
3285         return 0;
3286 }
3287
3288
3289 static int
3290 vmxnet3_resume(struct device *device)
3291 {
3292         int err, i = 0;
3293         unsigned long flags;
3294         struct pci_dev *pdev = to_pci_dev(device);
3295         struct net_device *netdev = pci_get_drvdata(pdev);
3296         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3297         struct Vmxnet3_PMConf *pmConf;
3298
3299         if (!netif_running(netdev))
3300                 return 0;
3301
3302         /* Destroy wake-up filters. */
3303         pmConf = adapter->pm_conf;
3304         memset(pmConf, 0, sizeof(*pmConf));
3305
3306         adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3307         adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3308                                                                   *pmConf));
3309         adapter->shared->devRead.pmConfDesc.confPA =
3310                 cpu_to_le64(adapter->pm_conf_pa);
3311
3312         netif_device_attach(netdev);
3313         pci_set_power_state(pdev, PCI_D0);
3314         pci_restore_state(pdev);
3315         err = pci_enable_device_mem(pdev);
3316         if (err != 0)
3317                 return err;
3318
3319         pci_enable_wake(pdev, PCI_D0, 0);
3320
3321         spin_lock_irqsave(&adapter->cmd_lock, flags);
3322         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3323                                VMXNET3_CMD_UPDATE_PMCFG);
3324         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3325         vmxnet3_alloc_intr_resources(adapter);
3326         vmxnet3_request_irqs(adapter);
3327         for (i = 0; i < adapter->num_rx_queues; i++)
3328                 napi_enable(&adapter->rx_queue[i].napi);
3329         vmxnet3_enable_all_intrs(adapter);
3330
3331         return 0;
3332 }
3333
3334 static const struct dev_pm_ops vmxnet3_pm_ops = {
3335         .suspend = vmxnet3_suspend,
3336         .resume = vmxnet3_resume,
3337 };
3338 #endif
3339
3340 static struct pci_driver vmxnet3_driver = {
3341         .name           = vmxnet3_driver_name,
3342         .id_table       = vmxnet3_pciid_table,
3343         .probe          = vmxnet3_probe_device,
3344         .remove         = vmxnet3_remove_device,
3345 #ifdef CONFIG_PM
3346         .driver.pm      = &vmxnet3_pm_ops,
3347 #endif
3348 };
3349
3350
3351 static int __init
3352 vmxnet3_init_module(void)
3353 {
3354         pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3355                 VMXNET3_DRIVER_VERSION_REPORT);
3356         return pci_register_driver(&vmxnet3_driver);
3357 }
3358
3359 module_init(vmxnet3_init_module);
3360
3361
3362 static void
3363 vmxnet3_exit_module(void)
3364 {
3365         pci_unregister_driver(&vmxnet3_driver);
3366 }
3367
3368 module_exit(vmxnet3_exit_module);
3369
3370 MODULE_AUTHOR("VMware, Inc.");
3371 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3372 MODULE_LICENSE("GPL v2");
3373 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);