drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9
  10 #include <linux/module.h>
  11
  12 #include <linux/stringify.h>
  13 #include <linux/kernel.h>
  14 #include <linux/timer.h>
  15 #include <linux/errno.h>
  16 #include <linux/ioport.h>
  17 #include <linux/slab.h>
  18 #include <linux/vmalloc.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/etherdevice.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/bitops.h>
  26 #include <linux/io.h>
  27 #include <linux/irq.h>
  28 #include <linux/delay.h>
  29 #include <asm/byteorder.h>
  30 #include <asm/page.h>
  31 #include <linux/time.h>
  32 #include <linux/mii.h>
  33 #include <linux/if.h>
  34 #include <linux/if_vlan.h>
  35 #include <net/ip.h>
  36 #include <net/tcp.h>
  37 #include <net/udp.h>
  38 #include <net/checksum.h>
  39 #include <net/ip6_checksum.h>
  40 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
  41 #include <net/vxlan.h>
  42 #endif
  43 #ifdef CONFIG_NET_RX_BUSY_POLL
  44 #include <net/busy_poll.h>
  45 #endif
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53
  54 #include "bnxt_hsi.h"
  55 #include "bnxt.h"
  56 #include "bnxt_sriov.h"
  57 #include "bnxt_ethtool.h"
  58
  59 #define BNXT_TX_TIMEOUT         (5 * HZ)
  60
  61 static const char version[] =
  62         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  63
  64 MODULE_LICENSE("GPL");
  65 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  66 MODULE_VERSION(DRV_MODULE_VERSION);
  67
  68 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  69 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  70 #define BNXT_RX_COPY_THRESH 256
  71
  72 #define BNXT_TX_PUSH_THRESH 164
  73
  74 enum board_idx {
  75         BCM57301,
  76         BCM57302,
  77         BCM57304,
  78         BCM57402,
  79         BCM57404,
  80         BCM57406,
  81         BCM57304_VF,
  82         BCM57404_VF,
  83 };
  84
  85 /* indexed by enum above */
  86 static const struct {
  87         char *name;
  88 } board_info[] = {
  89         { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
  90         { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
  91         { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
  92         { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
  93         { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
  94         { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
  95         { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
  96         { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
  97 };
  98
  99 static const struct pci_device_id bnxt_pci_tbl[] = {
 100         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 101         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 102         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 103         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 104         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 105         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 106 #ifdef CONFIG_BNXT_SRIOV
 107         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
 108         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
 109 #endif
 110         { 0 }
 111 };
 112
 113 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 114
 115 static const u16 bnxt_vf_req_snif[] = {
 116         HWRM_FUNC_CFG,
 117         HWRM_PORT_PHY_QCFG,
 118         HWRM_CFA_L2_FILTER_ALLOC,
 119 };
 120
 121 static const u16 bnxt_async_events_arr[] = {
 122         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 123         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 124         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 125 };
 126
 127 static bool bnxt_vf_pciid(enum board_idx idx)
 128 {
 129         return (idx == BCM57304_VF || idx == BCM57404_VF);
 130 }
 131
 132 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 133 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 134 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 135
 136 #define BNXT_CP_DB_REARM(db, raw_cons)                                  \
 137                 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
 138
 139 #define BNXT_CP_DB(db, raw_cons)                                        \
 140                 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
 141
 142 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 143                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 144
 145 static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
 146 {
 147         /* Tell compiler to fetch tx indices from memory. */
 148         barrier();
 149
 150         return bp->tx_ring_size -
 151                 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
 152 }
 153
 154 static const u16 bnxt_lhint_arr[] = {
 155         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 156         TX_BD_FLAGS_LHINT_512_TO_1023,
 157         TX_BD_FLAGS_LHINT_1024_TO_2047,
 158         TX_BD_FLAGS_LHINT_1024_TO_2047,
 159         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 160         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 161         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 162         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 163         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 164         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 165         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 166         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 167         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 168         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 169         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 170         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 171         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 172         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 173         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 174 };
 175
 176 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 177 {
 178         struct bnxt *bp = netdev_priv(dev);
 179         struct tx_bd *txbd;
 180         struct tx_bd_ext *txbd1;
 181         struct netdev_queue *txq;
 182         int i;
 183         dma_addr_t mapping;
 184         unsigned int length, pad = 0;
 185         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 186         u16 prod, last_frag;
 187         struct pci_dev *pdev = bp->pdev;
 188         struct bnxt_tx_ring_info *txr;
 189         struct bnxt_sw_tx_bd *tx_buf;
 190
 191         i = skb_get_queue_mapping(skb);
 192         if (unlikely(i >= bp->tx_nr_rings)) {
 193                 dev_kfree_skb_any(skb);
 194                 return NETDEV_TX_OK;
 195         }
 196
 197         txr = &bp->tx_ring[i];
 198         txq = netdev_get_tx_queue(dev, i);
 199         prod = txr->tx_prod;
 200
 201         free_size = bnxt_tx_avail(bp, txr);
 202         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 203                 netif_tx_stop_queue(txq);
 204                 return NETDEV_TX_BUSY;
 205         }
 206
 207         length = skb->len;
 208         len = skb_headlen(skb);
 209         last_frag = skb_shinfo(skb)->nr_frags;
 210
 211         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 212
 213         txbd->tx_bd_opaque = prod;
 214
 215         tx_buf = &txr->tx_buf_ring[prod];
 216         tx_buf->skb = skb;
 217         tx_buf->nr_frags = last_frag;
 218
 219         vlan_tag_flags = 0;
 220         cfa_action = 0;
 221         if (skb_vlan_tag_present(skb)) {
 222                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 223                                  skb_vlan_tag_get(skb);
 224                 /* Currently supports 8021Q, 8021AD vlan offloads
 225                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 226                  */
 227                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 228                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 229         }
 230
 231         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 232                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 233                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 234                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 235                 void *pdata = tx_push_buf->data;
 236                 u64 *end;
 237                 int j, push_len;
 238
 239                 /* Set COAL_NOW to be ready quickly for the next push */
 240                 tx_push->tx_bd_len_flags_type =
 241                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 242                                         TX_BD_TYPE_LONG_TX_BD |
 243                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 244                                         TX_BD_FLAGS_COAL_NOW |
 245                                         TX_BD_FLAGS_PACKET_END |
 246                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 247
 248                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 249                         tx_push1->tx_bd_hsize_lflags =
 250                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 251                 else
 252                         tx_push1->tx_bd_hsize_lflags = 0;
 253
 254                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 255                 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
 256
 257                 end = pdata + length;
 258                 end = PTR_ALIGN(end, 8) - 1;
 259                 *end = 0;
 260
 261                 skb_copy_from_linear_data(skb, pdata, len);
 262                 pdata += len;
 263                 for (j = 0; j < last_frag; j++) {
 264                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 265                         void *fptr;
 266
 267                         fptr = skb_frag_address_safe(frag);
 268                         if (!fptr)
 269                                 goto normal_tx;
 270
 271                         memcpy(pdata, fptr, skb_frag_size(frag));
 272                         pdata += skb_frag_size(frag);
 273                 }
 274
 275                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 276                 txbd->tx_bd_haddr = txr->data_mapping;
 277                 prod = NEXT_TX(prod);
 278                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 279                 memcpy(txbd, tx_push1, sizeof(*txbd));
 280                 prod = NEXT_TX(prod);
 281                 tx_push->doorbell =
 282                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 283                 txr->tx_prod = prod;
 284
 285                 netdev_tx_sent_queue(txq, skb->len);
 286
 287                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 288                 if (push_len > 16) {
 289                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
 290                         __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
 291                                          push_len - 16);
 292                 } else {
 293                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 294                                          push_len);
 295                 }
 296
 297                 tx_buf->is_push = 1;
 298                 goto tx_done;
 299         }
 300
 301 normal_tx:
 302         if (length < BNXT_MIN_PKT_SIZE) {
 303                 pad = BNXT_MIN_PKT_SIZE - length;
 304                 if (skb_pad(skb, pad)) {
 305                         /* SKB already freed. */
 306                         tx_buf->skb = NULL;
 307                         return NETDEV_TX_OK;
 308                 }
 309                 length = BNXT_MIN_PKT_SIZE;
 310         }
 311
 312         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 313
 314         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 315                 dev_kfree_skb_any(skb);
 316                 tx_buf->skb = NULL;
 317                 return NETDEV_TX_OK;
 318         }
 319
 320         dma_unmap_addr_set(tx_buf, mapping, mapping);
 321         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 322                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 323
 324         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 325
 326         prod = NEXT_TX(prod);
 327         txbd1 = (struct tx_bd_ext *)
 328                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 329
 330         txbd1->tx_bd_hsize_lflags = 0;
 331         if (skb_is_gso(skb)) {
 332                 u32 hdr_len;
 333
 334                 if (skb->encapsulation)
 335                         hdr_len = skb_inner_network_offset(skb) +
 336                                 skb_inner_network_header_len(skb) +
 337                                 inner_tcp_hdrlen(skb);
 338                 else
 339                         hdr_len = skb_transport_offset(skb) +
 340                                 tcp_hdrlen(skb);
 341
 342                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 343                                         TX_BD_FLAGS_T_IPID |
 344                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 345                 length = skb_shinfo(skb)->gso_size;
 346                 txbd1->tx_bd_mss = cpu_to_le32(length);
 347                 length += hdr_len;
 348         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 349                 txbd1->tx_bd_hsize_lflags =
 350                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 351                 txbd1->tx_bd_mss = 0;
 352         }
 353
 354         length >>= 9;
 355         flags |= bnxt_lhint_arr[length];
 356         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 357
 358         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 359         txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
 360         for (i = 0; i < last_frag; i++) {
 361                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 362
 363                 prod = NEXT_TX(prod);
 364                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 365
 366                 len = skb_frag_size(frag);
 367                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 368                                            DMA_TO_DEVICE);
 369
 370                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 371                         goto tx_dma_error;
 372
 373                 tx_buf = &txr->tx_buf_ring[prod];
 374                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 375
 376                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 377
 378                 flags = len << TX_BD_LEN_SHIFT;
 379                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 380         }
 381
 382         flags &= ~TX_BD_LEN;
 383         txbd->tx_bd_len_flags_type =
 384                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 385                             TX_BD_FLAGS_PACKET_END);
 386
 387         netdev_tx_sent_queue(txq, skb->len);
 388
 389         /* Sync BD data before updating doorbell */
 390         wmb();
 391
 392         prod = NEXT_TX(prod);
 393         txr->tx_prod = prod;
 394
 395         writel(DB_KEY_TX | prod, txr->tx_doorbell);
 396         writel(DB_KEY_TX | prod, txr->tx_doorbell);
 397
 398 tx_done:
 399
 400         mmiowb();
 401
 402         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 403                 netif_tx_stop_queue(txq);
 404
 405                 /* netif_tx_stop_queue() must be done before checking
 406                  * tx index in bnxt_tx_avail() below, because in
 407                  * bnxt_tx_int(), we update tx index before checking for
 408                  * netif_tx_queue_stopped().
 409                  */
 410                 smp_mb();
 411                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 412                         netif_tx_wake_queue(txq);
 413         }
 414         return NETDEV_TX_OK;
 415
 416 tx_dma_error:
 417         last_frag = i;
 418
 419         /* start back at beginning and unmap skb */
 420         prod = txr->tx_prod;
 421         tx_buf = &txr->tx_buf_ring[prod];
 422         tx_buf->skb = NULL;
 423         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 424                          skb_headlen(skb), PCI_DMA_TODEVICE);
 425         prod = NEXT_TX(prod);
 426
 427         /* unmap remaining mapped pages */
 428         for (i = 0; i < last_frag; i++) {
 429                 prod = NEXT_TX(prod);
 430                 tx_buf = &txr->tx_buf_ring[prod];
 431                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 432                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 433                                PCI_DMA_TODEVICE);
 434         }
 435
 436         dev_kfree_skb_any(skb);
 437         return NETDEV_TX_OK;
 438 }
 439
 440 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 441 {
 442         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 443         int index = txr - &bp->tx_ring[0];
 444         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index);
 445         u16 cons = txr->tx_cons;
 446         struct pci_dev *pdev = bp->pdev;
 447         int i;
 448         unsigned int tx_bytes = 0;
 449
 450         for (i = 0; i < nr_pkts; i++) {
 451                 struct bnxt_sw_tx_bd *tx_buf;
 452                 struct sk_buff *skb;
 453                 int j, last;
 454
 455                 tx_buf = &txr->tx_buf_ring[cons];
 456                 cons = NEXT_TX(cons);
 457                 skb = tx_buf->skb;
 458                 tx_buf->skb = NULL;
 459
 460                 if (tx_buf->is_push) {
 461                         tx_buf->is_push = 0;
 462                         goto next_tx_int;
 463                 }
 464
 465                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 466                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 467                 last = tx_buf->nr_frags;
 468
 469                 for (j = 0; j < last; j++) {
 470                         cons = NEXT_TX(cons);
 471                         tx_buf = &txr->tx_buf_ring[cons];
 472                         dma_unmap_page(
 473                                 &pdev->dev,
 474                                 dma_unmap_addr(tx_buf, mapping),
 475                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 476                                 PCI_DMA_TODEVICE);
 477                 }
 478
 479 next_tx_int:
 480                 cons = NEXT_TX(cons);
 481
 482                 tx_bytes += skb->len;
 483                 dev_kfree_skb_any(skb);
 484         }
 485
 486         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 487         txr->tx_cons = cons;
 488
 489         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 490          * before checking for netif_tx_queue_stopped().  Without the
 491          * memory barrier, there is a small possibility that bnxt_start_xmit()
 492          * will miss it and cause the queue to be stopped forever.
 493          */
 494         smp_mb();
 495
 496         if (unlikely(netif_tx_queue_stopped(txq)) &&
 497             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 498                 __netif_tx_lock(txq, smp_processor_id());
 499                 if (netif_tx_queue_stopped(txq) &&
 500                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 501                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 502                         netif_tx_wake_queue(txq);
 503                 __netif_tx_unlock(txq);
 504         }
 505 }
 506
 507 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 508                                        gfp_t gfp)
 509 {
 510         u8 *data;
 511         struct pci_dev *pdev = bp->pdev;
 512
 513         data = kmalloc(bp->rx_buf_size, gfp);
 514         if (!data)
 515                 return NULL;
 516
 517         *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET,
 518                                   bp->rx_buf_use_size, PCI_DMA_FROMDEVICE);
 519
 520         if (dma_mapping_error(&pdev->dev, *mapping)) {
 521                 kfree(data);
 522                 data = NULL;
 523         }
 524         return data;
 525 }
 526
 527 static inline int bnxt_alloc_rx_data(struct bnxt *bp,
 528                                      struct bnxt_rx_ring_info *rxr,
 529                                      u16 prod, gfp_t gfp)
 530 {
 531         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 532         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 533         u8 *data;
 534         dma_addr_t mapping;
 535
 536         data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 537         if (!data)
 538                 return -ENOMEM;
 539
 540         rx_buf->data = data;
 541         dma_unmap_addr_set(rx_buf, mapping, mapping);
 542
 543         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 544
 545         return 0;
 546 }
 547
 548 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,
 549                                u8 *data)
 550 {
 551         u16 prod = rxr->rx_prod;
 552         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 553         struct rx_bd *cons_bd, *prod_bd;
 554
 555         prod_rx_buf = &rxr->rx_buf_ring[prod];
 556         cons_rx_buf = &rxr->rx_buf_ring[cons];
 557
 558         prod_rx_buf->data = data;
 559
 560         dma_unmap_addr_set(prod_rx_buf, mapping,
 561                            dma_unmap_addr(cons_rx_buf, mapping));
 562
 563         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 564         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 565
 566         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 567 }
 568
 569 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 570 {
 571         u16 next, max = rxr->rx_agg_bmap_size;
 572
 573         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 574         if (next >= max)
 575                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 576         return next;
 577 }
 578
 579 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 580                                      struct bnxt_rx_ring_info *rxr,
 581                                      u16 prod, gfp_t gfp)
 582 {
 583         struct rx_bd *rxbd =
 584                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 585         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 586         struct pci_dev *pdev = bp->pdev;
 587         struct page *page;
 588         dma_addr_t mapping;
 589         u16 sw_prod = rxr->rx_sw_agg_prod;
 590
 591         page = alloc_page(gfp);
 592         if (!page)
 593                 return -ENOMEM;
 594
 595         mapping = dma_map_page(&pdev->dev, page, 0, PAGE_SIZE,
 596                                PCI_DMA_FROMDEVICE);
 597         if (dma_mapping_error(&pdev->dev, mapping)) {
 598                 __free_page(page);
 599                 return -EIO;
 600         }
 601
 602         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 603                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 604
 605         __set_bit(sw_prod, rxr->rx_agg_bmap);
 606         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 607         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 608
 609         rx_agg_buf->page = page;
 610         rx_agg_buf->mapping = mapping;
 611         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 612         rxbd->rx_bd_opaque = sw_prod;
 613         return 0;
 614 }
 615
 616 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
 617                                    u32 agg_bufs)
 618 {
 619         struct bnxt *bp = bnapi->bp;
 620         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 621         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 622         u16 prod = rxr->rx_agg_prod;
 623         u16 sw_prod = rxr->rx_sw_agg_prod;
 624         u32 i;
 625
 626         for (i = 0; i < agg_bufs; i++) {
 627                 u16 cons;
 628                 struct rx_agg_cmp *agg;
 629                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 630                 struct rx_bd *prod_bd;
 631                 struct page *page;
 632
 633                 agg = (struct rx_agg_cmp *)
 634                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 635                 cons = agg->rx_agg_cmp_opaque;
 636                 __clear_bit(cons, rxr->rx_agg_bmap);
 637
 638                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 639                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 640
 641                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 642                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 643                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 644
 645                 /* It is possible for sw_prod to be equal to cons, so
 646                  * set cons_rx_buf->page to NULL first.
 647                  */
 648                 page = cons_rx_buf->page;
 649                 cons_rx_buf->page = NULL;
 650                 prod_rx_buf->page = page;
 651
 652                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 653
 654                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 655
 656                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 657                 prod_bd->rx_bd_opaque = sw_prod;
 658
 659                 prod = NEXT_RX_AGG(prod);
 660                 sw_prod = NEXT_RX_AGG(sw_prod);
 661                 cp_cons = NEXT_CMP(cp_cons);
 662         }
 663         rxr->rx_agg_prod = prod;
 664         rxr->rx_sw_agg_prod = sw_prod;
 665 }
 666
 667 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 668                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 669                                    u16 prod, u8 *data, dma_addr_t dma_addr,
 670                                    unsigned int len)
 671 {
 672         int err;
 673         struct sk_buff *skb;
 674
 675         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 676         if (unlikely(err)) {
 677                 bnxt_reuse_rx_data(rxr, cons, data);
 678                 return NULL;
 679         }
 680
 681         skb = build_skb(data, 0);
 682         dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 683                          PCI_DMA_FROMDEVICE);
 684         if (!skb) {
 685                 kfree(data);
 686                 return NULL;
 687         }
 688
 689         skb_reserve(skb, BNXT_RX_OFFSET);
 690         skb_put(skb, len);
 691         return skb;
 692 }
 693
 694 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
 695                                      struct sk_buff *skb, u16 cp_cons,
 696                                      u32 agg_bufs)
 697 {
 698         struct pci_dev *pdev = bp->pdev;
 699         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 700         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 701         u16 prod = rxr->rx_agg_prod;
 702         u32 i;
 703
 704         for (i = 0; i < agg_bufs; i++) {
 705                 u16 cons, frag_len;
 706                 struct rx_agg_cmp *agg;
 707                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 708                 struct page *page;
 709                 dma_addr_t mapping;
 710
 711                 agg = (struct rx_agg_cmp *)
 712                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 713                 cons = agg->rx_agg_cmp_opaque;
 714                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 715                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 716
 717                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 718                 skb_fill_page_desc(skb, i, cons_rx_buf->page, 0, frag_len);
 719                 __clear_bit(cons, rxr->rx_agg_bmap);
 720
 721                 /* It is possible for bnxt_alloc_rx_page() to allocate
 722                  * a sw_prod index that equals the cons index, so we
 723                  * need to clear the cons entry now.
 724                  */
 725                 mapping = dma_unmap_addr(cons_rx_buf, mapping);
 726                 page = cons_rx_buf->page;
 727                 cons_rx_buf->page = NULL;
 728
 729                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 730                         struct skb_shared_info *shinfo;
 731                         unsigned int nr_frags;
 732
 733                         shinfo = skb_shinfo(skb);
 734                         nr_frags = --shinfo->nr_frags;
 735                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 736
 737                         dev_kfree_skb(skb);
 738
 739                         cons_rx_buf->page = page;
 740
 741                         /* Update prod since possibly some pages have been
 742                          * allocated already.
 743                          */
 744                         rxr->rx_agg_prod = prod;
 745                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
 746                         return NULL;
 747                 }
 748
 749                 dma_unmap_page(&pdev->dev, mapping, PAGE_SIZE,
 750                                PCI_DMA_FROMDEVICE);
 751
 752                 skb->data_len += frag_len;
 753                 skb->len += frag_len;
 754                 skb->truesize += PAGE_SIZE;
 755
 756                 prod = NEXT_RX_AGG(prod);
 757                 cp_cons = NEXT_CMP(cp_cons);
 758         }
 759         rxr->rx_agg_prod = prod;
 760         return skb;
 761 }
 762
 763 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 764                                u8 agg_bufs, u32 *raw_cons)
 765 {
 766         u16 last;
 767         struct rx_agg_cmp *agg;
 768
 769         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
 770         last = RING_CMP(*raw_cons);
 771         agg = (struct rx_agg_cmp *)
 772                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
 773         return RX_AGG_CMP_VALID(agg, *raw_cons);
 774 }
 775
 776 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
 777                                             unsigned int len,
 778                                             dma_addr_t mapping)
 779 {
 780         struct bnxt *bp = bnapi->bp;
 781         struct pci_dev *pdev = bp->pdev;
 782         struct sk_buff *skb;
 783
 784         skb = napi_alloc_skb(&bnapi->napi, len);
 785         if (!skb)
 786                 return NULL;
 787
 788         dma_sync_single_for_cpu(&pdev->dev, mapping,
 789                                 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE);
 790
 791         memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET);
 792
 793         dma_sync_single_for_device(&pdev->dev, mapping,
 794                                    bp->rx_copy_thresh,
 795                                    PCI_DMA_FROMDEVICE);
 796
 797         skb_put(skb, len);
 798         return skb;
 799 }
 800
 801 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 802                            struct rx_tpa_start_cmp *tpa_start,
 803                            struct rx_tpa_start_cmp_ext *tpa_start1)
 804 {
 805         u8 agg_id = TPA_START_AGG_ID(tpa_start);
 806         u16 cons, prod;
 807         struct bnxt_tpa_info *tpa_info;
 808         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 809         struct rx_bd *prod_bd;
 810         dma_addr_t mapping;
 811
 812         cons = tpa_start->rx_tpa_start_cmp_opaque;
 813         prod = rxr->rx_prod;
 814         cons_rx_buf = &rxr->rx_buf_ring[cons];
 815         prod_rx_buf = &rxr->rx_buf_ring[prod];
 816         tpa_info = &rxr->rx_tpa[agg_id];
 817
 818         prod_rx_buf->data = tpa_info->data;
 819
 820         mapping = tpa_info->mapping;
 821         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 822
 823         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 824
 825         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
 826
 827         tpa_info->data = cons_rx_buf->data;
 828         cons_rx_buf->data = NULL;
 829         tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping);
 830
 831         tpa_info->len =
 832                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
 833                                 RX_TPA_START_CMP_LEN_SHIFT;
 834         if (likely(TPA_START_HASH_VALID(tpa_start))) {
 835                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
 836
 837                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
 838                 tpa_info->gso_type = SKB_GSO_TCPV4;
 839                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
 840                 if (hash_type == 3)
 841                         tpa_info->gso_type = SKB_GSO_TCPV6;
 842                 tpa_info->rss_hash =
 843                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
 844         } else {
 845                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
 846                 tpa_info->gso_type = 0;
 847                 if (netif_msg_rx_err(bp))
 848                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
 849         }
 850         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
 851         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
 852
 853         rxr->rx_prod = NEXT_RX(prod);
 854         cons = NEXT_RX(cons);
 855         cons_rx_buf = &rxr->rx_buf_ring[cons];
 856
 857         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
 858         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
 859         cons_rx_buf->data = NULL;
 860 }
 861
 862 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
 863                            u16 cp_cons, u32 agg_bufs)
 864 {
 865         if (agg_bufs)
 866                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
 867 }
 868
 869 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
 870 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
 871
 872 static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
 873                                            struct rx_tpa_end_cmp *tpa_end,
 874                                            struct rx_tpa_end_cmp_ext *tpa_end1,
 875                                            struct sk_buff *skb)
 876 {
 877 #ifdef CONFIG_INET
 878         struct tcphdr *th;
 879         int payload_off, tcp_opt_len = 0;
 880         int len, nw_off;
 881         u16 segs;
 882
 883         segs = TPA_END_TPA_SEGS(tpa_end);
 884         if (segs == 1)
 885                 return skb;
 886
 887         NAPI_GRO_CB(skb)->count = segs;
 888         skb_shinfo(skb)->gso_size =
 889                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
 890         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
 891         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
 892                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
 893                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
 894         if (TPA_END_GRO_TS(tpa_end))
 895                 tcp_opt_len = 12;
 896
 897         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
 898                 struct iphdr *iph;
 899
 900                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
 901                          ETH_HLEN;
 902                 skb_set_network_header(skb, nw_off);
 903                 iph = ip_hdr(skb);
 904                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
 905                 len = skb->len - skb_transport_offset(skb);
 906                 th = tcp_hdr(skb);
 907                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
 908         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
 909                 struct ipv6hdr *iph;
 910
 911                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
 912                          ETH_HLEN;
 913                 skb_set_network_header(skb, nw_off);
 914                 iph = ipv6_hdr(skb);
 915                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
 916                 len = skb->len - skb_transport_offset(skb);
 917                 th = tcp_hdr(skb);
 918                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
 919         } else {
 920                 dev_kfree_skb_any(skb);
 921                 return NULL;
 922         }
 923         tcp_gro_complete(skb);
 924
 925         if (nw_off) { /* tunnel */
 926                 struct udphdr *uh = NULL;
 927
 928                 if (skb->protocol == htons(ETH_P_IP)) {
 929                         struct iphdr *iph = (struct iphdr *)skb->data;
 930
 931                         if (iph->protocol == IPPROTO_UDP)
 932                                 uh = (struct udphdr *)(iph + 1);
 933                 } else {
 934                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
 935
 936                         if (iph->nexthdr == IPPROTO_UDP)
 937                                 uh = (struct udphdr *)(iph + 1);
 938                 }
 939                 if (uh) {
 940                         if (uh->check)
 941                                 skb_shinfo(skb)->gso_type |=
 942                                         SKB_GSO_UDP_TUNNEL_CSUM;
 943                         else
 944                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
 945                 }
 946         }
 947 #endif
 948         return skb;
 949 }
 950
 951 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
 952                                            struct bnxt_napi *bnapi,
 953                                            u32 *raw_cons,
 954                                            struct rx_tpa_end_cmp *tpa_end,
 955                                            struct rx_tpa_end_cmp_ext *tpa_end1,
 956                                            bool *agg_event)
 957 {
 958         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 959         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 960         u8 agg_id = TPA_END_AGG_ID(tpa_end);
 961         u8 *data, agg_bufs;
 962         u16 cp_cons = RING_CMP(*raw_cons);
 963         unsigned int len;
 964         struct bnxt_tpa_info *tpa_info;
 965         dma_addr_t mapping;
 966         struct sk_buff *skb;
 967
 968         tpa_info = &rxr->rx_tpa[agg_id];
 969         data = tpa_info->data;
 970         prefetch(data);
 971         len = tpa_info->len;
 972         mapping = tpa_info->mapping;
 973
 974         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
 975                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
 976
 977         if (agg_bufs) {
 978                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
 979                         return ERR_PTR(-EBUSY);
 980
 981                 *agg_event = true;
 982                 cp_cons = NEXT_CMP(cp_cons);
 983         }
 984
 985         if (unlikely(agg_bufs > MAX_SKB_FRAGS)) {
 986                 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
 987                 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
 988                             agg_bufs, (int)MAX_SKB_FRAGS);
 989                 return NULL;
 990         }
 991
 992         if (len <= bp->rx_copy_thresh) {
 993                 skb = bnxt_copy_skb(bnapi, data, len, mapping);
 994                 if (!skb) {
 995                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
 996                         return NULL;
 997                 }
 998         } else {
 999                 u8 *new_data;
1000                 dma_addr_t new_mapping;
1001
1002                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1003                 if (!new_data) {
1004                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1005                         return NULL;
1006                 }
1007
1008                 tpa_info->data = new_data;
1009                 tpa_info->mapping = new_mapping;
1010
1011                 skb = build_skb(data, 0);
1012                 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size,
1013                                  PCI_DMA_FROMDEVICE);
1014
1015                 if (!skb) {
1016                         kfree(data);
1017                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1018                         return NULL;
1019                 }
1020                 skb_reserve(skb, BNXT_RX_OFFSET);
1021                 skb_put(skb, len);
1022         }
1023
1024         if (agg_bufs) {
1025                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1026                 if (!skb) {
1027                         /* Page reuse already handled by bnxt_rx_pages(). */
1028                         return NULL;
1029                 }
1030         }
1031         skb->protocol = eth_type_trans(skb, bp->dev);
1032
1033         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1034                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1035
1036         if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1037                 netdev_features_t features = skb->dev->features;
1038                 u16 vlan_proto = tpa_info->metadata >>
1039                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1040
1041                 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1042                      vlan_proto == ETH_P_8021Q) ||
1043                     ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1044                      vlan_proto == ETH_P_8021AD)) {
1045                         __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1046                                                tpa_info->metadata &
1047                                                RX_CMP_FLAGS2_METADATA_VID_MASK);
1048                 }
1049         }
1050
1051         skb_checksum_none_assert(skb);
1052         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1053                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1054                 skb->csum_level =
1055                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1056         }
1057
1058         if (TPA_END_GRO(tpa_end))
1059                 skb = bnxt_gro_skb(tpa_info, tpa_end, tpa_end1, skb);
1060
1061         return skb;
1062 }
1063
1064 /* returns the following:
1065  * 1       - 1 packet successfully received
1066  * 0       - successful TPA_START, packet not completed yet
1067  * -EBUSY  - completion ring does not have all the agg buffers yet
1068  * -ENOMEM - packet aborted due to out of memory
1069  * -EIO    - packet aborted due to hw error indicated in BD
1070  */
1071 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1072                        bool *agg_event)
1073 {
1074         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1075         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1076         struct net_device *dev = bp->dev;
1077         struct rx_cmp *rxcmp;
1078         struct rx_cmp_ext *rxcmp1;
1079         u32 tmp_raw_cons = *raw_cons;
1080         u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1081         struct bnxt_sw_rx_bd *rx_buf;
1082         unsigned int len;
1083         u8 *data, agg_bufs, cmp_type;
1084         dma_addr_t dma_addr;
1085         struct sk_buff *skb;
1086         int rc = 0;
1087
1088         rxcmp = (struct rx_cmp *)
1089                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1090
1091         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1092         cp_cons = RING_CMP(tmp_raw_cons);
1093         rxcmp1 = (struct rx_cmp_ext *)
1094                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1095
1096         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1097                 return -EBUSY;
1098
1099         cmp_type = RX_CMP_TYPE(rxcmp);
1100
1101         prod = rxr->rx_prod;
1102
1103         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1104                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1105                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1106
1107                 goto next_rx_no_prod;
1108
1109         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1110                 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1111                                    (struct rx_tpa_end_cmp *)rxcmp,
1112                                    (struct rx_tpa_end_cmp_ext *)rxcmp1,
1113                                    agg_event);
1114
1115                 if (unlikely(IS_ERR(skb)))
1116                         return -EBUSY;
1117
1118                 rc = -ENOMEM;
1119                 if (likely(skb)) {
1120                         skb_record_rx_queue(skb, bnapi->index);
1121                         skb_mark_napi_id(skb, &bnapi->napi);
1122                         if (bnxt_busy_polling(bnapi))
1123                                 netif_receive_skb(skb);
1124                         else
1125                                 napi_gro_receive(&bnapi->napi, skb);
1126                         rc = 1;
1127                 }
1128                 goto next_rx_no_prod;
1129         }
1130
1131         cons = rxcmp->rx_cmp_opaque;
1132         rx_buf = &rxr->rx_buf_ring[cons];
1133         data = rx_buf->data;
1134         prefetch(data);
1135
1136         agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >>
1137                                 RX_CMP_AGG_BUFS_SHIFT;
1138
1139         if (agg_bufs) {
1140                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1141                         return -EBUSY;
1142
1143                 cp_cons = NEXT_CMP(cp_cons);
1144                 *agg_event = true;
1145         }
1146
1147         rx_buf->data = NULL;
1148         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1149                 bnxt_reuse_rx_data(rxr, cons, data);
1150                 if (agg_bufs)
1151                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1152
1153                 rc = -EIO;
1154                 goto next_rx;
1155         }
1156
1157         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1158         dma_addr = dma_unmap_addr(rx_buf, mapping);
1159
1160         if (len <= bp->rx_copy_thresh) {
1161                 skb = bnxt_copy_skb(bnapi, data, len, dma_addr);
1162                 bnxt_reuse_rx_data(rxr, cons, data);
1163                 if (!skb) {
1164                         rc = -ENOMEM;
1165                         goto next_rx;
1166                 }
1167         } else {
1168                 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);
1169                 if (!skb) {
1170                         rc = -ENOMEM;
1171                         goto next_rx;
1172                 }
1173         }
1174
1175         if (agg_bufs) {
1176                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1177                 if (!skb) {
1178                         rc = -ENOMEM;
1179                         goto next_rx;
1180                 }
1181         }
1182
1183         if (RX_CMP_HASH_VALID(rxcmp)) {
1184                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1185                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1186
1187                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1188                 if (hash_type != 1 && hash_type != 3)
1189                         type = PKT_HASH_TYPE_L3;
1190                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1191         }
1192
1193         skb->protocol = eth_type_trans(skb, dev);
1194
1195         if (rxcmp1->rx_cmp_flags2 &
1196             cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) {
1197                 netdev_features_t features = skb->dev->features;
1198                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1199                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1200
1201                 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1202                      vlan_proto == ETH_P_8021Q) ||
1203                     ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1204                      vlan_proto == ETH_P_8021AD))
1205                         __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1206                                                meta_data &
1207                                                RX_CMP_FLAGS2_METADATA_VID_MASK);
1208         }
1209
1210         skb_checksum_none_assert(skb);
1211         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1212                 if (dev->features & NETIF_F_RXCSUM) {
1213                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1214                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1215                 }
1216         } else {
1217                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1218                         if (dev->features & NETIF_F_RXCSUM)
1219                                 cpr->rx_l4_csum_errors++;
1220                 }
1221         }
1222
1223         skb_record_rx_queue(skb, bnapi->index);
1224         skb_mark_napi_id(skb, &bnapi->napi);
1225         if (bnxt_busy_polling(bnapi))
1226                 netif_receive_skb(skb);
1227         else
1228                 napi_gro_receive(&bnapi->napi, skb);
1229         rc = 1;
1230
1231 next_rx:
1232         rxr->rx_prod = NEXT_RX(prod);
1233
1234 next_rx_no_prod:
1235         *raw_cons = tmp_raw_cons;
1236
1237         return rc;
1238 }
1239
1240 #define BNXT_GET_EVENT_PORT(data)       \
1241         ((data) &                               \
1242          HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1243
1244 #define BNXT_EVENT_POLICY_MASK  \
1245         HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_MASK
1246
1247 #define BNXT_EVENT_POLICY_SFT   \
1248         HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_SFT
1249
1250 #define BNXT_GET_EVENT_POLICY(data)     \
1251         (((data) & BNXT_EVENT_POLICY_MASK) >> BNXT_EVENT_POLICY_SFT)
1252
1253 static int bnxt_async_event_process(struct bnxt *bp,
1254                                     struct hwrm_async_event_cmpl *cmpl)
1255 {
1256         u16 event_id = le16_to_cpu(cmpl->event_id);
1257
1258         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1259         switch (event_id) {
1260         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1261                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1262                 break;
1263         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1264                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1265                 break;
1266         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1267                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1268                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1269
1270                 if (BNXT_VF(bp))
1271                         break;
1272
1273                 if (bp->pf.port_id != port_id)
1274                         break;
1275
1276                 bp->link_info.last_port_module_event =
1277                         BNXT_GET_EVENT_POLICY(data1);
1278
1279                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1280                 break;
1281         }
1282         default:
1283                 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
1284                            event_id);
1285                 goto async_event_process_exit;
1286         }
1287         schedule_work(&bp->sp_task);
1288 async_event_process_exit:
1289         return 0;
1290 }
1291
1292 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1293 {
1294         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1295         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1296         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1297                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1298
1299         switch (cmpl_type) {
1300         case CMPL_BASE_TYPE_HWRM_DONE:
1301                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1302                 if (seq_id == bp->hwrm_intr_seq_id)
1303                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1304                 else
1305                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1306                 break;
1307
1308         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1309                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1310
1311                 if ((vf_id < bp->pf.first_vf_id) ||
1312                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1313                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1314                                    vf_id);
1315                         return -EINVAL;
1316                 }
1317
1318                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1319                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1320                 schedule_work(&bp->sp_task);
1321                 break;
1322
1323         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1324                 bnxt_async_event_process(bp,
1325                                          (struct hwrm_async_event_cmpl *)txcmp);
1326
1327         default:
1328                 break;
1329         }
1330
1331         return 0;
1332 }
1333
1334 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1335 {
1336         struct bnxt_napi *bnapi = dev_instance;
1337         struct bnxt *bp = bnapi->bp;
1338         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1339         u32 cons = RING_CMP(cpr->cp_raw_cons);
1340
1341         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1342         napi_schedule(&bnapi->napi);
1343         return IRQ_HANDLED;
1344 }
1345
1346 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1347 {
1348         u32 raw_cons = cpr->cp_raw_cons;
1349         u16 cons = RING_CMP(raw_cons);
1350         struct tx_cmp *txcmp;
1351
1352         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1353
1354         return TX_CMP_VALID(txcmp, raw_cons);
1355 }
1356
1357 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1358 {
1359         struct bnxt_napi *bnapi = dev_instance;
1360         struct bnxt *bp = bnapi->bp;
1361         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1362         u32 cons = RING_CMP(cpr->cp_raw_cons);
1363         u32 int_status;
1364
1365         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1366
1367         if (!bnxt_has_work(bp, cpr)) {
1368                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1369                 /* return if erroneous interrupt */
1370                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1371                         return IRQ_NONE;
1372         }
1373
1374         /* disable ring IRQ */
1375         BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1376
1377         /* Return here if interrupt is shared and is disabled. */
1378         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1379                 return IRQ_HANDLED;
1380
1381         napi_schedule(&bnapi->napi);
1382         return IRQ_HANDLED;
1383 }
1384
1385 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1386 {
1387         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1388         u32 raw_cons = cpr->cp_raw_cons;
1389         u32 cons;
1390         int tx_pkts = 0;
1391         int rx_pkts = 0;
1392         bool rx_event = false;
1393         bool agg_event = false;
1394         struct tx_cmp *txcmp;
1395
1396         while (1) {
1397                 int rc;
1398
1399                 cons = RING_CMP(raw_cons);
1400                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1401
1402                 if (!TX_CMP_VALID(txcmp, raw_cons))
1403                         break;
1404
1405                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1406                         tx_pkts++;
1407                         /* return full budget so NAPI will complete. */
1408                         if (unlikely(tx_pkts > bp->tx_wake_thresh))
1409                                 rx_pkts = budget;
1410                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1411                         rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event);
1412                         if (likely(rc >= 0))
1413                                 rx_pkts += rc;
1414                         else if (rc == -EBUSY)  /* partial completion */
1415                                 break;
1416                         rx_event = true;
1417                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1418                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1419                                     (TX_CMP_TYPE(txcmp) ==
1420                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1421                                     (TX_CMP_TYPE(txcmp) ==
1422                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1423                         bnxt_hwrm_handler(bp, txcmp);
1424                 }
1425                 raw_cons = NEXT_RAW_CMP(raw_cons);
1426
1427                 if (rx_pkts == budget)
1428                         break;
1429         }
1430
1431         cpr->cp_raw_cons = raw_cons;
1432         /* ACK completion ring before freeing tx ring and producing new
1433          * buffers in rx/agg rings to prevent overflowing the completion
1434          * ring.
1435          */
1436         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1437
1438         if (tx_pkts)
1439                 bnxt_tx_int(bp, bnapi, tx_pkts);
1440
1441         if (rx_event) {
1442                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1443
1444                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1445                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1446                 if (agg_event) {
1447                         writel(DB_KEY_RX | rxr->rx_agg_prod,
1448                                rxr->rx_agg_doorbell);
1449                         writel(DB_KEY_RX | rxr->rx_agg_prod,
1450                                rxr->rx_agg_doorbell);
1451                 }
1452         }
1453         return rx_pkts;
1454 }
1455
1456 static int bnxt_poll(struct napi_struct *napi, int budget)
1457 {
1458         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1459         struct bnxt *bp = bnapi->bp;
1460         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1461         int work_done = 0;
1462
1463         if (!bnxt_lock_napi(bnapi))
1464                 return budget;
1465
1466         while (1) {
1467                 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
1468
1469                 if (work_done >= budget)
1470                         break;
1471
1472                 if (!bnxt_has_work(bp, cpr)) {
1473                         napi_complete(napi);
1474                         BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1475                         break;
1476                 }
1477         }
1478         mmiowb();
1479         bnxt_unlock_napi(bnapi);
1480         return work_done;
1481 }
1482
1483 #ifdef CONFIG_NET_RX_BUSY_POLL
1484 static int bnxt_busy_poll(struct napi_struct *napi)
1485 {
1486         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1487         struct bnxt *bp = bnapi->bp;
1488         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1489         int rx_work, budget = 4;
1490
1491         if (atomic_read(&bp->intr_sem) != 0)
1492                 return LL_FLUSH_FAILED;
1493
1494         if (!bnxt_lock_poll(bnapi))
1495                 return LL_FLUSH_BUSY;
1496
1497         rx_work = bnxt_poll_work(bp, bnapi, budget);
1498
1499         BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1500
1501         bnxt_unlock_poll(bnapi);
1502         return rx_work;
1503 }
1504 #endif
1505
1506 static void bnxt_free_tx_skbs(struct bnxt *bp)
1507 {
1508         int i, max_idx;
1509         struct pci_dev *pdev = bp->pdev;
1510
1511         if (!bp->tx_ring)
1512                 return;
1513
1514         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
1515         for (i = 0; i < bp->tx_nr_rings; i++) {
1516                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1517                 int j;
1518
1519                 for (j = 0; j < max_idx;) {
1520                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
1521                         struct sk_buff *skb = tx_buf->skb;
1522                         int k, last;
1523
1524                         if (!skb) {
1525                                 j++;
1526                                 continue;
1527                         }
1528
1529                         tx_buf->skb = NULL;
1530
1531                         if (tx_buf->is_push) {
1532                                 dev_kfree_skb(skb);
1533                                 j += 2;
1534                                 continue;
1535                         }
1536
1537                         dma_unmap_single(&pdev->dev,
1538                                          dma_unmap_addr(tx_buf, mapping),
1539                                          skb_headlen(skb),
1540                                          PCI_DMA_TODEVICE);
1541
1542                         last = tx_buf->nr_frags;
1543                         j += 2;
1544                         for (k = 0; k < last; k++, j++) {
1545                                 int ring_idx = j & bp->tx_ring_mask;
1546                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
1547
1548                                 tx_buf = &txr->tx_buf_ring[ring_idx];
1549                                 dma_unmap_page(
1550                                         &pdev->dev,
1551                                         dma_unmap_addr(tx_buf, mapping),
1552                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
1553                         }
1554                         dev_kfree_skb(skb);
1555                 }
1556                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
1557         }
1558 }
1559
1560 static void bnxt_free_rx_skbs(struct bnxt *bp)
1561 {
1562         int i, max_idx, max_agg_idx;
1563         struct pci_dev *pdev = bp->pdev;
1564
1565         if (!bp->rx_ring)
1566                 return;
1567
1568         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
1569         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
1570         for (i = 0; i < bp->rx_nr_rings; i++) {
1571                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1572                 int j;
1573
1574                 if (rxr->rx_tpa) {
1575                         for (j = 0; j < MAX_TPA; j++) {
1576                                 struct bnxt_tpa_info *tpa_info =
1577                                                         &rxr->rx_tpa[j];
1578                                 u8 *data = tpa_info->data;
1579
1580                                 if (!data)
1581                                         continue;
1582
1583                                 dma_unmap_single(
1584                                         &pdev->dev,
1585                                         dma_unmap_addr(tpa_info, mapping),
1586                                         bp->rx_buf_use_size,
1587                                         PCI_DMA_FROMDEVICE);
1588
1589                                 tpa_info->data = NULL;
1590
1591                                 kfree(data);
1592                         }
1593                 }
1594
1595                 for (j = 0; j < max_idx; j++) {
1596                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
1597                         u8 *data = rx_buf->data;
1598
1599                         if (!data)
1600                                 continue;
1601
1602                         dma_unmap_single(&pdev->dev,
1603                                          dma_unmap_addr(rx_buf, mapping),
1604                                          bp->rx_buf_use_size,
1605                                          PCI_DMA_FROMDEVICE);
1606
1607                         rx_buf->data = NULL;
1608
1609                         kfree(data);
1610                 }
1611
1612                 for (j = 0; j < max_agg_idx; j++) {
1613                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
1614                                 &rxr->rx_agg_ring[j];
1615                         struct page *page = rx_agg_buf->page;
1616
1617                         if (!page)
1618                                 continue;
1619
1620                         dma_unmap_page(&pdev->dev,
1621                                        dma_unmap_addr(rx_agg_buf, mapping),
1622                                        PAGE_SIZE, PCI_DMA_FROMDEVICE);
1623
1624                         rx_agg_buf->page = NULL;
1625                         __clear_bit(j, rxr->rx_agg_bmap);
1626
1627                         __free_page(page);
1628                 }
1629         }
1630 }
1631
1632 static void bnxt_free_skbs(struct bnxt *bp)
1633 {
1634         bnxt_free_tx_skbs(bp);
1635         bnxt_free_rx_skbs(bp);
1636 }
1637
1638 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1639 {
1640         struct pci_dev *pdev = bp->pdev;
1641         int i;
1642
1643         for (i = 0; i < ring->nr_pages; i++) {
1644                 if (!ring->pg_arr[i])
1645                         continue;
1646
1647                 dma_free_coherent(&pdev->dev, ring->page_size,
1648                                   ring->pg_arr[i], ring->dma_arr[i]);
1649
1650                 ring->pg_arr[i] = NULL;
1651         }
1652         if (ring->pg_tbl) {
1653                 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
1654                                   ring->pg_tbl, ring->pg_tbl_map);
1655                 ring->pg_tbl = NULL;
1656         }
1657         if (ring->vmem_size && *ring->vmem) {
1658                 vfree(*ring->vmem);
1659                 *ring->vmem = NULL;
1660         }
1661 }
1662
1663 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1664 {
1665         int i;
1666         struct pci_dev *pdev = bp->pdev;
1667
1668         if (ring->nr_pages > 1) {
1669                 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
1670                                                   ring->nr_pages * 8,
1671                                                   &ring->pg_tbl_map,
1672                                                   GFP_KERNEL);
1673                 if (!ring->pg_tbl)
1674                         return -ENOMEM;
1675         }
1676
1677         for (i = 0; i < ring->nr_pages; i++) {
1678                 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
1679                                                      ring->page_size,
1680                                                      &ring->dma_arr[i],
1681                                                      GFP_KERNEL);
1682                 if (!ring->pg_arr[i])
1683                         return -ENOMEM;
1684
1685                 if (ring->nr_pages > 1)
1686                         ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
1687         }
1688
1689         if (ring->vmem_size) {
1690                 *ring->vmem = vzalloc(ring->vmem_size);
1691                 if (!(*ring->vmem))
1692                         return -ENOMEM;
1693         }
1694         return 0;
1695 }
1696
1697 static void bnxt_free_rx_rings(struct bnxt *bp)
1698 {
1699         int i;
1700
1701         if (!bp->rx_ring)
1702                 return;
1703
1704         for (i = 0; i < bp->rx_nr_rings; i++) {
1705                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1706                 struct bnxt_ring_struct *ring;
1707
1708                 kfree(rxr->rx_tpa);
1709                 rxr->rx_tpa = NULL;
1710
1711                 kfree(rxr->rx_agg_bmap);
1712                 rxr->rx_agg_bmap = NULL;
1713
1714                 ring = &rxr->rx_ring_struct;
1715                 bnxt_free_ring(bp, ring);
1716
1717                 ring = &rxr->rx_agg_ring_struct;
1718                 bnxt_free_ring(bp, ring);
1719         }
1720 }
1721
1722 static int bnxt_alloc_rx_rings(struct bnxt *bp)
1723 {
1724         int i, rc, agg_rings = 0, tpa_rings = 0;
1725
1726         if (!bp->rx_ring)
1727                 return -ENOMEM;
1728
1729         if (bp->flags & BNXT_FLAG_AGG_RINGS)
1730                 agg_rings = 1;
1731
1732         if (bp->flags & BNXT_FLAG_TPA)
1733                 tpa_rings = 1;
1734
1735         for (i = 0; i < bp->rx_nr_rings; i++) {
1736                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1737                 struct bnxt_ring_struct *ring;
1738
1739                 ring = &rxr->rx_ring_struct;
1740
1741                 rc = bnxt_alloc_ring(bp, ring);
1742                 if (rc)
1743                         return rc;
1744
1745                 if (agg_rings) {
1746                         u16 mem_size;
1747
1748                         ring = &rxr->rx_agg_ring_struct;
1749                         rc = bnxt_alloc_ring(bp, ring);
1750                         if (rc)
1751                                 return rc;
1752
1753                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
1754                         mem_size = rxr->rx_agg_bmap_size / 8;
1755                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
1756                         if (!rxr->rx_agg_bmap)
1757                                 return -ENOMEM;
1758
1759                         if (tpa_rings) {
1760                                 rxr->rx_tpa = kcalloc(MAX_TPA,
1761                                                 sizeof(struct bnxt_tpa_info),
1762                                                 GFP_KERNEL);
1763                                 if (!rxr->rx_tpa)
1764                                         return -ENOMEM;
1765                         }
1766                 }
1767         }
1768         return 0;
1769 }
1770
1771 static void bnxt_free_tx_rings(struct bnxt *bp)
1772 {
1773         int i;
1774         struct pci_dev *pdev = bp->pdev;
1775
1776         if (!bp->tx_ring)
1777                 return;
1778
1779         for (i = 0; i < bp->tx_nr_rings; i++) {
1780                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1781                 struct bnxt_ring_struct *ring;
1782
1783                 if (txr->tx_push) {
1784                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
1785                                           txr->tx_push, txr->tx_push_mapping);
1786                         txr->tx_push = NULL;
1787                 }
1788
1789                 ring = &txr->tx_ring_struct;
1790
1791                 bnxt_free_ring(bp, ring);
1792         }
1793 }
1794
1795 static int bnxt_alloc_tx_rings(struct bnxt *bp)
1796 {
1797         int i, j, rc;
1798         struct pci_dev *pdev = bp->pdev;
1799
1800         bp->tx_push_size = 0;
1801         if (bp->tx_push_thresh) {
1802                 int push_size;
1803
1804                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
1805                                         bp->tx_push_thresh);
1806
1807                 if (push_size > 256) {
1808                         push_size = 0;
1809                         bp->tx_push_thresh = 0;
1810                 }
1811
1812                 bp->tx_push_size = push_size;
1813         }
1814
1815         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
1816                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1817                 struct bnxt_ring_struct *ring;
1818
1819                 ring = &txr->tx_ring_struct;
1820
1821                 rc = bnxt_alloc_ring(bp, ring);
1822                 if (rc)
1823                         return rc;
1824
1825                 if (bp->tx_push_size) {
1826                         dma_addr_t mapping;
1827
1828                         /* One pre-allocated DMA buffer to backup
1829                          * TX push operation
1830                          */
1831                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
1832                                                 bp->tx_push_size,
1833                                                 &txr->tx_push_mapping,
1834                                                 GFP_KERNEL);
1835
1836                         if (!txr->tx_push)
1837                                 return -ENOMEM;
1838
1839                         mapping = txr->tx_push_mapping +
1840                                 sizeof(struct tx_push_bd);
1841                         txr->data_mapping = cpu_to_le64(mapping);
1842
1843                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
1844                 }
1845                 ring->queue_id = bp->q_info[j].queue_id;
1846                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
1847                         j++;
1848         }
1849         return 0;
1850 }
1851
1852 static void bnxt_free_cp_rings(struct bnxt *bp)
1853 {
1854         int i;
1855
1856         if (!bp->bnapi)
1857                 return;
1858
1859         for (i = 0; i < bp->cp_nr_rings; i++) {
1860                 struct bnxt_napi *bnapi = bp->bnapi[i];
1861                 struct bnxt_cp_ring_info *cpr;
1862                 struct bnxt_ring_struct *ring;
1863
1864                 if (!bnapi)
1865                         continue;
1866
1867                 cpr = &bnapi->cp_ring;
1868                 ring = &cpr->cp_ring_struct;
1869
1870                 bnxt_free_ring(bp, ring);
1871         }
1872 }
1873
1874 static int bnxt_alloc_cp_rings(struct bnxt *bp)
1875 {
1876         int i, rc;
1877
1878         for (i = 0; i < bp->cp_nr_rings; i++) {
1879                 struct bnxt_napi *bnapi = bp->bnapi[i];
1880                 struct bnxt_cp_ring_info *cpr;
1881                 struct bnxt_ring_struct *ring;
1882
1883                 if (!bnapi)
1884                         continue;
1885
1886                 cpr = &bnapi->cp_ring;
1887                 ring = &cpr->cp_ring_struct;
1888
1889                 rc = bnxt_alloc_ring(bp, ring);
1890                 if (rc)
1891                         return rc;
1892         }
1893         return 0;
1894 }
1895
1896 static void bnxt_init_ring_struct(struct bnxt *bp)
1897 {
1898         int i;
1899
1900         for (i = 0; i < bp->cp_nr_rings; i++) {
1901                 struct bnxt_napi *bnapi = bp->bnapi[i];
1902                 struct bnxt_cp_ring_info *cpr;
1903                 struct bnxt_rx_ring_info *rxr;
1904                 struct bnxt_tx_ring_info *txr;
1905                 struct bnxt_ring_struct *ring;
1906
1907                 if (!bnapi)
1908                         continue;
1909
1910                 cpr = &bnapi->cp_ring;
1911                 ring = &cpr->cp_ring_struct;
1912                 ring->nr_pages = bp->cp_nr_pages;
1913                 ring->page_size = HW_CMPD_RING_SIZE;
1914                 ring->pg_arr = (void **)cpr->cp_desc_ring;
1915                 ring->dma_arr = cpr->cp_desc_mapping;
1916                 ring->vmem_size = 0;
1917
1918                 rxr = bnapi->rx_ring;
1919                 if (!rxr)
1920                         goto skip_rx;
1921
1922                 ring = &rxr->rx_ring_struct;
1923                 ring->nr_pages = bp->rx_nr_pages;
1924                 ring->page_size = HW_RXBD_RING_SIZE;
1925                 ring->pg_arr = (void **)rxr->rx_desc_ring;
1926                 ring->dma_arr = rxr->rx_desc_mapping;
1927                 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
1928                 ring->vmem = (void **)&rxr->rx_buf_ring;
1929
1930                 ring = &rxr->rx_agg_ring_struct;
1931                 ring->nr_pages = bp->rx_agg_nr_pages;
1932                 ring->page_size = HW_RXBD_RING_SIZE;
1933                 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
1934                 ring->dma_arr = rxr->rx_agg_desc_mapping;
1935                 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
1936                 ring->vmem = (void **)&rxr->rx_agg_ring;
1937
1938 skip_rx:
1939                 txr = bnapi->tx_ring;
1940                 if (!txr)
1941                         continue;
1942
1943                 ring = &txr->tx_ring_struct;
1944                 ring->nr_pages = bp->tx_nr_pages;
1945                 ring->page_size = HW_RXBD_RING_SIZE;
1946                 ring->pg_arr = (void **)txr->tx_desc_ring;
1947                 ring->dma_arr = txr->tx_desc_mapping;
1948                 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
1949                 ring->vmem = (void **)&txr->tx_buf_ring;
1950         }
1951 }
1952
1953 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
1954 {
1955         int i;
1956         u32 prod;
1957         struct rx_bd **rx_buf_ring;
1958
1959         rx_buf_ring = (struct rx_bd **)ring->pg_arr;
1960         for (i = 0, prod = 0; i < ring->nr_pages; i++) {
1961                 int j;
1962                 struct rx_bd *rxbd;
1963
1964                 rxbd = rx_buf_ring[i];
1965                 if (!rxbd)
1966                         continue;
1967
1968                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
1969                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
1970                         rxbd->rx_bd_opaque = prod;
1971                 }
1972         }
1973 }
1974
1975 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
1976 {
1977         struct net_device *dev = bp->dev;
1978         struct bnxt_rx_ring_info *rxr;
1979         struct bnxt_ring_struct *ring;
1980         u32 prod, type;
1981         int i;
1982
1983         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
1984                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
1985
1986         if (NET_IP_ALIGN == 2)
1987                 type |= RX_BD_FLAGS_SOP;
1988
1989         rxr = &bp->rx_ring[ring_nr];
1990         ring = &rxr->rx_ring_struct;
1991         bnxt_init_rxbd_pages(ring, type);
1992
1993         prod = rxr->rx_prod;
1994         for (i = 0; i < bp->rx_ring_size; i++) {
1995                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
1996                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
1997                                     ring_nr, i, bp->rx_ring_size);
1998                         break;
1999                 }
2000                 prod = NEXT_RX(prod);
2001         }
2002         rxr->rx_prod = prod;
2003         ring->fw_ring_id = INVALID_HW_RING_ID;
2004
2005         ring = &rxr->rx_agg_ring_struct;
2006         ring->fw_ring_id = INVALID_HW_RING_ID;
2007
2008         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2009                 return 0;
2010
2011         type = ((u32)PAGE_SIZE << RX_BD_LEN_SHIFT) |
2012                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2013
2014         bnxt_init_rxbd_pages(ring, type);
2015
2016         prod = rxr->rx_agg_prod;
2017         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2018                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2019                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2020                                     ring_nr, i, bp->rx_ring_size);
2021                         break;
2022                 }
2023                 prod = NEXT_RX_AGG(prod);
2024         }
2025         rxr->rx_agg_prod = prod;
2026
2027         if (bp->flags & BNXT_FLAG_TPA) {
2028                 if (rxr->rx_tpa) {
2029                         u8 *data;
2030                         dma_addr_t mapping;
2031
2032                         for (i = 0; i < MAX_TPA; i++) {
2033                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2034                                                             GFP_KERNEL);
2035                                 if (!data)
2036                                         return -ENOMEM;
2037
2038                                 rxr->rx_tpa[i].data = data;
2039                                 rxr->rx_tpa[i].mapping = mapping;
2040                         }
2041                 } else {
2042                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2043                         return -ENOMEM;
2044                 }
2045         }
2046
2047         return 0;
2048 }
2049
2050 static int bnxt_init_rx_rings(struct bnxt *bp)
2051 {
2052         int i, rc = 0;
2053
2054         for (i = 0; i < bp->rx_nr_rings; i++) {
2055                 rc = bnxt_init_one_rx_ring(bp, i);
2056                 if (rc)
2057                         break;
2058         }
2059
2060         return rc;
2061 }
2062
2063 static int bnxt_init_tx_rings(struct bnxt *bp)
2064 {
2065         u16 i;
2066
2067         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2068                                    MAX_SKB_FRAGS + 1);
2069
2070         for (i = 0; i < bp->tx_nr_rings; i++) {
2071                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2072                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2073
2074                 ring->fw_ring_id = INVALID_HW_RING_ID;
2075         }
2076
2077         return 0;
2078 }
2079
2080 static void bnxt_free_ring_grps(struct bnxt *bp)
2081 {
2082         kfree(bp->grp_info);
2083         bp->grp_info = NULL;
2084 }
2085
2086 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2087 {
2088         int i;
2089
2090         if (irq_re_init) {
2091                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2092                                        sizeof(struct bnxt_ring_grp_info),
2093                                        GFP_KERNEL);
2094                 if (!bp->grp_info)
2095                         return -ENOMEM;
2096         }
2097         for (i = 0; i < bp->cp_nr_rings; i++) {
2098                 if (irq_re_init)
2099                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2100                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2101                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2102                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2103                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2104         }
2105         return 0;
2106 }
2107
2108 static void bnxt_free_vnics(struct bnxt *bp)
2109 {
2110         kfree(bp->vnic_info);
2111         bp->vnic_info = NULL;
2112         bp->nr_vnics = 0;
2113 }
2114
2115 static int bnxt_alloc_vnics(struct bnxt *bp)
2116 {
2117         int num_vnics = 1;
2118
2119 #ifdef CONFIG_RFS_ACCEL
2120         if (bp->flags & BNXT_FLAG_RFS)
2121                 num_vnics += bp->rx_nr_rings;
2122 #endif
2123
2124         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2125                                 GFP_KERNEL);
2126         if (!bp->vnic_info)
2127                 return -ENOMEM;
2128
2129         bp->nr_vnics = num_vnics;
2130         return 0;
2131 }
2132
2133 static void bnxt_init_vnics(struct bnxt *bp)
2134 {
2135         int i;
2136
2137         for (i = 0; i < bp->nr_vnics; i++) {
2138                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2139
2140                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2141                 vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
2142                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2143
2144                 if (bp->vnic_info[i].rss_hash_key) {
2145                         if (i == 0)
2146                                 prandom_bytes(vnic->rss_hash_key,
2147                                               HW_HASH_KEY_SIZE);
2148                         else
2149                                 memcpy(vnic->rss_hash_key,
2150                                        bp->vnic_info[0].rss_hash_key,
2151                                        HW_HASH_KEY_SIZE);
2152                 }
2153         }
2154 }
2155
2156 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2157 {
2158         int pages;
2159
2160         pages = ring_size / desc_per_pg;
2161
2162         if (!pages)
2163                 return 1;
2164
2165         pages++;
2166
2167         while (pages & (pages - 1))
2168                 pages++;
2169
2170         return pages;
2171 }
2172
2173 static void bnxt_set_tpa_flags(struct bnxt *bp)
2174 {
2175         bp->flags &= ~BNXT_FLAG_TPA;
2176         if (bp->dev->features & NETIF_F_LRO)
2177                 bp->flags |= BNXT_FLAG_LRO;
2178         if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
2179                 bp->flags |= BNXT_FLAG_GRO;
2180 }
2181
2182 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2183  * be set on entry.
2184  */
2185 void bnxt_set_ring_params(struct bnxt *bp)
2186 {
2187         u32 ring_size, rx_size, rx_space;
2188         u32 agg_factor = 0, agg_ring_size = 0;
2189
2190         /* 8 for CRC and VLAN */
2191         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2192
2193         rx_space = rx_size + NET_SKB_PAD +
2194                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2195
2196         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2197         ring_size = bp->rx_ring_size;
2198         bp->rx_agg_ring_size = 0;
2199         bp->rx_agg_nr_pages = 0;
2200
2201         if (bp->flags & BNXT_FLAG_TPA)
2202                 agg_factor = 4;
2203
2204         bp->flags &= ~BNXT_FLAG_JUMBO;
2205         if (rx_space > PAGE_SIZE) {
2206                 u32 jumbo_factor;
2207
2208                 bp->flags |= BNXT_FLAG_JUMBO;
2209                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2210                 if (jumbo_factor > agg_factor)
2211                         agg_factor = jumbo_factor;
2212         }
2213         agg_ring_size = ring_size * agg_factor;
2214
2215         if (agg_ring_size) {
2216                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2217                                                         RX_DESC_CNT);
2218                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2219                         u32 tmp = agg_ring_size;
2220
2221                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2222                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2223                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2224                                     tmp, agg_ring_size);
2225                 }
2226                 bp->rx_agg_ring_size = agg_ring_size;
2227                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2228                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2229                 rx_space = rx_size + NET_SKB_PAD +
2230                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2231         }
2232
2233         bp->rx_buf_use_size = rx_size;
2234         bp->rx_buf_size = rx_space;
2235
2236         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2237         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2238
2239         ring_size = bp->tx_ring_size;
2240         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2241         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2242
2243         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2244         bp->cp_ring_size = ring_size;
2245
2246         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2247         if (bp->cp_nr_pages > MAX_CP_PAGES) {
2248                 bp->cp_nr_pages = MAX_CP_PAGES;
2249                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2250                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2251                             ring_size, bp->cp_ring_size);
2252         }
2253         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2254         bp->cp_ring_mask = bp->cp_bit - 1;
2255 }
2256
2257 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2258 {
2259         int i;
2260         struct bnxt_vnic_info *vnic;
2261         struct pci_dev *pdev = bp->pdev;
2262
2263         if (!bp->vnic_info)
2264                 return;
2265
2266         for (i = 0; i < bp->nr_vnics; i++) {
2267                 vnic = &bp->vnic_info[i];
2268
2269                 kfree(vnic->fw_grp_ids);
2270                 vnic->fw_grp_ids = NULL;
2271
2272                 kfree(vnic->uc_list);
2273                 vnic->uc_list = NULL;
2274
2275                 if (vnic->mc_list) {
2276                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2277                                           vnic->mc_list, vnic->mc_list_mapping);
2278                         vnic->mc_list = NULL;
2279                 }
2280
2281                 if (vnic->rss_table) {
2282                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
2283                                           vnic->rss_table,
2284                                           vnic->rss_table_dma_addr);
2285                         vnic->rss_table = NULL;
2286                 }
2287
2288                 vnic->rss_hash_key = NULL;
2289                 vnic->flags = 0;
2290         }
2291 }
2292
2293 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2294 {
2295         int i, rc = 0, size;
2296         struct bnxt_vnic_info *vnic;
2297         struct pci_dev *pdev = bp->pdev;
2298         int max_rings;
2299
2300         for (i = 0; i < bp->nr_vnics; i++) {
2301                 vnic = &bp->vnic_info[i];
2302
2303                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2304                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2305
2306                         if (mem_size > 0) {
2307                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2308                                 if (!vnic->uc_list) {
2309                                         rc = -ENOMEM;
2310                                         goto out;
2311                                 }
2312                         }
2313                 }
2314
2315                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2316                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2317                         vnic->mc_list =
2318                                 dma_alloc_coherent(&pdev->dev,
2319                                                    vnic->mc_list_size,
2320                                                    &vnic->mc_list_mapping,
2321                                                    GFP_KERNEL);
2322                         if (!vnic->mc_list) {
2323                                 rc = -ENOMEM;
2324                                 goto out;
2325                         }
2326                 }
2327
2328                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2329                         max_rings = bp->rx_nr_rings;
2330                 else
2331                         max_rings = 1;
2332
2333                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2334                 if (!vnic->fw_grp_ids) {
2335                         rc = -ENOMEM;
2336                         goto out;
2337                 }
2338
2339                 /* Allocate rss table and hash key */
2340                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2341                                                      &vnic->rss_table_dma_addr,
2342                                                      GFP_KERNEL);
2343                 if (!vnic->rss_table) {
2344                         rc = -ENOMEM;
2345                         goto out;
2346                 }
2347
2348                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2349
2350                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2351                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2352         }
2353         return 0;
2354
2355 out:
2356         return rc;
2357 }
2358
2359 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2360 {
2361         struct pci_dev *pdev = bp->pdev;
2362
2363         dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2364                           bp->hwrm_cmd_resp_dma_addr);
2365
2366         bp->hwrm_cmd_resp_addr = NULL;
2367         if (bp->hwrm_dbg_resp_addr) {
2368                 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2369                                   bp->hwrm_dbg_resp_addr,
2370                                   bp->hwrm_dbg_resp_dma_addr);
2371
2372                 bp->hwrm_dbg_resp_addr = NULL;
2373         }
2374 }
2375
2376 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2377 {
2378         struct pci_dev *pdev = bp->pdev;
2379
2380         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2381                                                    &bp->hwrm_cmd_resp_dma_addr,
2382                                                    GFP_KERNEL);
2383         if (!bp->hwrm_cmd_resp_addr)
2384                 return -ENOMEM;
2385         bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2386                                                     HWRM_DBG_REG_BUF_SIZE,
2387                                                     &bp->hwrm_dbg_resp_dma_addr,
2388                                                     GFP_KERNEL);
2389         if (!bp->hwrm_dbg_resp_addr)
2390                 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2391
2392         return 0;
2393 }
2394
2395 static void bnxt_free_stats(struct bnxt *bp)
2396 {
2397         u32 size, i;
2398         struct pci_dev *pdev = bp->pdev;
2399
2400         if (bp->hw_rx_port_stats) {
2401                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
2402                                   bp->hw_rx_port_stats,
2403                                   bp->hw_rx_port_stats_map);
2404                 bp->hw_rx_port_stats = NULL;
2405                 bp->flags &= ~BNXT_FLAG_PORT_STATS;
2406         }
2407
2408         if (!bp->bnapi)
2409                 return;
2410
2411         size = sizeof(struct ctx_hw_stats);
2412
2413         for (i = 0; i < bp->cp_nr_rings; i++) {
2414                 struct bnxt_napi *bnapi = bp->bnapi[i];
2415                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2416
2417                 if (cpr->hw_stats) {
2418                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
2419                                           cpr->hw_stats_map);
2420                         cpr->hw_stats = NULL;
2421                 }
2422         }
2423 }
2424
2425 static int bnxt_alloc_stats(struct bnxt *bp)
2426 {
2427         u32 size, i;
2428         struct pci_dev *pdev = bp->pdev;
2429
2430         size = sizeof(struct ctx_hw_stats);
2431
2432         for (i = 0; i < bp->cp_nr_rings; i++) {
2433                 struct bnxt_napi *bnapi = bp->bnapi[i];
2434                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2435
2436                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
2437                                                    &cpr->hw_stats_map,
2438                                                    GFP_KERNEL);
2439                 if (!cpr->hw_stats)
2440                         return -ENOMEM;
2441
2442                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
2443         }
2444
2445         if (BNXT_PF(bp)) {
2446                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
2447                                          sizeof(struct tx_port_stats) + 1024;
2448
2449                 bp->hw_rx_port_stats =
2450                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
2451                                            &bp->hw_rx_port_stats_map,
2452                                            GFP_KERNEL);
2453                 if (!bp->hw_rx_port_stats)
2454                         return -ENOMEM;
2455
2456                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
2457                                        512;
2458                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
2459                                            sizeof(struct rx_port_stats) + 512;
2460                 bp->flags |= BNXT_FLAG_PORT_STATS;
2461         }
2462         return 0;
2463 }
2464
2465 static void bnxt_clear_ring_indices(struct bnxt *bp)
2466 {
2467         int i;
2468
2469         if (!bp->bnapi)
2470                 return;
2471
2472         for (i = 0; i < bp->cp_nr_rings; i++) {
2473                 struct bnxt_napi *bnapi = bp->bnapi[i];
2474                 struct bnxt_cp_ring_info *cpr;
2475                 struct bnxt_rx_ring_info *rxr;
2476                 struct bnxt_tx_ring_info *txr;
2477
2478                 if (!bnapi)
2479                         continue;
2480
2481                 cpr = &bnapi->cp_ring;
2482                 cpr->cp_raw_cons = 0;
2483
2484                 txr = bnapi->tx_ring;
2485                 if (txr) {
2486                         txr->tx_prod = 0;
2487                         txr->tx_cons = 0;
2488                 }
2489
2490                 rxr = bnapi->rx_ring;
2491                 if (rxr) {
2492                         rxr->rx_prod = 0;
2493                         rxr->rx_agg_prod = 0;
2494                         rxr->rx_sw_agg_prod = 0;
2495                 }
2496         }
2497 }
2498
2499 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
2500 {
2501 #ifdef CONFIG_RFS_ACCEL
2502         int i;
2503
2504         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
2505          * safe to delete the hash table.
2506          */
2507         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
2508                 struct hlist_head *head;
2509                 struct hlist_node *tmp;
2510                 struct bnxt_ntuple_filter *fltr;
2511
2512                 head = &bp->ntp_fltr_hash_tbl[i];
2513                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
2514                         hlist_del(&fltr->hash);
2515                         kfree(fltr);
2516                 }
2517         }
2518         if (irq_reinit) {
2519                 kfree(bp->ntp_fltr_bmap);
2520                 bp->ntp_fltr_bmap = NULL;
2521         }
2522         bp->ntp_fltr_count = 0;
2523 #endif
2524 }
2525
2526 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
2527 {
2528 #ifdef CONFIG_RFS_ACCEL
2529         int i, rc = 0;
2530
2531         if (!(bp->flags & BNXT_FLAG_RFS))
2532                 return 0;
2533
2534         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
2535                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
2536
2537         bp->ntp_fltr_count = 0;
2538         bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
2539                                     GFP_KERNEL);
2540
2541         if (!bp->ntp_fltr_bmap)
2542                 rc = -ENOMEM;
2543
2544         return rc;
2545 #else
2546         return 0;
2547 #endif
2548 }
2549
2550 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
2551 {
2552         bnxt_free_vnic_attributes(bp);
2553         bnxt_free_tx_rings(bp);
2554         bnxt_free_rx_rings(bp);
2555         bnxt_free_cp_rings(bp);
2556         bnxt_free_ntp_fltrs(bp, irq_re_init);
2557         if (irq_re_init) {
2558                 bnxt_free_stats(bp);
2559                 bnxt_free_ring_grps(bp);
2560                 bnxt_free_vnics(bp);
2561                 kfree(bp->tx_ring);
2562                 bp->tx_ring = NULL;
2563                 kfree(bp->rx_ring);
2564                 bp->rx_ring = NULL;
2565                 kfree(bp->bnapi);
2566                 bp->bnapi = NULL;
2567         } else {
2568                 bnxt_clear_ring_indices(bp);
2569         }
2570 }
2571
2572 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
2573 {
2574         int i, j, rc, size, arr_size;
2575         void *bnapi;
2576
2577         if (irq_re_init) {
2578                 /* Allocate bnapi mem pointer array and mem block for
2579                  * all queues
2580                  */
2581                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
2582                                 bp->cp_nr_rings);
2583                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
2584                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
2585                 if (!bnapi)
2586                         return -ENOMEM;
2587
2588                 bp->bnapi = bnapi;
2589                 bnapi += arr_size;
2590                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
2591                         bp->bnapi[i] = bnapi;
2592                         bp->bnapi[i]->index = i;
2593                         bp->bnapi[i]->bp = bp;
2594                 }
2595
2596                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
2597                                       sizeof(struct bnxt_rx_ring_info),
2598                                       GFP_KERNEL);
2599                 if (!bp->rx_ring)
2600                         return -ENOMEM;
2601
2602                 for (i = 0; i < bp->rx_nr_rings; i++) {
2603                         bp->rx_ring[i].bnapi = bp->bnapi[i];
2604                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
2605                 }
2606
2607                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
2608                                       sizeof(struct bnxt_tx_ring_info),
2609                                       GFP_KERNEL);
2610                 if (!bp->tx_ring)
2611                         return -ENOMEM;
2612
2613                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
2614                         j = 0;
2615                 else
2616                         j = bp->rx_nr_rings;
2617
2618                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
2619                         bp->tx_ring[i].bnapi = bp->bnapi[j];
2620                         bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
2621                 }
2622
2623                 rc = bnxt_alloc_stats(bp);
2624                 if (rc)
2625                         goto alloc_mem_err;
2626
2627                 rc = bnxt_alloc_ntp_fltrs(bp);
2628                 if (rc)
2629                         goto alloc_mem_err;
2630
2631                 rc = bnxt_alloc_vnics(bp);
2632                 if (rc)
2633                         goto alloc_mem_err;
2634         }
2635
2636         bnxt_init_ring_struct(bp);
2637
2638         rc = bnxt_alloc_rx_rings(bp);
2639         if (rc)
2640                 goto alloc_mem_err;
2641
2642         rc = bnxt_alloc_tx_rings(bp);
2643         if (rc)
2644                 goto alloc_mem_err;
2645
2646         rc = bnxt_alloc_cp_rings(bp);
2647         if (rc)
2648                 goto alloc_mem_err;
2649
2650         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
2651                                   BNXT_VNIC_UCAST_FLAG;
2652         rc = bnxt_alloc_vnic_attributes(bp);
2653         if (rc)
2654                 goto alloc_mem_err;
2655         return 0;
2656
2657 alloc_mem_err:
2658         bnxt_free_mem(bp, true);
2659         return rc;
2660 }
2661
2662 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
2663                             u16 cmpl_ring, u16 target_id)
2664 {
2665         struct input *req = request;
2666
2667         req->req_type = cpu_to_le16(req_type);
2668         req->cmpl_ring = cpu_to_le16(cmpl_ring);
2669         req->target_id = cpu_to_le16(target_id);
2670         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
2671 }
2672
2673 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
2674                                  int timeout, bool silent)
2675 {
2676         int i, intr_process, rc;
2677         struct input *req = msg;
2678         u32 *data = msg;
2679         __le32 *resp_len, *valid;
2680         u16 cp_ring_id, len = 0;
2681         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
2682
2683         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
2684         memset(resp, 0, PAGE_SIZE);
2685         cp_ring_id = le16_to_cpu(req->cmpl_ring);
2686         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
2687
2688         /* Write request msg to hwrm channel */
2689         __iowrite32_copy(bp->bar0, data, msg_len / 4);
2690
2691         for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4)
2692                 writel(0, bp->bar0 + i);
2693
2694         /* currently supports only one outstanding message */
2695         if (intr_process)
2696                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
2697
2698         /* Ring channel doorbell */
2699         writel(1, bp->bar0 + 0x100);
2700
2701         if (!timeout)
2702                 timeout = DFLT_HWRM_CMD_TIMEOUT;
2703
2704         i = 0;
2705         if (intr_process) {
2706                 /* Wait until hwrm response cmpl interrupt is processed */
2707                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
2708                        i++ < timeout) {
2709                         usleep_range(600, 800);
2710                 }
2711
2712                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
2713                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
2714                                    le16_to_cpu(req->req_type));
2715                         return -1;
2716                 }
2717         } else {
2718                 /* Check if response len is updated */
2719                 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
2720                 for (i = 0; i < timeout; i++) {
2721                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
2722                               HWRM_RESP_LEN_SFT;
2723                         if (len)
2724                                 break;
2725                         usleep_range(600, 800);
2726                 }
2727
2728                 if (i >= timeout) {
2729                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
2730                                    timeout, le16_to_cpu(req->req_type),
2731                                    le16_to_cpu(req->seq_id), *resp_len);
2732                         return -1;
2733                 }
2734
2735                 /* Last word of resp contains valid bit */
2736                 valid = bp->hwrm_cmd_resp_addr + len - 4;
2737                 for (i = 0; i < timeout; i++) {
2738                         if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
2739                                 break;
2740                         usleep_range(600, 800);
2741                 }
2742
2743                 if (i >= timeout) {
2744                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
2745                                    timeout, le16_to_cpu(req->req_type),
2746                                    le16_to_cpu(req->seq_id), len, *valid);
2747                         return -1;
2748                 }
2749         }
2750
2751         rc = le16_to_cpu(resp->error_code);
2752         if (rc && !silent)
2753                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
2754                            le16_to_cpu(resp->req_type),
2755                            le16_to_cpu(resp->seq_id), rc);
2756         return rc;
2757 }
2758
2759 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2760 {
2761         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
2762 }
2763
2764 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2765 {
2766         int rc;
2767
2768         mutex_lock(&bp->hwrm_cmd_lock);
2769         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
2770         mutex_unlock(&bp->hwrm_cmd_lock);
2771         return rc;
2772 }
2773
2774 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
2775                              int timeout)
2776 {
2777         int rc;
2778
2779         mutex_lock(&bp->hwrm_cmd_lock);
2780         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
2781         mutex_unlock(&bp->hwrm_cmd_lock);
2782         return rc;
2783 }
2784
2785 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
2786 {
2787         struct hwrm_func_drv_rgtr_input req = {0};
2788         int i;
2789         DECLARE_BITMAP(async_events_bmap, 256);
2790         u32 *events = (u32 *)async_events_bmap;
2791
2792         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
2793
2794         req.enables =
2795                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
2796                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
2797                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
2798
2799         memset(async_events_bmap, 0, sizeof(async_events_bmap));
2800         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
2801                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
2802
2803         for (i = 0; i < 8; i++)
2804                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
2805
2806         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
2807         req.ver_maj = DRV_VER_MAJ;
2808         req.ver_min = DRV_VER_MIN;
2809         req.ver_upd = DRV_VER_UPD;
2810
2811         if (BNXT_PF(bp)) {
2812                 DECLARE_BITMAP(vf_req_snif_bmap, 256);
2813                 u32 *data = (u32 *)vf_req_snif_bmap;
2814
2815                 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap));
2816                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++)
2817                         __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap);
2818
2819                 for (i = 0; i < 8; i++)
2820                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
2821
2822                 req.enables |=
2823                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
2824         }
2825
2826         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2827 }
2828
2829 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
2830 {
2831         struct hwrm_func_drv_unrgtr_input req = {0};
2832
2833         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
2834         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2835 }
2836
2837 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
2838 {
2839         u32 rc = 0;
2840         struct hwrm_tunnel_dst_port_free_input req = {0};
2841
2842         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
2843         req.tunnel_type = tunnel_type;
2844
2845         switch (tunnel_type) {
2846         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
2847                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
2848                 break;
2849         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
2850                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
2851                 break;
2852         default:
2853                 break;
2854         }
2855
2856         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2857         if (rc)
2858                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
2859                            rc);
2860         return rc;
2861 }
2862
2863 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
2864                                            u8 tunnel_type)
2865 {
2866         u32 rc = 0;
2867         struct hwrm_tunnel_dst_port_alloc_input req = {0};
2868         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2869
2870         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
2871
2872         req.tunnel_type = tunnel_type;
2873         req.tunnel_dst_port_val = port;
2874
2875         mutex_lock(&bp->hwrm_cmd_lock);
2876         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2877         if (rc) {
2878                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
2879                            rc);
2880                 goto err_out;
2881         }
2882
2883         if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN)
2884                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
2885
2886         else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE)
2887                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
2888 err_out:
2889         mutex_unlock(&bp->hwrm_cmd_lock);
2890         return rc;
2891 }
2892
2893 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
2894 {
2895         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
2896         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2897
2898         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
2899         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
2900
2901         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
2902         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
2903         req.mask = cpu_to_le32(vnic->rx_mask);
2904         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2905 }
2906
2907 #ifdef CONFIG_RFS_ACCEL
2908 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
2909                                             struct bnxt_ntuple_filter *fltr)
2910 {
2911         struct hwrm_cfa_ntuple_filter_free_input req = {0};
2912
2913         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
2914         req.ntuple_filter_id = fltr->filter_id;
2915         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2916 }
2917
2918 #define BNXT_NTP_FLTR_FLAGS                                     \
2919         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
2920          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
2921          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
2922          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
2923          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
2924          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
2925          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
2926          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
2927          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
2928          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
2929          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
2930          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
2931          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
2932          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
2933
2934 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
2935                                              struct bnxt_ntuple_filter *fltr)
2936 {
2937         int rc = 0;
2938         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
2939         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
2940                 bp->hwrm_cmd_resp_addr;
2941         struct flow_keys *keys = &fltr->fkeys;
2942         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
2943
2944         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
2945         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[0];
2946
2947         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
2948
2949         req.ethertype = htons(ETH_P_IP);
2950         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
2951         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
2952         req.ip_protocol = keys->basic.ip_proto;
2953
2954         req.src_ipaddr[0] = keys->addrs.v4addrs.src;
2955         req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
2956         req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
2957         req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
2958
2959         req.src_port = keys->ports.src;
2960         req.src_port_mask = cpu_to_be16(0xffff);
2961         req.dst_port = keys->ports.dst;
2962         req.dst_port_mask = cpu_to_be16(0xffff);
2963
2964         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
2965         mutex_lock(&bp->hwrm_cmd_lock);
2966         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2967         if (!rc)
2968                 fltr->filter_id = resp->ntuple_filter_id;
2969         mutex_unlock(&bp->hwrm_cmd_lock);
2970         return rc;
2971 }
2972 #endif
2973
2974 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
2975                                      u8 *mac_addr)
2976 {
2977         u32 rc = 0;
2978         struct hwrm_cfa_l2_filter_alloc_input req = {0};
2979         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2980
2981         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
2982         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX |
2983                                 CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
2984         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
2985         req.enables =
2986                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
2987                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
2988                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
2989         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
2990         req.l2_addr_mask[0] = 0xff;
2991         req.l2_addr_mask[1] = 0xff;
2992         req.l2_addr_mask[2] = 0xff;
2993         req.l2_addr_mask[3] = 0xff;
2994         req.l2_addr_mask[4] = 0xff;
2995         req.l2_addr_mask[5] = 0xff;
2996
2997         mutex_lock(&bp->hwrm_cmd_lock);
2998         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2999         if (!rc)
3000                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3001                                                         resp->l2_filter_id;
3002         mutex_unlock(&bp->hwrm_cmd_lock);
3003         return rc;
3004 }
3005
3006 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3007 {
3008         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3009         int rc = 0;
3010
3011         /* Any associated ntuple filters will also be cleared by firmware. */
3012         mutex_lock(&bp->hwrm_cmd_lock);
3013         for (i = 0; i < num_of_vnics; i++) {
3014                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3015
3016                 for (j = 0; j < vnic->uc_filter_count; j++) {
3017                         struct hwrm_cfa_l2_filter_free_input req = {0};
3018
3019                         bnxt_hwrm_cmd_hdr_init(bp, &req,
3020                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
3021
3022                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
3023
3024                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3025                                                 HWRM_CMD_TIMEOUT);
3026                 }
3027                 vnic->uc_filter_count = 0;
3028         }
3029         mutex_unlock(&bp->hwrm_cmd_lock);
3030
3031         return rc;
3032 }
3033
3034 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3035 {
3036         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3037         struct hwrm_vnic_tpa_cfg_input req = {0};
3038
3039         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3040
3041         if (tpa_flags) {
3042                 u16 mss = bp->dev->mtu - 40;
3043                 u32 nsegs, n, segs = 0, flags;
3044
3045                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3046                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3047                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3048                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3049                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3050                 if (tpa_flags & BNXT_FLAG_GRO)
3051                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3052
3053                 req.flags = cpu_to_le32(flags);
3054
3055                 req.enables =
3056                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3057                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3058                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3059
3060                 /* Number of segs are log2 units, and first packet is not
3061                  * included as part of this units.
3062                  */
3063                 if (mss <= PAGE_SIZE) {
3064                         n = PAGE_SIZE / mss;
3065                         nsegs = (MAX_SKB_FRAGS - 1) * n;
3066                 } else {
3067                         n = mss / PAGE_SIZE;
3068                         if (mss & (PAGE_SIZE - 1))
3069                                 n++;
3070                         nsegs = (MAX_SKB_FRAGS - n) / n;
3071                 }
3072
3073                 segs = ilog2(nsegs);
3074                 req.max_agg_segs = cpu_to_le16(segs);
3075                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3076
3077                 req.min_agg_len = cpu_to_le32(512);
3078         }
3079         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3080
3081         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3082 }
3083
3084 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3085 {
3086         u32 i, j, max_rings;
3087         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3088         struct hwrm_vnic_rss_cfg_input req = {0};
3089
3090         if (vnic->fw_rss_cos_lb_ctx == INVALID_HW_RING_ID)
3091                 return 0;
3092
3093         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3094         if (set_rss) {
3095                 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
3096                                  BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
3097                                  BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
3098                                  BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
3099
3100                 req.hash_type = cpu_to_le32(vnic->hash_type);
3101
3102                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3103                         max_rings = bp->rx_nr_rings;
3104                 else
3105                         max_rings = 1;
3106
3107                 /* Fill the RSS indirection table with ring group ids */
3108                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3109                         if (j == max_rings)
3110                                 j = 0;
3111                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3112                 }
3113
3114                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3115                 req.hash_key_tbl_addr =
3116                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
3117         }
3118         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3119         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3120 }
3121
3122 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3123 {
3124         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3125         struct hwrm_vnic_plcmodes_cfg_input req = {0};
3126
3127         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3128         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3129                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3130                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3131         req.enables =
3132                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3133                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3134         /* thresholds not implemented in firmware yet */
3135         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3136         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3137         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3138         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3139 }
3140
3141 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id)
3142 {
3143         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3144
3145         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3146         req.rss_cos_lb_ctx_id =
3147                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx);
3148
3149         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3150         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3151 }
3152
3153 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3154 {
3155         int i;
3156
3157         for (i = 0; i < bp->nr_vnics; i++) {
3158                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3159
3160                 if (vnic->fw_rss_cos_lb_ctx != INVALID_HW_RING_ID)
3161                         bnxt_hwrm_vnic_ctx_free_one(bp, i);
3162         }
3163         bp->rsscos_nr_ctxs = 0;
3164 }
3165
3166 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id)
3167 {
3168         int rc;
3169         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3170         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3171                                                 bp->hwrm_cmd_resp_addr;
3172
3173         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3174                                -1);
3175
3176         mutex_lock(&bp->hwrm_cmd_lock);
3177         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3178         if (!rc)
3179                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx =
3180                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
3181         mutex_unlock(&bp->hwrm_cmd_lock);
3182
3183         return rc;
3184 }
3185
3186 static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3187 {
3188         unsigned int ring = 0, grp_idx;
3189         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3190         struct hwrm_vnic_cfg_input req = {0};
3191
3192         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3193         /* Only RSS support for now TBD: COS & LB */
3194         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP |
3195                                   VNIC_CFG_REQ_ENABLES_RSS_RULE);
3196         req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3197         req.cos_rule = cpu_to_le16(0xffff);
3198         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3199                 ring = 0;
3200         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
3201                 ring = vnic_id - 1;
3202
3203         grp_idx = bp->rx_ring[ring].bnapi->index;
3204         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3205         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3206
3207         req.lb_rule = cpu_to_le16(0xffff);
3208         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3209                               VLAN_HLEN);
3210
3211         if (bp->flags & BNXT_FLAG_STRIP_VLAN)
3212                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
3213
3214         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3215 }
3216
3217 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
3218 {
3219         u32 rc = 0;
3220
3221         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
3222                 struct hwrm_vnic_free_input req = {0};
3223
3224                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
3225                 req.vnic_id =
3226                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
3227
3228                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3229                 if (rc)
3230                         return rc;
3231                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
3232         }
3233         return rc;
3234 }
3235
3236 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
3237 {
3238         u16 i;
3239
3240         for (i = 0; i < bp->nr_vnics; i++)
3241                 bnxt_hwrm_vnic_free_one(bp, i);
3242 }
3243
3244 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
3245                                 unsigned int start_rx_ring_idx,
3246                                 unsigned int nr_rings)
3247 {
3248         int rc = 0;
3249         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
3250         struct hwrm_vnic_alloc_input req = {0};
3251         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3252
3253         /* map ring groups to this vnic */
3254         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
3255                 grp_idx = bp->rx_ring[i].bnapi->index;
3256                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
3257                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
3258                                    j, nr_rings);
3259                         break;
3260                 }
3261                 bp->vnic_info[vnic_id].fw_grp_ids[j] =
3262                                         bp->grp_info[grp_idx].fw_grp_id;
3263         }
3264
3265         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3266         if (vnic_id == 0)
3267                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
3268
3269         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
3270
3271         mutex_lock(&bp->hwrm_cmd_lock);
3272         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3273         if (!rc)
3274                 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
3275         mutex_unlock(&bp->hwrm_cmd_lock);
3276         return rc;
3277 }
3278
3279 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
3280 {
3281         u16 i;
3282         u32 rc = 0;
3283
3284         mutex_lock(&bp->hwrm_cmd_lock);
3285         for (i = 0; i < bp->rx_nr_rings; i++) {
3286                 struct hwrm_ring_grp_alloc_input req = {0};
3287                 struct hwrm_ring_grp_alloc_output *resp =
3288                                         bp->hwrm_cmd_resp_addr;
3289                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
3290
3291                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
3292
3293                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
3294                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
3295                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
3296                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
3297
3298                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3299                                         HWRM_CMD_TIMEOUT);
3300                 if (rc)
3301                         break;
3302
3303                 bp->grp_info[grp_idx].fw_grp_id =
3304                         le32_to_cpu(resp->ring_group_id);
3305         }
3306         mutex_unlock(&bp->hwrm_cmd_lock);
3307         return rc;
3308 }
3309
3310 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
3311 {
3312         u16 i;
3313         u32 rc = 0;
3314         struct hwrm_ring_grp_free_input req = {0};
3315
3316         if (!bp->grp_info)
3317                 return 0;
3318
3319         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
3320
3321         mutex_lock(&bp->hwrm_cmd_lock);
3322         for (i = 0; i < bp->cp_nr_rings; i++) {
3323                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
3324                         continue;
3325                 req.ring_group_id =
3326                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
3327
3328                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3329                                         HWRM_CMD_TIMEOUT);
3330                 if (rc)
3331                         break;
3332                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3333         }
3334         mutex_unlock(&bp->hwrm_cmd_lock);
3335         return rc;
3336 }
3337
3338 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
3339                                     struct bnxt_ring_struct *ring,
3340                                     u32 ring_type, u32 map_index,
3341                                     u32 stats_ctx_id)
3342 {
3343         int rc = 0, err = 0;
3344         struct hwrm_ring_alloc_input req = {0};
3345         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3346         u16 ring_id;
3347
3348         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
3349
3350         req.enables = 0;
3351         if (ring->nr_pages > 1) {
3352                 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
3353                 /* Page size is in log2 units */
3354                 req.page_size = BNXT_PAGE_SHIFT;
3355                 req.page_tbl_depth = 1;
3356         } else {
3357                 req.page_tbl_addr =  cpu_to_le64(ring->dma_arr[0]);
3358         }
3359         req.fbo = 0;
3360         /* Association of ring index with doorbell index and MSIX number */
3361         req.logical_id = cpu_to_le16(map_index);
3362
3363         switch (ring_type) {
3364         case HWRM_RING_ALLOC_TX:
3365                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
3366                 /* Association of transmit ring with completion ring */
3367                 req.cmpl_ring_id =
3368                         cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
3369                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
3370                 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
3371                 req.queue_id = cpu_to_le16(ring->queue_id);
3372                 break;
3373         case HWRM_RING_ALLOC_RX:
3374                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3375                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
3376                 break;
3377         case HWRM_RING_ALLOC_AGG:
3378                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3379                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
3380                 break;
3381         case HWRM_RING_ALLOC_CMPL:
3382                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
3383                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
3384                 if (bp->flags & BNXT_FLAG_USING_MSIX)
3385                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
3386                 break;
3387         default:
3388                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
3389                            ring_type);
3390                 return -1;
3391         }
3392
3393         mutex_lock(&bp->hwrm_cmd_lock);
3394         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3395         err = le16_to_cpu(resp->error_code);
3396         ring_id = le16_to_cpu(resp->ring_id);
3397         mutex_unlock(&bp->hwrm_cmd_lock);
3398
3399         if (rc || err) {
3400                 switch (ring_type) {
3401                 case RING_FREE_REQ_RING_TYPE_CMPL:
3402                         netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
3403                                    rc, err);
3404                         return -1;
3405
3406                 case RING_FREE_REQ_RING_TYPE_RX:
3407                         netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
3408                                    rc, err);
3409                         return -1;
3410
3411                 case RING_FREE_REQ_RING_TYPE_TX:
3412                         netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
3413                                    rc, err);
3414                         return -1;
3415
3416                 default:
3417                         netdev_err(bp->dev, "Invalid ring\n");
3418                         return -1;
3419                 }
3420         }
3421         ring->fw_ring_id = ring_id;
3422         return rc;
3423 }
3424
3425 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
3426 {
3427         int i, rc = 0;
3428
3429         for (i = 0; i < bp->cp_nr_rings; i++) {
3430                 struct bnxt_napi *bnapi = bp->bnapi[i];
3431                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3432                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3433
3434                 cpr->cp_doorbell = bp->bar1 + i * 0x80;
3435                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
3436                                               INVALID_STATS_CTX_ID);
3437                 if (rc)
3438                         goto err_out;
3439                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3440                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
3441         }
3442
3443         for (i = 0; i < bp->tx_nr_rings; i++) {
3444                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3445                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3446                 u32 map_idx = txr->bnapi->index;
3447                 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
3448
3449                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
3450                                               map_idx, fw_stats_ctx);
3451                 if (rc)
3452                         goto err_out;
3453                 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
3454         }
3455
3456         for (i = 0; i < bp->rx_nr_rings; i++) {
3457                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3458                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3459                 u32 map_idx = rxr->bnapi->index;
3460
3461                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
3462                                               map_idx, INVALID_STATS_CTX_ID);
3463                 if (rc)
3464                         goto err_out;
3465                 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
3466                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
3467                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3468         }
3469
3470         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3471                 for (i = 0; i < bp->rx_nr_rings; i++) {
3472                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3473                         struct bnxt_ring_struct *ring =
3474                                                 &rxr->rx_agg_ring_struct;
3475                         u32 grp_idx = rxr->bnapi->index;
3476                         u32 map_idx = grp_idx + bp->rx_nr_rings;
3477
3478                         rc = hwrm_ring_alloc_send_msg(bp, ring,
3479                                                       HWRM_RING_ALLOC_AGG,
3480                                                       map_idx,
3481                                                       INVALID_STATS_CTX_ID);
3482                         if (rc)
3483                                 goto err_out;
3484
3485                         rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
3486                         writel(DB_KEY_RX | rxr->rx_agg_prod,
3487                                rxr->rx_agg_doorbell);
3488                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
3489                 }
3490         }
3491 err_out:
3492         return rc;
3493 }
3494
3495 static int hwrm_ring_free_send_msg(struct bnxt *bp,
3496                                    struct bnxt_ring_struct *ring,
3497                                    u32 ring_type, int cmpl_ring_id)
3498 {
3499         int rc;
3500         struct hwrm_ring_free_input req = {0};
3501         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
3502         u16 error_code;
3503
3504         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
3505         req.ring_type = ring_type;
3506         req.ring_id = cpu_to_le16(ring->fw_ring_id);
3507
3508         mutex_lock(&bp->hwrm_cmd_lock);
3509         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3510         error_code = le16_to_cpu(resp->error_code);
3511         mutex_unlock(&bp->hwrm_cmd_lock);
3512
3513         if (rc || error_code) {
3514                 switch (ring_type) {
3515                 case RING_FREE_REQ_RING_TYPE_CMPL:
3516                         netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
3517                                    rc);
3518                         return rc;
3519                 case RING_FREE_REQ_RING_TYPE_RX:
3520                         netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
3521                                    rc);
3522                         return rc;
3523                 case RING_FREE_REQ_RING_TYPE_TX:
3524                         netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
3525                                    rc);
3526                         return rc;
3527                 default:
3528                         netdev_err(bp->dev, "Invalid ring\n");
3529                         return -1;
3530                 }
3531         }
3532         return 0;
3533 }
3534
3535 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
3536 {
3537         int i;
3538
3539         if (!bp->bnapi)
3540                 return;
3541
3542         for (i = 0; i < bp->tx_nr_rings; i++) {
3543                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3544                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3545                 u32 grp_idx = txr->bnapi->index;
3546                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3547
3548                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3549                         hwrm_ring_free_send_msg(bp, ring,
3550                                                 RING_FREE_REQ_RING_TYPE_TX,
3551                                                 close_path ? cmpl_ring_id :
3552                                                 INVALID_HW_RING_ID);
3553                         ring->fw_ring_id = INVALID_HW_RING_ID;
3554                 }
3555         }
3556
3557         for (i = 0; i < bp->rx_nr_rings; i++) {
3558                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3559                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3560                 u32 grp_idx = rxr->bnapi->index;
3561                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3562
3563                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3564                         hwrm_ring_free_send_msg(bp, ring,
3565                                                 RING_FREE_REQ_RING_TYPE_RX,
3566                                                 close_path ? cmpl_ring_id :
3567                                                 INVALID_HW_RING_ID);
3568                         ring->fw_ring_id = INVALID_HW_RING_ID;
3569                         bp->grp_info[grp_idx].rx_fw_ring_id =
3570                                 INVALID_HW_RING_ID;
3571                 }
3572         }
3573
3574         for (i = 0; i < bp->rx_nr_rings; i++) {
3575                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3576                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
3577                 u32 grp_idx = rxr->bnapi->index;
3578                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3579
3580                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3581                         hwrm_ring_free_send_msg(bp, ring,
3582                                                 RING_FREE_REQ_RING_TYPE_RX,
3583                                                 close_path ? cmpl_ring_id :
3584                                                 INVALID_HW_RING_ID);
3585                         ring->fw_ring_id = INVALID_HW_RING_ID;
3586                         bp->grp_info[grp_idx].agg_fw_ring_id =
3587                                 INVALID_HW_RING_ID;
3588                 }
3589         }
3590
3591         for (i = 0; i < bp->cp_nr_rings; i++) {
3592                 struct bnxt_napi *bnapi = bp->bnapi[i];
3593                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3594                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3595
3596                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3597                         hwrm_ring_free_send_msg(bp, ring,
3598                                                 RING_FREE_REQ_RING_TYPE_CMPL,
3599                                                 INVALID_HW_RING_ID);
3600                         ring->fw_ring_id = INVALID_HW_RING_ID;
3601                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3602                 }
3603         }
3604 }
3605
3606 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
3607         u32 buf_tmrs, u16 flags,
3608         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
3609 {
3610         req->flags = cpu_to_le16(flags);
3611         req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
3612         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
3613         req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
3614         req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
3615         /* Minimum time between 2 interrupts set to buf_tmr x 2 */
3616         req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
3617         req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
3618         req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
3619 }
3620
3621 int bnxt_hwrm_set_coal(struct bnxt *bp)
3622 {
3623         int i, rc = 0;
3624         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
3625                                                            req_tx = {0}, *req;
3626         u16 max_buf, max_buf_irq;
3627         u16 buf_tmr, buf_tmr_irq;
3628         u32 flags;
3629
3630         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
3631                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3632         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
3633                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3634
3635         /* Each rx completion (2 records) should be DMAed immediately.
3636          * DMA 1/4 of the completion buffers at a time.
3637          */
3638         max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
3639         /* max_buf must not be zero */
3640         max_buf = clamp_t(u16, max_buf, 1, 63);
3641         max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
3642         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
3643         /* buf timer set to 1/4 of interrupt timer */
3644         buf_tmr = max_t(u16, buf_tmr / 4, 1);
3645         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
3646         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3647
3648         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3649
3650         /* RING_IDLE generates more IRQs for lower latency.  Enable it only
3651          * if coal_ticks is less than 25 us.
3652          */
3653         if (bp->rx_coal_ticks < 25)
3654                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
3655
3656         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3657                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
3658
3659         /* max_buf must not be zero */
3660         max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
3661         max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
3662         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
3663         /* buf timer set to 1/4 of interrupt timer */
3664         buf_tmr = max_t(u16, buf_tmr / 4, 1);
3665         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
3666         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3667
3668         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3669         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3670                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
3671
3672         mutex_lock(&bp->hwrm_cmd_lock);
3673         for (i = 0; i < bp->cp_nr_rings; i++) {
3674                 struct bnxt_napi *bnapi = bp->bnapi[i];
3675
3676                 req = &req_rx;
3677                 if (!bnapi->rx_ring)
3678                         req = &req_tx;
3679                 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
3680
3681                 rc = _hwrm_send_message(bp, req, sizeof(*req),
3682                                         HWRM_CMD_TIMEOUT);
3683                 if (rc)
3684                         break;
3685         }
3686         mutex_unlock(&bp->hwrm_cmd_lock);
3687         return rc;
3688 }
3689
3690 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
3691 {
3692         int rc = 0, i;
3693         struct hwrm_stat_ctx_free_input req = {0};
3694
3695         if (!bp->bnapi)
3696                 return 0;
3697
3698         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
3699
3700         mutex_lock(&bp->hwrm_cmd_lock);
3701         for (i = 0; i < bp->cp_nr_rings; i++) {
3702                 struct bnxt_napi *bnapi = bp->bnapi[i];
3703                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3704
3705                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
3706                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
3707
3708                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3709                                                 HWRM_CMD_TIMEOUT);
3710                         if (rc)
3711                                 break;
3712
3713                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3714                 }
3715         }
3716         mutex_unlock(&bp->hwrm_cmd_lock);
3717         return rc;
3718 }
3719
3720 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
3721 {
3722         int rc = 0, i;
3723         struct hwrm_stat_ctx_alloc_input req = {0};
3724         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3725
3726         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
3727
3728         req.update_period_ms = cpu_to_le32(1000);
3729
3730         mutex_lock(&bp->hwrm_cmd_lock);
3731         for (i = 0; i < bp->cp_nr_rings; i++) {
3732                 struct bnxt_napi *bnapi = bp->bnapi[i];
3733                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3734
3735                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
3736
3737                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3738                                         HWRM_CMD_TIMEOUT);
3739                 if (rc)
3740                         break;
3741
3742                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
3743
3744                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
3745         }
3746         mutex_unlock(&bp->hwrm_cmd_lock);
3747         return 0;
3748 }
3749
3750 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
3751 {
3752         int rc = 0;
3753         struct hwrm_func_qcaps_input req = {0};
3754         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
3755
3756         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
3757         req.fid = cpu_to_le16(0xffff);
3758
3759         mutex_lock(&bp->hwrm_cmd_lock);
3760         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3761         if (rc)
3762                 goto hwrm_func_qcaps_exit;
3763
3764         if (BNXT_PF(bp)) {
3765                 struct bnxt_pf_info *pf = &bp->pf;
3766
3767                 pf->fw_fid = le16_to_cpu(resp->fid);
3768                 pf->port_id = le16_to_cpu(resp->port_id);
3769                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
3770                 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
3771                 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3772                 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3773                 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3774                 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3775                 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3776                 if (!pf->max_hw_ring_grps)
3777                         pf->max_hw_ring_grps = pf->max_tx_rings;
3778                 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3779                 pf->max_vnics = le16_to_cpu(resp->max_vnics);
3780                 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3781                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
3782                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
3783                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
3784                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
3785                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
3786                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
3787                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
3788                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
3789         } else {
3790 #ifdef CONFIG_BNXT_SRIOV
3791                 struct bnxt_vf_info *vf = &bp->vf;
3792
3793                 vf->fw_fid = le16_to_cpu(resp->fid);
3794                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
3795                 if (is_valid_ether_addr(vf->mac_addr))
3796                         /* overwrite netdev dev_adr with admin VF MAC */
3797                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
3798                 else
3799                         random_ether_addr(bp->dev->dev_addr);
3800
3801                 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3802                 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3803                 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3804                 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3805                 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3806                 if (!vf->max_hw_ring_grps)
3807                         vf->max_hw_ring_grps = vf->max_tx_rings;
3808                 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3809                 vf->max_vnics = le16_to_cpu(resp->max_vnics);
3810                 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3811 #endif
3812         }
3813
3814         bp->tx_push_thresh = 0;
3815         if (resp->flags &
3816             cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
3817                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
3818
3819 hwrm_func_qcaps_exit:
3820         mutex_unlock(&bp->hwrm_cmd_lock);
3821         return rc;
3822 }
3823
3824 static int bnxt_hwrm_func_reset(struct bnxt *bp)
3825 {
3826         struct hwrm_func_reset_input req = {0};
3827
3828         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
3829         req.enables = 0;
3830
3831         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
3832 }
3833
3834 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
3835 {
3836         int rc = 0;
3837         struct hwrm_queue_qportcfg_input req = {0};
3838         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
3839         u8 i, *qptr;
3840
3841         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
3842
3843         mutex_lock(&bp->hwrm_cmd_lock);
3844         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3845         if (rc)
3846                 goto qportcfg_exit;
3847
3848         if (!resp->max_configurable_queues) {
3849                 rc = -EINVAL;
3850                 goto qportcfg_exit;
3851         }
3852         bp->max_tc = resp->max_configurable_queues;
3853         if (bp->max_tc > BNXT_MAX_QUEUE)
3854                 bp->max_tc = BNXT_MAX_QUEUE;
3855
3856         qptr = &resp->queue_id0;
3857         for (i = 0; i < bp->max_tc; i++) {
3858                 bp->q_info[i].queue_id = *qptr++;
3859                 bp->q_info[i].queue_profile = *qptr++;
3860         }
3861
3862 qportcfg_exit:
3863         mutex_unlock(&bp->hwrm_cmd_lock);
3864         return rc;
3865 }
3866
3867 static int bnxt_hwrm_ver_get(struct bnxt *bp)
3868 {
3869         int rc;
3870         struct hwrm_ver_get_input req = {0};
3871         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
3872
3873         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
3874         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
3875         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
3876         req.hwrm_intf_min = HWRM_VERSION_MINOR;
3877         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
3878         mutex_lock(&bp->hwrm_cmd_lock);
3879         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3880         if (rc)
3881                 goto hwrm_ver_get_exit;
3882
3883         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
3884
3885         bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
3886                              resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
3887         if (resp->hwrm_intf_maj < 1) {
3888                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
3889                             resp->hwrm_intf_maj, resp->hwrm_intf_min,
3890                             resp->hwrm_intf_upd);
3891                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
3892         }
3893         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
3894                  resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
3895                  resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
3896
3897         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
3898         if (!bp->hwrm_cmd_timeout)
3899                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
3900
3901         if (resp->hwrm_intf_maj >= 1)
3902                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
3903
3904 hwrm_ver_get_exit:
3905         mutex_unlock(&bp->hwrm_cmd_lock);
3906         return rc;
3907 }
3908
3909 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
3910 {
3911         int rc;
3912         struct bnxt_pf_info *pf = &bp->pf;
3913         struct hwrm_port_qstats_input req = {0};
3914
3915         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
3916                 return 0;
3917
3918         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
3919         req.port_id = cpu_to_le16(pf->port_id);
3920         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
3921         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
3922         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3923         return rc;
3924 }
3925
3926 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
3927 {
3928         if (bp->vxlan_port_cnt) {
3929                 bnxt_hwrm_tunnel_dst_port_free(
3930                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
3931         }
3932         bp->vxlan_port_cnt = 0;
3933         if (bp->nge_port_cnt) {
3934                 bnxt_hwrm_tunnel_dst_port_free(
3935                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
3936         }
3937         bp->nge_port_cnt = 0;
3938 }
3939
3940 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
3941 {
3942         int rc, i;
3943         u32 tpa_flags = 0;
3944
3945         if (set_tpa)
3946                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
3947         for (i = 0; i < bp->nr_vnics; i++) {
3948                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
3949                 if (rc) {
3950                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
3951                                    rc, i);
3952                         return rc;
3953                 }
3954         }
3955         return 0;
3956 }
3957
3958 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
3959 {
3960         int i;
3961
3962         for (i = 0; i < bp->nr_vnics; i++)
3963                 bnxt_hwrm_vnic_set_rss(bp, i, false);
3964 }
3965
3966 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
3967                                     bool irq_re_init)
3968 {
3969         if (bp->vnic_info) {
3970                 bnxt_hwrm_clear_vnic_filter(bp);
3971                 /* clear all RSS setting before free vnic ctx */
3972                 bnxt_hwrm_clear_vnic_rss(bp);
3973                 bnxt_hwrm_vnic_ctx_free(bp);
3974                 /* before free the vnic, undo the vnic tpa settings */
3975                 if (bp->flags & BNXT_FLAG_TPA)
3976                         bnxt_set_tpa(bp, false);
3977                 bnxt_hwrm_vnic_free(bp);
3978         }
3979         bnxt_hwrm_ring_free(bp, close_path);
3980         bnxt_hwrm_ring_grp_free(bp);
3981         if (irq_re_init) {
3982                 bnxt_hwrm_stat_ctx_free(bp);
3983                 bnxt_hwrm_free_tunnel_ports(bp);
3984         }
3985 }
3986
3987 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
3988 {
3989         int rc;
3990
3991         /* allocate context for vnic */
3992         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id);
3993         if (rc) {
3994                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
3995                            vnic_id, rc);
3996                 goto vnic_setup_err;
3997         }
3998         bp->rsscos_nr_ctxs++;
3999
4000         /* configure default vnic, ring grp */
4001         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
4002         if (rc) {
4003                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
4004                            vnic_id, rc);
4005                 goto vnic_setup_err;
4006         }
4007
4008         /* Enable RSS hashing on vnic */
4009         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
4010         if (rc) {
4011                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
4012                            vnic_id, rc);
4013                 goto vnic_setup_err;
4014         }
4015
4016         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4017                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
4018                 if (rc) {
4019                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
4020                                    vnic_id, rc);
4021                 }
4022         }
4023
4024 vnic_setup_err:
4025         return rc;
4026 }
4027
4028 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
4029 {
4030 #ifdef CONFIG_RFS_ACCEL
4031         int i, rc = 0;
4032
4033         for (i = 0; i < bp->rx_nr_rings; i++) {
4034                 u16 vnic_id = i + 1;
4035                 u16 ring_id = i;
4036
4037                 if (vnic_id >= bp->nr_vnics)
4038                         break;
4039
4040                 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG;
4041                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
4042                 if (rc) {
4043                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4044                                    vnic_id, rc);
4045                         break;
4046                 }
4047                 rc = bnxt_setup_vnic(bp, vnic_id);
4048                 if (rc)
4049                         break;
4050         }
4051         return rc;
4052 #else
4053         return 0;
4054 #endif
4055 }
4056
4057 static int bnxt_cfg_rx_mode(struct bnxt *);
4058
4059 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
4060 {
4061         int rc = 0;
4062
4063         if (irq_re_init) {
4064                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
4065                 if (rc) {
4066                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
4067                                    rc);
4068                         goto err_out;
4069                 }
4070         }
4071
4072         rc = bnxt_hwrm_ring_alloc(bp);
4073         if (rc) {
4074                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
4075                 goto err_out;
4076         }
4077
4078         rc = bnxt_hwrm_ring_grp_alloc(bp);
4079         if (rc) {
4080                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
4081                 goto err_out;
4082         }
4083
4084         /* default vnic 0 */
4085         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, bp->rx_nr_rings);
4086         if (rc) {
4087                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
4088                 goto err_out;
4089         }
4090
4091         rc = bnxt_setup_vnic(bp, 0);
4092         if (rc)
4093                 goto err_out;
4094
4095         if (bp->flags & BNXT_FLAG_RFS) {
4096                 rc = bnxt_alloc_rfs_vnics(bp);
4097                 if (rc)
4098                         goto err_out;
4099         }
4100
4101         if (bp->flags & BNXT_FLAG_TPA) {
4102                 rc = bnxt_set_tpa(bp, true);
4103                 if (rc)
4104                         goto err_out;
4105         }
4106
4107         if (BNXT_VF(bp))
4108                 bnxt_update_vf_mac(bp);
4109
4110         /* Filter for default vnic 0 */
4111         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
4112         if (rc) {
4113                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
4114                 goto err_out;
4115         }
4116         bp->vnic_info[0].uc_filter_count = 1;
4117
4118         bp->vnic_info[0].rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
4119
4120         if ((bp->dev->flags & IFF_PROMISC) && BNXT_PF(bp))
4121                 bp->vnic_info[0].rx_mask |=
4122                                 CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
4123
4124         rc = bnxt_cfg_rx_mode(bp);
4125         if (rc)
4126                 goto err_out;
4127
4128         rc = bnxt_hwrm_set_coal(bp);
4129         if (rc)
4130                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
4131                             rc);
4132
4133         return 0;
4134
4135 err_out:
4136         bnxt_hwrm_resource_free(bp, 0, true);
4137
4138         return rc;
4139 }
4140
4141 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
4142 {
4143         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
4144         return 0;
4145 }
4146
4147 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
4148 {
4149         bnxt_init_rx_rings(bp);
4150         bnxt_init_tx_rings(bp);
4151         bnxt_init_ring_grps(bp, irq_re_init);
4152         bnxt_init_vnics(bp);
4153
4154         return bnxt_init_chip(bp, irq_re_init);
4155 }
4156
4157 static void bnxt_disable_int(struct bnxt *bp)
4158 {
4159         int i;
4160
4161         if (!bp->bnapi)
4162                 return;
4163
4164         for (i = 0; i < bp->cp_nr_rings; i++) {
4165                 struct bnxt_napi *bnapi = bp->bnapi[i];
4166                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4167
4168                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4169         }
4170 }
4171
4172 static void bnxt_enable_int(struct bnxt *bp)
4173 {
4174         int i;
4175
4176         atomic_set(&bp->intr_sem, 0);
4177         for (i = 0; i < bp->cp_nr_rings; i++) {
4178                 struct bnxt_napi *bnapi = bp->bnapi[i];
4179                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4180
4181                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
4182         }
4183 }
4184
4185 static int bnxt_set_real_num_queues(struct bnxt *bp)
4186 {
4187         int rc;
4188         struct net_device *dev = bp->dev;
4189
4190         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings);
4191         if (rc)
4192                 return rc;
4193
4194         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
4195         if (rc)
4196                 return rc;
4197
4198 #ifdef CONFIG_RFS_ACCEL
4199         if (bp->flags & BNXT_FLAG_RFS)
4200                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
4201 #endif
4202
4203         return rc;
4204 }
4205
4206 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4207                            bool shared)
4208 {
4209         int _rx = *rx, _tx = *tx;
4210
4211         if (shared) {
4212                 *rx = min_t(int, _rx, max);
4213                 *tx = min_t(int, _tx, max);
4214         } else {
4215                 if (max < 2)
4216                         return -ENOMEM;
4217
4218                 while (_rx + _tx > max) {
4219                         if (_rx > _tx && _rx > 1)
4220                                 _rx--;
4221                         else if (_tx > 1)
4222                                 _tx--;
4223                 }
4224                 *rx = _rx;
4225                 *tx = _tx;
4226         }
4227         return 0;
4228 }
4229
4230 static int bnxt_setup_msix(struct bnxt *bp)
4231 {
4232         struct msix_entry *msix_ent;
4233         struct net_device *dev = bp->dev;
4234         int i, total_vecs, rc = 0, min = 1;
4235         const int len = sizeof(bp->irq_tbl[0].name);
4236
4237         bp->flags &= ~BNXT_FLAG_USING_MSIX;
4238         total_vecs = bp->cp_nr_rings;
4239
4240         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
4241         if (!msix_ent)
4242                 return -ENOMEM;
4243
4244         for (i = 0; i < total_vecs; i++) {
4245                 msix_ent[i].entry = i;
4246                 msix_ent[i].vector = 0;
4247         }
4248
4249         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
4250                 min = 2;
4251
4252         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
4253         if (total_vecs < 0) {
4254                 rc = -ENODEV;
4255                 goto msix_setup_exit;
4256         }
4257
4258         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
4259         if (bp->irq_tbl) {
4260                 int tcs;
4261
4262                 /* Trim rings based upon num of vectors allocated */
4263                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
4264                                      total_vecs, min == 1);
4265                 if (rc)
4266                         goto msix_setup_exit;
4267
4268                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4269                 tcs = netdev_get_num_tc(dev);
4270                 if (tcs > 1) {
4271                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs;
4272                         if (bp->tx_nr_rings_per_tc == 0) {
4273                                 netdev_reset_tc(dev);
4274                                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4275                         } else {
4276                                 int i, off, count;
4277
4278                                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs;
4279                                 for (i = 0; i < tcs; i++) {
4280                                         count = bp->tx_nr_rings_per_tc;
4281                                         off = i * count;
4282                                         netdev_set_tc_queue(dev, i, count, off);
4283                                 }
4284                         }
4285                 }
4286                 bp->cp_nr_rings = total_vecs;
4287
4288                 for (i = 0; i < bp->cp_nr_rings; i++) {
4289                         char *attr;
4290
4291                         bp->irq_tbl[i].vector = msix_ent[i].vector;
4292                         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4293                                 attr = "TxRx";
4294                         else if (i < bp->rx_nr_rings)
4295                                 attr = "rx";
4296                         else
4297                                 attr = "tx";
4298
4299                         snprintf(bp->irq_tbl[i].name, len,
4300                                  "%s-%s-%d", dev->name, attr, i);
4301                         bp->irq_tbl[i].handler = bnxt_msix;
4302                 }
4303                 rc = bnxt_set_real_num_queues(bp);
4304                 if (rc)
4305                         goto msix_setup_exit;
4306         } else {
4307                 rc = -ENOMEM;
4308                 goto msix_setup_exit;
4309         }
4310         bp->flags |= BNXT_FLAG_USING_MSIX;
4311         kfree(msix_ent);
4312         return 0;
4313
4314 msix_setup_exit:
4315         netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc);
4316         pci_disable_msix(bp->pdev);
4317         kfree(msix_ent);
4318         return rc;
4319 }
4320
4321 static int bnxt_setup_inta(struct bnxt *bp)
4322 {
4323         int rc;
4324         const int len = sizeof(bp->irq_tbl[0].name);
4325
4326         if (netdev_get_num_tc(bp->dev))
4327                 netdev_reset_tc(bp->dev);
4328
4329         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
4330         if (!bp->irq_tbl) {
4331                 rc = -ENOMEM;
4332                 return rc;
4333         }
4334         bp->rx_nr_rings = 1;
4335         bp->tx_nr_rings = 1;
4336         bp->cp_nr_rings = 1;
4337         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4338         bp->flags |= BNXT_FLAG_SHARED_RINGS;
4339         bp->irq_tbl[0].vector = bp->pdev->irq;
4340         snprintf(bp->irq_tbl[0].name, len,
4341                  "%s-%s-%d", bp->dev->name, "TxRx", 0);
4342         bp->irq_tbl[0].handler = bnxt_inta;
4343         rc = bnxt_set_real_num_queues(bp);
4344         return rc;
4345 }
4346
4347 static int bnxt_setup_int_mode(struct bnxt *bp)
4348 {
4349         int rc = 0;
4350
4351         if (bp->flags & BNXT_FLAG_MSIX_CAP)
4352                 rc = bnxt_setup_msix(bp);
4353
4354         if (!(bp->flags & BNXT_FLAG_USING_MSIX)) {
4355                 /* fallback to INTA */
4356                 rc = bnxt_setup_inta(bp);
4357         }
4358         return rc;
4359 }
4360
4361 static void bnxt_free_irq(struct bnxt *bp)
4362 {
4363         struct bnxt_irq *irq;
4364         int i;
4365
4366 #ifdef CONFIG_RFS_ACCEL
4367         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
4368         bp->dev->rx_cpu_rmap = NULL;
4369 #endif
4370         if (!bp->irq_tbl)
4371                 return;
4372
4373         for (i = 0; i < bp->cp_nr_rings; i++) {
4374                 irq = &bp->irq_tbl[i];
4375                 if (irq->requested)
4376                         free_irq(irq->vector, bp->bnapi[i]);
4377                 irq->requested = 0;
4378         }
4379         if (bp->flags & BNXT_FLAG_USING_MSIX)
4380                 pci_disable_msix(bp->pdev);
4381         kfree(bp->irq_tbl);
4382         bp->irq_tbl = NULL;
4383 }
4384
4385 static int bnxt_request_irq(struct bnxt *bp)
4386 {
4387         int i, j, rc = 0;
4388         unsigned long flags = 0;
4389 #ifdef CONFIG_RFS_ACCEL
4390         struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
4391 #endif
4392
4393         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
4394                 flags = IRQF_SHARED;
4395
4396         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
4397                 struct bnxt_irq *irq = &bp->irq_tbl[i];
4398 #ifdef CONFIG_RFS_ACCEL
4399                 if (rmap && bp->bnapi[i]->rx_ring) {
4400                         rc = irq_cpu_rmap_add(rmap, irq->vector);
4401                         if (rc)
4402                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
4403                                             j);
4404                         j++;
4405                 }
4406 #endif
4407                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
4408                                  bp->bnapi[i]);
4409                 if (rc)
4410                         break;
4411
4412                 irq->requested = 1;
4413         }
4414         return rc;
4415 }
4416
4417 static void bnxt_del_napi(struct bnxt *bp)
4418 {
4419         int i;
4420
4421         if (!bp->bnapi)
4422                 return;
4423
4424         for (i = 0; i < bp->cp_nr_rings; i++) {
4425                 struct bnxt_napi *bnapi = bp->bnapi[i];
4426
4427                 napi_hash_del(&bnapi->napi);
4428                 netif_napi_del(&bnapi->napi);
4429         }
4430 }
4431
4432 static void bnxt_init_napi(struct bnxt *bp)
4433 {
4434         int i;
4435         struct bnxt_napi *bnapi;
4436
4437         if (bp->flags & BNXT_FLAG_USING_MSIX) {
4438                 for (i = 0; i < bp->cp_nr_rings; i++) {
4439                         bnapi = bp->bnapi[i];
4440                         netif_napi_add(bp->dev, &bnapi->napi,
4441                                        bnxt_poll, 64);
4442                 }
4443         } else {
4444                 bnapi = bp->bnapi[0];
4445                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
4446         }
4447 }
4448
4449 static void bnxt_disable_napi(struct bnxt *bp)
4450 {
4451         int i;
4452
4453         if (!bp->bnapi)
4454                 return;
4455
4456         for (i = 0; i < bp->cp_nr_rings; i++) {
4457                 napi_disable(&bp->bnapi[i]->napi);
4458                 bnxt_disable_poll(bp->bnapi[i]);
4459         }
4460 }
4461
4462 static void bnxt_enable_napi(struct bnxt *bp)
4463 {
4464         int i;
4465
4466         for (i = 0; i < bp->cp_nr_rings; i++) {
4467                 bnxt_enable_poll(bp->bnapi[i]);
4468                 napi_enable(&bp->bnapi[i]->napi);
4469         }
4470 }
4471
4472 static void bnxt_tx_disable(struct bnxt *bp)
4473 {
4474         int i;
4475         struct bnxt_tx_ring_info *txr;
4476         struct netdev_queue *txq;
4477
4478         if (bp->tx_ring) {
4479                 for (i = 0; i < bp->tx_nr_rings; i++) {
4480                         txr = &bp->tx_ring[i];
4481                         txq = netdev_get_tx_queue(bp->dev, i);
4482                         __netif_tx_lock(txq, smp_processor_id());
4483                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
4484                         __netif_tx_unlock(txq);
4485                 }
4486         }
4487         /* Stop all TX queues */
4488         netif_tx_disable(bp->dev);
4489         netif_carrier_off(bp->dev);
4490 }
4491
4492 static void bnxt_tx_enable(struct bnxt *bp)
4493 {
4494         int i;
4495         struct bnxt_tx_ring_info *txr;
4496         struct netdev_queue *txq;
4497
4498         for (i = 0; i < bp->tx_nr_rings; i++) {
4499                 txr = &bp->tx_ring[i];
4500                 txq = netdev_get_tx_queue(bp->dev, i);
4501                 txr->dev_state = 0;
4502         }
4503         netif_tx_wake_all_queues(bp->dev);
4504         if (bp->link_info.link_up)
4505                 netif_carrier_on(bp->dev);
4506 }
4507
4508 static void bnxt_report_link(struct bnxt *bp)
4509 {
4510         if (bp->link_info.link_up) {
4511                 const char *duplex;
4512                 const char *flow_ctrl;
4513                 u16 speed;
4514
4515                 netif_carrier_on(bp->dev);
4516                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
4517                         duplex = "full";
4518                 else
4519                         duplex = "half";
4520                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
4521                         flow_ctrl = "ON - receive & transmit";
4522                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
4523                         flow_ctrl = "ON - transmit";
4524                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
4525                         flow_ctrl = "ON - receive";
4526                 else
4527                         flow_ctrl = "none";
4528                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
4529                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
4530                             speed, duplex, flow_ctrl);
4531                 if (bp->flags & BNXT_FLAG_EEE_CAP)
4532                         netdev_info(bp->dev, "EEE is %s\n",
4533                                     bp->eee.eee_active ? "active" :
4534                                                          "not active");
4535         } else {
4536                 netif_carrier_off(bp->dev);
4537                 netdev_err(bp->dev, "NIC Link is Down\n");
4538         }
4539 }
4540
4541 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
4542 {
4543         int rc = 0;
4544         struct hwrm_port_phy_qcaps_input req = {0};
4545         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4546
4547         if (bp->hwrm_spec_code < 0x10201)
4548                 return 0;
4549
4550         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
4551
4552         mutex_lock(&bp->hwrm_cmd_lock);
4553         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4554         if (rc)
4555                 goto hwrm_phy_qcaps_exit;
4556
4557         if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
4558                 struct ethtool_eee *eee = &bp->eee;
4559                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
4560
4561                 bp->flags |= BNXT_FLAG_EEE_CAP;
4562                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4563                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
4564                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
4565                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
4566                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
4567         }
4568
4569 hwrm_phy_qcaps_exit:
4570         mutex_unlock(&bp->hwrm_cmd_lock);
4571         return rc;
4572 }
4573
4574 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
4575 {
4576         int rc = 0;
4577         struct bnxt_link_info *link_info = &bp->link_info;
4578         struct hwrm_port_phy_qcfg_input req = {0};
4579         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4580         u8 link_up = link_info->link_up;
4581
4582         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
4583
4584         mutex_lock(&bp->hwrm_cmd_lock);
4585         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4586         if (rc) {
4587                 mutex_unlock(&bp->hwrm_cmd_lock);
4588                 return rc;
4589         }
4590
4591         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
4592         link_info->phy_link_status = resp->link;
4593         link_info->duplex =  resp->duplex;
4594         link_info->pause = resp->pause;
4595         link_info->auto_mode = resp->auto_mode;
4596         link_info->auto_pause_setting = resp->auto_pause;
4597         link_info->lp_pause = resp->link_partner_adv_pause;
4598         link_info->force_pause_setting = resp->force_pause;
4599         link_info->duplex_setting = resp->duplex;
4600         if (link_info->phy_link_status == BNXT_LINK_LINK)
4601                 link_info->link_speed = le16_to_cpu(resp->link_speed);
4602         else
4603                 link_info->link_speed = 0;
4604         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
4605         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
4606         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
4607         link_info->lp_auto_link_speeds =
4608                 le16_to_cpu(resp->link_partner_adv_speeds);
4609         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
4610         link_info->phy_ver[0] = resp->phy_maj;
4611         link_info->phy_ver[1] = resp->phy_min;
4612         link_info->phy_ver[2] = resp->phy_bld;
4613         link_info->media_type = resp->media_type;
4614         link_info->transceiver = resp->xcvr_pkg_type;
4615         link_info->phy_addr = resp->eee_config_phy_addr &
4616                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
4617
4618         if (bp->flags & BNXT_FLAG_EEE_CAP) {
4619                 struct ethtool_eee *eee = &bp->eee;
4620                 u16 fw_speeds;
4621
4622                 eee->eee_active = 0;
4623                 if (resp->eee_config_phy_addr &
4624                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
4625                         eee->eee_active = 1;
4626                         fw_speeds = le16_to_cpu(
4627                                 resp->link_partner_adv_eee_link_speed_mask);
4628                         eee->lp_advertised =
4629                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4630                 }
4631
4632                 /* Pull initial EEE config */
4633                 if (!chng_link_state) {
4634                         if (resp->eee_config_phy_addr &
4635                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
4636                                 eee->eee_enabled = 1;
4637
4638                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
4639                         eee->advertised =
4640                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4641
4642                         if (resp->eee_config_phy_addr &
4643                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
4644                                 __le32 tmr;
4645
4646                                 eee->tx_lpi_enabled = 1;
4647                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
4648                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
4649                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
4650                         }
4651                 }
4652         }
4653         /* TODO: need to add more logic to report VF link */
4654         if (chng_link_state) {
4655                 if (link_info->phy_link_status == BNXT_LINK_LINK)
4656                         link_info->link_up = 1;
4657                 else
4658                         link_info->link_up = 0;
4659                 if (link_up != link_info->link_up)
4660                         bnxt_report_link(bp);
4661         } else {
4662                 /* alwasy link down if not require to update link state */
4663                 link_info->link_up = 0;
4664         }
4665         mutex_unlock(&bp->hwrm_cmd_lock);
4666         return 0;
4667 }
4668
4669 static void
4670 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
4671 {
4672         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
4673                 if (bp->hwrm_spec_code >= 0x10201)
4674                         req->auto_pause =
4675                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
4676                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4677                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
4678                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4679                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
4680                 req->enables |=
4681                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4682         } else {
4683                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4684                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
4685                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4686                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
4687                 req->enables |=
4688                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
4689                 if (bp->hwrm_spec_code >= 0x10201) {
4690                         req->auto_pause = req->force_pause;
4691                         req->enables |= cpu_to_le32(
4692                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4693                 }
4694         }
4695 }
4696
4697 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
4698                                       struct hwrm_port_phy_cfg_input *req)
4699 {
4700         u8 autoneg = bp->link_info.autoneg;
4701         u16 fw_link_speed = bp->link_info.req_link_speed;
4702         u32 advertising = bp->link_info.advertising;
4703
4704         if (autoneg & BNXT_AUTONEG_SPEED) {
4705                 req->auto_mode |=
4706                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
4707
4708                 req->enables |= cpu_to_le32(
4709                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
4710                 req->auto_link_speed_mask = cpu_to_le16(advertising);
4711
4712                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
4713                 req->flags |=
4714                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
4715         } else {
4716                 req->force_link_speed = cpu_to_le16(fw_link_speed);
4717                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
4718         }
4719
4720         /* tell chimp that the setting takes effect immediately */
4721         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
4722 }
4723
4724 int bnxt_hwrm_set_pause(struct bnxt *bp)
4725 {
4726         struct hwrm_port_phy_cfg_input req = {0};
4727         int rc;
4728
4729         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4730         bnxt_hwrm_set_pause_common(bp, &req);
4731
4732         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
4733             bp->link_info.force_link_chng)
4734                 bnxt_hwrm_set_link_common(bp, &req);
4735
4736         mutex_lock(&bp->hwrm_cmd_lock);
4737         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4738         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
4739                 /* since changing of pause setting doesn't trigger any link
4740                  * change event, the driver needs to update the current pause
4741                  * result upon successfully return of the phy_cfg command
4742                  */
4743                 bp->link_info.pause =
4744                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
4745                 bp->link_info.auto_pause_setting = 0;
4746                 if (!bp->link_info.force_link_chng)
4747                         bnxt_report_link(bp);
4748         }
4749         bp->link_info.force_link_chng = false;
4750         mutex_unlock(&bp->hwrm_cmd_lock);
4751         return rc;
4752 }
4753
4754 static void bnxt_hwrm_set_eee(struct bnxt *bp,
4755                               struct hwrm_port_phy_cfg_input *req)
4756 {
4757         struct ethtool_eee *eee = &bp->eee;
4758
4759         if (eee->eee_enabled) {
4760                 u16 eee_speeds;
4761                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
4762
4763                 if (eee->tx_lpi_enabled)
4764                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
4765                 else
4766                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
4767
4768                 req->flags |= cpu_to_le32(flags);
4769                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
4770                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
4771                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
4772         } else {
4773                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
4774         }
4775 }
4776
4777 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
4778 {
4779         struct hwrm_port_phy_cfg_input req = {0};
4780
4781         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4782         if (set_pause)
4783                 bnxt_hwrm_set_pause_common(bp, &req);
4784
4785         bnxt_hwrm_set_link_common(bp, &req);
4786
4787         if (set_eee)
4788                 bnxt_hwrm_set_eee(bp, &req);
4789         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4790 }
4791
4792 static bool bnxt_eee_config_ok(struct bnxt *bp)
4793 {
4794         struct ethtool_eee *eee = &bp->eee;
4795         struct bnxt_link_info *link_info = &bp->link_info;
4796
4797         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
4798                 return true;
4799
4800         if (eee->eee_enabled) {
4801                 u32 advertising =
4802                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
4803
4804                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4805                         eee->eee_enabled = 0;
4806                         return false;
4807                 }
4808                 if (eee->advertised & ~advertising) {
4809                         eee->advertised = advertising & eee->supported;
4810                         return false;
4811                 }
4812         }
4813         return true;
4814 }
4815
4816 static int bnxt_update_phy_setting(struct bnxt *bp)
4817 {
4818         int rc;
4819         bool update_link = false;
4820         bool update_pause = false;
4821         bool update_eee = false;
4822         struct bnxt_link_info *link_info = &bp->link_info;
4823
4824         rc = bnxt_update_link(bp, true);
4825         if (rc) {
4826                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
4827                            rc);
4828                 return rc;
4829         }
4830         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
4831             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
4832             link_info->req_flow_ctrl)
4833                 update_pause = true;
4834         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
4835             link_info->force_pause_setting != link_info->req_flow_ctrl)
4836                 update_pause = true;
4837         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4838                 if (BNXT_AUTO_MODE(link_info->auto_mode))
4839                         update_link = true;
4840                 if (link_info->req_link_speed != link_info->force_link_speed)
4841                         update_link = true;
4842                 if (link_info->req_duplex != link_info->duplex_setting)
4843                         update_link = true;
4844         } else {
4845                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
4846                         update_link = true;
4847                 if (link_info->advertising != link_info->auto_link_speeds)
4848                         update_link = true;
4849         }
4850
4851         if (!bnxt_eee_config_ok(bp))
4852                 update_eee = true;
4853
4854         if (update_link)
4855                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
4856         else if (update_pause)
4857                 rc = bnxt_hwrm_set_pause(bp);
4858         if (rc) {
4859                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
4860                            rc);
4861                 return rc;
4862         }
4863
4864         return rc;
4865 }
4866
4867 /* Common routine to pre-map certain register block to different GRC window.
4868  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
4869  * in PF and 3 windows in VF that can be customized to map in different
4870  * register blocks.
4871  */
4872 static void bnxt_preset_reg_win(struct bnxt *bp)
4873 {
4874         if (BNXT_PF(bp)) {
4875                 /* CAG registers map to GRC window #4 */
4876                 writel(BNXT_CAG_REG_BASE,
4877                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
4878         }
4879 }
4880
4881 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
4882 {
4883         int rc = 0;
4884
4885         bnxt_preset_reg_win(bp);
4886         netif_carrier_off(bp->dev);
4887         if (irq_re_init) {
4888                 rc = bnxt_setup_int_mode(bp);
4889                 if (rc) {
4890                         netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
4891                                    rc);
4892                         return rc;
4893                 }
4894         }
4895         if ((bp->flags & BNXT_FLAG_RFS) &&
4896             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
4897                 /* disable RFS if falling back to INTA */
4898                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
4899                 bp->flags &= ~BNXT_FLAG_RFS;
4900         }
4901
4902         rc = bnxt_alloc_mem(bp, irq_re_init);
4903         if (rc) {
4904                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
4905                 goto open_err_free_mem;
4906         }
4907
4908         if (irq_re_init) {
4909                 bnxt_init_napi(bp);
4910                 rc = bnxt_request_irq(bp);
4911                 if (rc) {
4912                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
4913                         goto open_err;
4914                 }
4915         }
4916
4917         bnxt_enable_napi(bp);
4918
4919         rc = bnxt_init_nic(bp, irq_re_init);
4920         if (rc) {
4921                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
4922                 goto open_err;
4923         }
4924
4925         if (link_re_init) {
4926                 rc = bnxt_update_phy_setting(bp);
4927                 if (rc)
4928                         netdev_warn(bp->dev, "failed to update phy settings\n");
4929         }
4930
4931         if (irq_re_init) {
4932 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
4933                 vxlan_get_rx_port(bp->dev);
4934 #endif
4935                 if (!bnxt_hwrm_tunnel_dst_port_alloc(
4936                                 bp, htons(0x17c1),
4937                                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
4938                         bp->nge_port_cnt = 1;
4939         }
4940
4941         set_bit(BNXT_STATE_OPEN, &bp->state);
4942         bnxt_enable_int(bp);
4943         /* Enable TX queues */
4944         bnxt_tx_enable(bp);
4945         mod_timer(&bp->timer, jiffies + bp->current_interval);
4946         bnxt_update_link(bp, true);
4947
4948         return 0;
4949
4950 open_err:
4951         bnxt_disable_napi(bp);
4952         bnxt_del_napi(bp);
4953
4954 open_err_free_mem:
4955         bnxt_free_skbs(bp);
4956         bnxt_free_irq(bp);
4957         bnxt_free_mem(bp, true);
4958         return rc;
4959 }
4960
4961 /* rtnl_lock held */
4962 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
4963 {
4964         int rc = 0;
4965
4966         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
4967         if (rc) {
4968                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
4969                 dev_close(bp->dev);
4970         }
4971         return rc;
4972 }
4973
4974 static int bnxt_open(struct net_device *dev)
4975 {
4976         struct bnxt *bp = netdev_priv(dev);
4977         int rc = 0;
4978
4979         rc = bnxt_hwrm_func_reset(bp);
4980         if (rc) {
4981                 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n",
4982                            rc);
4983                 rc = -1;
4984                 return rc;
4985         }
4986         return __bnxt_open_nic(bp, true, true);
4987 }
4988
4989 static void bnxt_disable_int_sync(struct bnxt *bp)
4990 {
4991         int i;
4992
4993         atomic_inc(&bp->intr_sem);
4994         if (!netif_running(bp->dev))
4995                 return;
4996
4997         bnxt_disable_int(bp);
4998         for (i = 0; i < bp->cp_nr_rings; i++)
4999                 synchronize_irq(bp->irq_tbl[i].vector);
5000 }
5001
5002 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5003 {
5004         int rc = 0;
5005
5006 #ifdef CONFIG_BNXT_SRIOV
5007         if (bp->sriov_cfg) {
5008                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
5009                                                       !bp->sriov_cfg,
5010                                                       BNXT_SRIOV_CFG_WAIT_TMO);
5011                 if (rc)
5012                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
5013         }
5014 #endif
5015         /* Change device state to avoid TX queue wake up's */
5016         bnxt_tx_disable(bp);
5017
5018         clear_bit(BNXT_STATE_OPEN, &bp->state);
5019         smp_mb__after_atomic();
5020         while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state))
5021                 msleep(20);
5022
5023         /* Flush rings before disabling interrupts */
5024         bnxt_shutdown_nic(bp, irq_re_init);
5025
5026         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
5027
5028         bnxt_disable_napi(bp);
5029         bnxt_disable_int_sync(bp);
5030         del_timer_sync(&bp->timer);
5031         bnxt_free_skbs(bp);
5032
5033         if (irq_re_init) {
5034                 bnxt_free_irq(bp);
5035                 bnxt_del_napi(bp);
5036         }
5037         bnxt_free_mem(bp, irq_re_init);
5038         return rc;
5039 }
5040
5041 static int bnxt_close(struct net_device *dev)
5042 {
5043         struct bnxt *bp = netdev_priv(dev);
5044
5045         bnxt_close_nic(bp, true, true);
5046         return 0;
5047 }
5048
5049 /* rtnl_lock held */
5050 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5051 {
5052         switch (cmd) {
5053         case SIOCGMIIPHY:
5054                 /* fallthru */
5055         case SIOCGMIIREG: {
5056                 if (!netif_running(dev))
5057                         return -EAGAIN;
5058
5059                 return 0;
5060         }
5061
5062         case SIOCSMIIREG:
5063                 if (!netif_running(dev))
5064                         return -EAGAIN;
5065
5066                 return 0;
5067
5068         default:
5069                 /* do nothing */
5070                 break;
5071         }
5072         return -EOPNOTSUPP;
5073 }
5074
5075 static struct rtnl_link_stats64 *
5076 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
5077 {
5078         u32 i;
5079         struct bnxt *bp = netdev_priv(dev);
5080
5081         memset(stats, 0, sizeof(struct rtnl_link_stats64));
5082
5083         if (!bp->bnapi)
5084                 return stats;
5085
5086         /* TODO check if we need to synchronize with bnxt_close path */
5087         for (i = 0; i < bp->cp_nr_rings; i++) {
5088                 struct bnxt_napi *bnapi = bp->bnapi[i];
5089                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5090                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
5091
5092                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
5093                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
5094                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
5095
5096                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
5097                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
5098                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
5099
5100                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
5101                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
5102                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
5103
5104                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
5105                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
5106                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
5107
5108                 stats->rx_missed_errors +=
5109                         le64_to_cpu(hw_stats->rx_discard_pkts);
5110
5111                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
5112
5113                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
5114         }
5115
5116         if (bp->flags & BNXT_FLAG_PORT_STATS) {
5117                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
5118                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
5119
5120                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
5121                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
5122                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
5123                                           le64_to_cpu(rx->rx_ovrsz_frames) +
5124                                           le64_to_cpu(rx->rx_runt_frames);
5125                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
5126                                    le64_to_cpu(rx->rx_jbr_frames);
5127                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
5128                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
5129                 stats->tx_errors = le64_to_cpu(tx->tx_err);
5130         }
5131
5132         return stats;
5133 }
5134
5135 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
5136 {
5137         struct net_device *dev = bp->dev;
5138         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5139         struct netdev_hw_addr *ha;
5140         u8 *haddr;
5141         int mc_count = 0;
5142         bool update = false;
5143         int off = 0;
5144
5145         netdev_for_each_mc_addr(ha, dev) {
5146                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
5147                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5148                         vnic->mc_list_count = 0;
5149                         return false;
5150                 }
5151                 haddr = ha->addr;
5152                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
5153                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
5154                         update = true;
5155                 }
5156                 off += ETH_ALEN;
5157                 mc_count++;
5158         }
5159         if (mc_count)
5160                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
5161
5162         if (mc_count != vnic->mc_list_count) {
5163                 vnic->mc_list_count = mc_count;
5164                 update = true;
5165         }
5166         return update;
5167 }
5168
5169 static bool bnxt_uc_list_updated(struct bnxt *bp)
5170 {
5171         struct net_device *dev = bp->dev;
5172         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5173         struct netdev_hw_addr *ha;
5174         int off = 0;
5175
5176         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
5177                 return true;
5178
5179         netdev_for_each_uc_addr(ha, dev) {
5180                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
5181                         return true;
5182
5183                 off += ETH_ALEN;
5184         }
5185         return false;
5186 }
5187
5188 static void bnxt_set_rx_mode(struct net_device *dev)
5189 {
5190         struct bnxt *bp = netdev_priv(dev);
5191         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5192         u32 mask = vnic->rx_mask;
5193         bool mc_update = false;
5194         bool uc_update;
5195
5196         if (!netif_running(dev))
5197                 return;
5198
5199         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
5200                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
5201                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
5202
5203         /* Only allow PF to be in promiscuous mode */
5204         if ((dev->flags & IFF_PROMISC) && BNXT_PF(bp))
5205                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5206
5207         uc_update = bnxt_uc_list_updated(bp);
5208
5209         if (dev->flags & IFF_ALLMULTI) {
5210                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5211                 vnic->mc_list_count = 0;
5212         } else {
5213                 mc_update = bnxt_mc_list_updated(bp, &mask);
5214         }
5215
5216         if (mask != vnic->rx_mask || uc_update || mc_update) {
5217                 vnic->rx_mask = mask;
5218
5219                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
5220                 schedule_work(&bp->sp_task);
5221         }
5222 }
5223
5224 static int bnxt_cfg_rx_mode(struct bnxt *bp)
5225 {
5226         struct net_device *dev = bp->dev;
5227         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5228         struct netdev_hw_addr *ha;
5229         int i, off = 0, rc;
5230         bool uc_update;
5231
5232         netif_addr_lock_bh(dev);
5233         uc_update = bnxt_uc_list_updated(bp);
5234         netif_addr_unlock_bh(dev);
5235
5236         if (!uc_update)
5237                 goto skip_uc;
5238
5239         mutex_lock(&bp->hwrm_cmd_lock);
5240         for (i = 1; i < vnic->uc_filter_count; i++) {
5241                 struct hwrm_cfa_l2_filter_free_input req = {0};
5242
5243                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
5244                                        -1);
5245
5246                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
5247
5248                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5249                                         HWRM_CMD_TIMEOUT);
5250         }
5251         mutex_unlock(&bp->hwrm_cmd_lock);
5252
5253         vnic->uc_filter_count = 1;
5254
5255         netif_addr_lock_bh(dev);
5256         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
5257                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5258         } else {
5259                 netdev_for_each_uc_addr(ha, dev) {
5260                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
5261                         off += ETH_ALEN;
5262                         vnic->uc_filter_count++;
5263                 }
5264         }
5265         netif_addr_unlock_bh(dev);
5266
5267         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
5268                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
5269                 if (rc) {
5270                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
5271                                    rc);
5272                         vnic->uc_filter_count = i;
5273                         return rc;
5274                 }
5275         }
5276
5277 skip_uc:
5278         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
5279         if (rc)
5280                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
5281                            rc);
5282
5283         return rc;
5284 }
5285
5286 static bool bnxt_rfs_capable(struct bnxt *bp)
5287 {
5288 #ifdef CONFIG_RFS_ACCEL
5289         struct bnxt_pf_info *pf = &bp->pf;
5290         int vnics;
5291
5292         if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP))
5293                 return false;
5294
5295         vnics = 1 + bp->rx_nr_rings;
5296         if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics)
5297                 return false;
5298
5299         return true;
5300 #else
5301         return false;
5302 #endif
5303 }
5304
5305 static netdev_features_t bnxt_fix_features(struct net_device *dev,
5306                                            netdev_features_t features)
5307 {
5308         struct bnxt *bp = netdev_priv(dev);
5309
5310         if (!bnxt_rfs_capable(bp))
5311                 features &= ~NETIF_F_NTUPLE;
5312         return features;
5313 }
5314
5315 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
5316 {
5317         struct bnxt *bp = netdev_priv(dev);
5318         u32 flags = bp->flags;
5319         u32 changes;
5320         int rc = 0;
5321         bool re_init = false;
5322         bool update_tpa = false;
5323
5324         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
5325         if ((features & NETIF_F_GRO) && (bp->pdev->revision > 0))
5326                 flags |= BNXT_FLAG_GRO;
5327         if (features & NETIF_F_LRO)
5328                 flags |= BNXT_FLAG_LRO;
5329
5330         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5331                 flags |= BNXT_FLAG_STRIP_VLAN;
5332
5333         if (features & NETIF_F_NTUPLE)
5334                 flags |= BNXT_FLAG_RFS;
5335
5336         changes = flags ^ bp->flags;
5337         if (changes & BNXT_FLAG_TPA) {
5338                 update_tpa = true;
5339                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
5340                     (flags & BNXT_FLAG_TPA) == 0)
5341                         re_init = true;
5342         }
5343
5344         if (changes & ~BNXT_FLAG_TPA)
5345                 re_init = true;
5346
5347         if (flags != bp->flags) {
5348                 u32 old_flags = bp->flags;
5349
5350                 bp->flags = flags;
5351
5352                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5353                         if (update_tpa)
5354                                 bnxt_set_ring_params(bp);
5355                         return rc;
5356                 }
5357
5358                 if (re_init) {
5359                         bnxt_close_nic(bp, false, false);
5360                         if (update_tpa)
5361                                 bnxt_set_ring_params(bp);
5362
5363                         return bnxt_open_nic(bp, false, false);
5364                 }
5365                 if (update_tpa) {
5366                         rc = bnxt_set_tpa(bp,
5367                                           (flags & BNXT_FLAG_TPA) ?
5368                                           true : false);
5369                         if (rc)
5370                                 bp->flags = old_flags;
5371                 }
5372         }
5373         return rc;
5374 }
5375
5376 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
5377 {
5378         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
5379         int i = bnapi->index;
5380
5381         if (!txr)
5382                 return;
5383
5384         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
5385                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
5386                     txr->tx_cons);
5387 }
5388
5389 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
5390 {
5391         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
5392         int i = bnapi->index;
5393
5394         if (!rxr)
5395                 return;
5396
5397         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
5398                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
5399                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
5400                     rxr->rx_sw_agg_prod);
5401 }
5402
5403 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
5404 {
5405         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5406         int i = bnapi->index;
5407
5408         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
5409                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
5410 }
5411
5412 static void bnxt_dbg_dump_states(struct bnxt *bp)
5413 {
5414         int i;
5415         struct bnxt_napi *bnapi;
5416
5417         for (i = 0; i < bp->cp_nr_rings; i++) {
5418                 bnapi = bp->bnapi[i];
5419                 if (netif_msg_drv(bp)) {
5420                         bnxt_dump_tx_sw_state(bnapi);
5421                         bnxt_dump_rx_sw_state(bnapi);
5422                         bnxt_dump_cp_sw_state(bnapi);
5423                 }
5424         }
5425 }
5426
5427 static void bnxt_reset_task(struct bnxt *bp)
5428 {
5429         bnxt_dbg_dump_states(bp);
5430         if (netif_running(bp->dev)) {
5431                 bnxt_close_nic(bp, false, false);
5432                 bnxt_open_nic(bp, false, false);
5433         }
5434 }
5435
5436 static void bnxt_tx_timeout(struct net_device *dev)
5437 {
5438         struct bnxt *bp = netdev_priv(dev);
5439
5440         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
5441         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
5442         schedule_work(&bp->sp_task);
5443 }
5444
5445 #ifdef CONFIG_NET_POLL_CONTROLLER
5446 static void bnxt_poll_controller(struct net_device *dev)
5447 {
5448         struct bnxt *bp = netdev_priv(dev);
5449         int i;
5450
5451         for (i = 0; i < bp->cp_nr_rings; i++) {
5452                 struct bnxt_irq *irq = &bp->irq_tbl[i];
5453
5454                 disable_irq(irq->vector);
5455                 irq->handler(irq->vector, bp->bnapi[i]);
5456                 enable_irq(irq->vector);
5457         }
5458 }
5459 #endif
5460
5461 static void bnxt_timer(unsigned long data)
5462 {
5463         struct bnxt *bp = (struct bnxt *)data;
5464         struct net_device *dev = bp->dev;
5465
5466         if (!netif_running(dev))
5467                 return;
5468
5469         if (atomic_read(&bp->intr_sem) != 0)
5470                 goto bnxt_restart_timer;
5471
5472         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
5473                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
5474                 schedule_work(&bp->sp_task);
5475         }
5476 bnxt_restart_timer:
5477         mod_timer(&bp->timer, jiffies + bp->current_interval);
5478 }
5479
5480 static void bnxt_port_module_event(struct bnxt *bp)
5481 {
5482         struct bnxt_link_info *link_info = &bp->link_info;
5483         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
5484
5485         if (bnxt_update_link(bp, true))
5486                 return;
5487
5488         if (link_info->last_port_module_event != 0) {
5489                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
5490                             bp->pf.port_id);
5491                 if (bp->hwrm_spec_code >= 0x10201) {
5492                         netdev_warn(bp->dev, "Module part number %s\n",
5493                                     resp->phy_vendor_partnumber);
5494                 }
5495         }
5496         if (link_info->last_port_module_event == 1)
5497                 netdev_warn(bp->dev, "TX is disabled\n");
5498         if (link_info->last_port_module_event == 3)
5499                 netdev_warn(bp->dev, "Shutdown SFP+ module\n");
5500 }
5501
5502 static void bnxt_cfg_ntp_filters(struct bnxt *);
5503
5504 static void bnxt_sp_task(struct work_struct *work)
5505 {
5506         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
5507         int rc;
5508
5509         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5510         smp_mb__after_atomic();
5511         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5512                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5513                 return;
5514         }
5515
5516         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
5517                 bnxt_cfg_rx_mode(bp);
5518
5519         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
5520                 bnxt_cfg_ntp_filters(bp);
5521         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
5522                 rc = bnxt_update_link(bp, true);
5523                 if (rc)
5524                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
5525                                    rc);
5526         }
5527         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
5528                 bnxt_hwrm_exec_fwd_req(bp);
5529         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
5530                 bnxt_hwrm_tunnel_dst_port_alloc(
5531                         bp, bp->vxlan_port,
5532                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5533         }
5534         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
5535                 bnxt_hwrm_tunnel_dst_port_free(
5536                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5537         }
5538         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) {
5539                 /* bnxt_reset_task() calls bnxt_close_nic() which waits
5540                  * for BNXT_STATE_IN_SP_TASK to clear.
5541                  */
5542                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5543                 rtnl_lock();
5544                 bnxt_reset_task(bp);
5545                 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5546                 rtnl_unlock();
5547         }
5548
5549         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event))
5550                 bnxt_port_module_event(bp);
5551
5552         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
5553                 bnxt_hwrm_port_qstats(bp);
5554
5555         smp_mb__before_atomic();
5556         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5557 }
5558
5559 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
5560 {
5561         int rc;
5562         struct bnxt *bp = netdev_priv(dev);
5563
5564         SET_NETDEV_DEV(dev, &pdev->dev);
5565
5566         /* enable device (incl. PCI PM wakeup), and bus-mastering */
5567         rc = pci_enable_device(pdev);
5568         if (rc) {
5569                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
5570                 goto init_err;
5571         }
5572
5573         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
5574                 dev_err(&pdev->dev,
5575                         "Cannot find PCI device base address, aborting\n");
5576                 rc = -ENODEV;
5577                 goto init_err_disable;
5578         }
5579
5580         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
5581         if (rc) {
5582                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
5583                 goto init_err_disable;
5584         }
5585
5586         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
5587             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
5588                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
5589                 goto init_err_disable;
5590         }
5591
5592         pci_set_master(pdev);
5593
5594         bp->dev = dev;
5595         bp->pdev = pdev;
5596
5597         bp->bar0 = pci_ioremap_bar(pdev, 0);
5598         if (!bp->bar0) {
5599                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
5600                 rc = -ENOMEM;
5601                 goto init_err_release;
5602         }
5603
5604         bp->bar1 = pci_ioremap_bar(pdev, 2);
5605         if (!bp->bar1) {
5606                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
5607                 rc = -ENOMEM;
5608                 goto init_err_release;
5609         }
5610
5611         bp->bar2 = pci_ioremap_bar(pdev, 4);
5612         if (!bp->bar2) {
5613                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
5614                 rc = -ENOMEM;
5615                 goto init_err_release;
5616         }
5617
5618         pci_enable_pcie_error_reporting(pdev);
5619
5620         INIT_WORK(&bp->sp_task, bnxt_sp_task);
5621
5622         spin_lock_init(&bp->ntp_fltr_lock);
5623
5624         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
5625         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
5626
5627         /* tick values in micro seconds */
5628         bp->rx_coal_ticks = 12;
5629         bp->rx_coal_bufs = 30;
5630         bp->rx_coal_ticks_irq = 1;
5631         bp->rx_coal_bufs_irq = 2;
5632
5633         bp->tx_coal_ticks = 25;
5634         bp->tx_coal_bufs = 30;
5635         bp->tx_coal_ticks_irq = 2;
5636         bp->tx_coal_bufs_irq = 2;
5637
5638         init_timer(&bp->timer);
5639         bp->timer.data = (unsigned long)bp;
5640         bp->timer.function = bnxt_timer;
5641         bp->current_interval = BNXT_TIMER_INTERVAL;
5642
5643         clear_bit(BNXT_STATE_OPEN, &bp->state);
5644
5645         return 0;
5646
5647 init_err_release:
5648         if (bp->bar2) {
5649                 pci_iounmap(pdev, bp->bar2);
5650                 bp->bar2 = NULL;
5651         }
5652
5653         if (bp->bar1) {
5654                 pci_iounmap(pdev, bp->bar1);
5655                 bp->bar1 = NULL;
5656         }
5657
5658         if (bp->bar0) {
5659                 pci_iounmap(pdev, bp->bar0);
5660                 bp->bar0 = NULL;
5661         }
5662
5663         pci_release_regions(pdev);
5664
5665 init_err_disable:
5666         pci_disable_device(pdev);
5667
5668 init_err:
5669         return rc;
5670 }
5671
5672 /* rtnl_lock held */
5673 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
5674 {
5675         struct sockaddr *addr = p;
5676         struct bnxt *bp = netdev_priv(dev);
5677         int rc = 0;
5678
5679         if (!is_valid_ether_addr(addr->sa_data))
5680                 return -EADDRNOTAVAIL;
5681
5682 #ifdef CONFIG_BNXT_SRIOV
5683         if (BNXT_VF(bp) && is_valid_ether_addr(bp->vf.mac_addr))
5684                 return -EADDRNOTAVAIL;
5685 #endif
5686
5687         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
5688                 return 0;
5689
5690         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5691         if (netif_running(dev)) {
5692                 bnxt_close_nic(bp, false, false);
5693                 rc = bnxt_open_nic(bp, false, false);
5694         }
5695
5696         return rc;
5697 }
5698
5699 /* rtnl_lock held */
5700 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
5701 {
5702         struct bnxt *bp = netdev_priv(dev);
5703
5704         if (new_mtu < 60 || new_mtu > 9000)
5705                 return -EINVAL;
5706
5707         if (netif_running(dev))
5708                 bnxt_close_nic(bp, false, false);
5709
5710         dev->mtu = new_mtu;
5711         bnxt_set_ring_params(bp);
5712
5713         if (netif_running(dev))
5714                 return bnxt_open_nic(bp, false, false);
5715
5716         return 0;
5717 }
5718
5719 static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
5720                          struct tc_to_netdev *ntc)
5721 {
5722         struct bnxt *bp = netdev_priv(dev);
5723         u8 tc;
5724
5725         if (ntc->type != TC_SETUP_MQPRIO)
5726                 return -EINVAL;
5727
5728         tc = ntc->tc;
5729
5730         if (tc > bp->max_tc) {
5731                 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n",
5732                            tc, bp->max_tc);
5733                 return -EINVAL;
5734         }
5735
5736         if (netdev_get_num_tc(dev) == tc)
5737                 return 0;
5738
5739         if (tc) {
5740                 int max_rx_rings, max_tx_rings, rc;
5741                 bool sh = false;
5742
5743                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5744                         sh = true;
5745
5746                 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
5747                 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings)
5748                         return -ENOMEM;
5749         }
5750
5751         /* Needs to close the device and do hw resource re-allocations */
5752         if (netif_running(bp->dev))
5753                 bnxt_close_nic(bp, true, false);
5754
5755         if (tc) {
5756                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
5757                 netdev_set_num_tc(dev, tc);
5758         } else {
5759                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
5760                 netdev_reset_tc(dev);
5761         }
5762         bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
5763         bp->num_stat_ctxs = bp->cp_nr_rings;
5764
5765         if (netif_running(bp->dev))
5766                 return bnxt_open_nic(bp, true, false);
5767
5768         return 0;
5769 }
5770
5771 #ifdef CONFIG_RFS_ACCEL
5772 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
5773                             struct bnxt_ntuple_filter *f2)
5774 {
5775         struct flow_keys *keys1 = &f1->fkeys;
5776         struct flow_keys *keys2 = &f2->fkeys;
5777
5778         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
5779             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
5780             keys1->ports.ports == keys2->ports.ports &&
5781             keys1->basic.ip_proto == keys2->basic.ip_proto &&
5782             keys1->basic.n_proto == keys2->basic.n_proto &&
5783             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr))
5784                 return true;
5785
5786         return false;
5787 }
5788
5789 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
5790                               u16 rxq_index, u32 flow_id)
5791 {
5792         struct bnxt *bp = netdev_priv(dev);
5793         struct bnxt_ntuple_filter *fltr, *new_fltr;
5794         struct flow_keys *fkeys;
5795         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
5796         int rc = 0, idx, bit_id;
5797         struct hlist_head *head;
5798
5799         if (skb->encapsulation)
5800                 return -EPROTONOSUPPORT;
5801
5802         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
5803         if (!new_fltr)
5804                 return -ENOMEM;
5805
5806         fkeys = &new_fltr->fkeys;
5807         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
5808                 rc = -EPROTONOSUPPORT;
5809                 goto err_free;
5810         }
5811
5812         if ((fkeys->basic.n_proto != htons(ETH_P_IP)) ||
5813             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
5814              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
5815                 rc = -EPROTONOSUPPORT;
5816                 goto err_free;
5817         }
5818
5819         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
5820
5821         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
5822         head = &bp->ntp_fltr_hash_tbl[idx];
5823         rcu_read_lock();
5824         hlist_for_each_entry_rcu(fltr, head, hash) {
5825                 if (bnxt_fltr_match(fltr, new_fltr)) {
5826                         rcu_read_unlock();
5827                         rc = 0;
5828                         goto err_free;
5829                 }
5830         }
5831         rcu_read_unlock();
5832
5833         spin_lock_bh(&bp->ntp_fltr_lock);
5834         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5835                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
5836         if (bit_id < 0) {
5837                 spin_unlock_bh(&bp->ntp_fltr_lock);
5838                 rc = -ENOMEM;
5839                 goto err_free;
5840         }
5841
5842         new_fltr->sw_id = (u16)bit_id;
5843         new_fltr->flow_id = flow_id;
5844         new_fltr->rxq = rxq_index;
5845         hlist_add_head_rcu(&new_fltr->hash, head);
5846         bp->ntp_fltr_count++;
5847         spin_unlock_bh(&bp->ntp_fltr_lock);
5848
5849         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
5850         schedule_work(&bp->sp_task);
5851
5852         return new_fltr->sw_id;
5853
5854 err_free:
5855         kfree(new_fltr);
5856         return rc;
5857 }
5858
5859 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
5860 {
5861         int i;
5862
5863         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
5864                 struct hlist_head *head;
5865                 struct hlist_node *tmp;
5866                 struct bnxt_ntuple_filter *fltr;
5867                 int rc;
5868
5869                 head = &bp->ntp_fltr_hash_tbl[i];
5870                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
5871                         bool del = false;
5872
5873                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
5874                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
5875                                                         fltr->flow_id,
5876                                                         fltr->sw_id)) {
5877                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
5878                                                                          fltr);
5879                                         del = true;
5880                                 }
5881                         } else {
5882                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
5883                                                                        fltr);
5884                                 if (rc)
5885                                         del = true;
5886                                 else
5887                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
5888                         }
5889
5890                         if (del) {
5891                                 spin_lock_bh(&bp->ntp_fltr_lock);
5892                                 hlist_del_rcu(&fltr->hash);
5893                                 bp->ntp_fltr_count--;
5894                                 spin_unlock_bh(&bp->ntp_fltr_lock);
5895                                 synchronize_rcu();
5896                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
5897                                 kfree(fltr);
5898                         }
5899                 }
5900         }
5901         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
5902                 netdev_info(bp->dev, "Receive PF driver unload event!");
5903 }
5904
5905 #else
5906
5907 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
5908 {
5909 }
5910
5911 #endif /* CONFIG_RFS_ACCEL */
5912
5913 static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
5914                                 __be16 port)
5915 {
5916         struct bnxt *bp = netdev_priv(dev);
5917
5918         if (!netif_running(dev))
5919                 return;
5920
5921         if (sa_family != AF_INET6 && sa_family != AF_INET)
5922                 return;
5923
5924         if (bp->vxlan_port_cnt && bp->vxlan_port != port)
5925                 return;
5926
5927         bp->vxlan_port_cnt++;
5928         if (bp->vxlan_port_cnt == 1) {
5929                 bp->vxlan_port = port;
5930                 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
5931                 schedule_work(&bp->sp_task);
5932         }
5933 }
5934
5935 static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
5936                                 __be16 port)
5937 {
5938         struct bnxt *bp = netdev_priv(dev);
5939
5940         if (!netif_running(dev))
5941                 return;
5942
5943         if (sa_family != AF_INET6 && sa_family != AF_INET)
5944                 return;
5945
5946         if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
5947                 bp->vxlan_port_cnt--;
5948
5949                 if (bp->vxlan_port_cnt == 0) {
5950                         set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
5951                         schedule_work(&bp->sp_task);
5952                 }
5953         }
5954 }
5955
5956 static const struct net_device_ops bnxt_netdev_ops = {
5957         .ndo_open               = bnxt_open,
5958         .ndo_start_xmit         = bnxt_start_xmit,
5959         .ndo_stop               = bnxt_close,
5960         .ndo_get_stats64        = bnxt_get_stats64,
5961         .ndo_set_rx_mode        = bnxt_set_rx_mode,
5962         .ndo_do_ioctl           = bnxt_ioctl,
5963         .ndo_validate_addr      = eth_validate_addr,
5964         .ndo_set_mac_address    = bnxt_change_mac_addr,
5965         .ndo_change_mtu         = bnxt_change_mtu,
5966         .ndo_fix_features       = bnxt_fix_features,
5967         .ndo_set_features       = bnxt_set_features,
5968         .ndo_tx_timeout         = bnxt_tx_timeout,
5969 #ifdef CONFIG_BNXT_SRIOV
5970         .ndo_get_vf_config      = bnxt_get_vf_config,
5971         .ndo_set_vf_mac         = bnxt_set_vf_mac,
5972         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
5973         .ndo_set_vf_rate        = bnxt_set_vf_bw,
5974         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
5975         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
5976 #endif
5977 #ifdef CONFIG_NET_POLL_CONTROLLER
5978         .ndo_poll_controller    = bnxt_poll_controller,
5979 #endif
5980         .ndo_setup_tc           = bnxt_setup_tc,
5981 #ifdef CONFIG_RFS_ACCEL
5982         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
5983 #endif
5984         .ndo_add_vxlan_port     = bnxt_add_vxlan_port,
5985         .ndo_del_vxlan_port     = bnxt_del_vxlan_port,
5986 #ifdef CONFIG_NET_RX_BUSY_POLL
5987         .ndo_busy_poll          = bnxt_busy_poll,
5988 #endif
5989 };
5990
5991 static void bnxt_remove_one(struct pci_dev *pdev)
5992 {
5993         struct net_device *dev = pci_get_drvdata(pdev);
5994         struct bnxt *bp = netdev_priv(dev);
5995
5996         if (BNXT_PF(bp))
5997                 bnxt_sriov_disable(bp);
5998
5999         pci_disable_pcie_error_reporting(pdev);
6000         unregister_netdev(dev);
6001         cancel_work_sync(&bp->sp_task);
6002         bp->sp_event = 0;
6003
6004         bnxt_hwrm_func_drv_unrgtr(bp);
6005         bnxt_free_hwrm_resources(bp);
6006         pci_iounmap(pdev, bp->bar2);
6007         pci_iounmap(pdev, bp->bar1);
6008         pci_iounmap(pdev, bp->bar0);
6009         free_netdev(dev);
6010
6011         pci_release_regions(pdev);
6012         pci_disable_device(pdev);
6013 }
6014
6015 static int bnxt_probe_phy(struct bnxt *bp)
6016 {
6017         int rc = 0;
6018         struct bnxt_link_info *link_info = &bp->link_info;
6019
6020         rc = bnxt_hwrm_phy_qcaps(bp);
6021         if (rc) {
6022                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
6023                            rc);
6024                 return rc;
6025         }
6026
6027         rc = bnxt_update_link(bp, false);
6028         if (rc) {
6029                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
6030                            rc);
6031                 return rc;
6032         }
6033
6034         /*initialize the ethool setting copy with NVM settings */
6035         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
6036                 link_info->autoneg = BNXT_AUTONEG_SPEED;
6037                 if (bp->hwrm_spec_code >= 0x10201) {
6038                         if (link_info->auto_pause_setting &
6039                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
6040                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6041                 } else {
6042                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6043                 }
6044                 link_info->advertising = link_info->auto_link_speeds;
6045         } else {
6046                 link_info->req_link_speed = link_info->force_link_speed;
6047                 link_info->req_duplex = link_info->duplex_setting;
6048         }
6049         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
6050                 link_info->req_flow_ctrl =
6051                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
6052         else
6053                 link_info->req_flow_ctrl = link_info->force_pause_setting;
6054         return rc;
6055 }
6056
6057 static int bnxt_get_max_irq(struct pci_dev *pdev)
6058 {
6059         u16 ctrl;
6060
6061         if (!pdev->msix_cap)
6062                 return 1;
6063
6064         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
6065         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
6066 }
6067
6068 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
6069                                 int *max_cp)
6070 {
6071         int max_ring_grps = 0;
6072
6073 #ifdef CONFIG_BNXT_SRIOV
6074         if (!BNXT_PF(bp)) {
6075                 *max_tx = bp->vf.max_tx_rings;
6076                 *max_rx = bp->vf.max_rx_rings;
6077                 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
6078                 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
6079                 max_ring_grps = bp->vf.max_hw_ring_grps;
6080         } else
6081 #endif
6082         {
6083                 *max_tx = bp->pf.max_tx_rings;
6084                 *max_rx = bp->pf.max_rx_rings;
6085                 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
6086                 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
6087                 max_ring_grps = bp->pf.max_hw_ring_grps;
6088         }
6089
6090         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6091                 *max_rx >>= 1;
6092         *max_rx = min_t(int, *max_rx, max_ring_grps);
6093 }
6094
6095 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
6096 {
6097         int rx, tx, cp;
6098
6099         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
6100         if (!rx || !tx || !cp)
6101                 return -ENOMEM;
6102
6103         *max_rx = rx;
6104         *max_tx = tx;
6105         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
6106 }
6107
6108 static int bnxt_set_dflt_rings(struct bnxt *bp)
6109 {
6110         int dflt_rings, max_rx_rings, max_tx_rings, rc;
6111         bool sh = true;
6112
6113         if (sh)
6114                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
6115         dflt_rings = netif_get_num_default_rss_queues();
6116         rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6117         if (rc)
6118                 return rc;
6119         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
6120         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
6121         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
6122         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
6123                                bp->tx_nr_rings + bp->rx_nr_rings;
6124         bp->num_stat_ctxs = bp->cp_nr_rings;
6125         return rc;
6126 }
6127
6128 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6129 {
6130         static int version_printed;
6131         struct net_device *dev;
6132         struct bnxt *bp;
6133         int rc, max_irqs;
6134
6135         if (version_printed++ == 0)
6136                 pr_info("%s", version);
6137
6138         max_irqs = bnxt_get_max_irq(pdev);
6139         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
6140         if (!dev)
6141                 return -ENOMEM;
6142
6143         bp = netdev_priv(dev);
6144
6145         if (bnxt_vf_pciid(ent->driver_data))
6146                 bp->flags |= BNXT_FLAG_VF;
6147
6148         if (pdev->msix_cap)
6149                 bp->flags |= BNXT_FLAG_MSIX_CAP;
6150
6151         rc = bnxt_init_board(pdev, dev);
6152         if (rc < 0)
6153                 goto init_err_free;
6154
6155         dev->netdev_ops = &bnxt_netdev_ops;
6156         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
6157         dev->ethtool_ops = &bnxt_ethtool_ops;
6158
6159         pci_set_drvdata(pdev, dev);
6160
6161         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6162                            NETIF_F_TSO | NETIF_F_TSO6 |
6163                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6164                            NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT |
6165                            NETIF_F_RXHASH |
6166                            NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO;
6167
6168         dev->hw_enc_features =
6169                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6170                         NETIF_F_TSO | NETIF_F_TSO6 |
6171                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6172                         NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT;
6173         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
6174         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
6175                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
6176         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
6177         dev->priv_flags |= IFF_UNICAST_FLT;
6178
6179 #ifdef CONFIG_BNXT_SRIOV
6180         init_waitqueue_head(&bp->sriov_cfg_wait);
6181 #endif
6182         rc = bnxt_alloc_hwrm_resources(bp);
6183         if (rc)
6184                 goto init_err;
6185
6186         mutex_init(&bp->hwrm_cmd_lock);
6187         bnxt_hwrm_ver_get(bp);
6188
6189         rc = bnxt_hwrm_func_drv_rgtr(bp);
6190         if (rc)
6191                 goto init_err;
6192
6193         /* Get the MAX capabilities for this function */
6194         rc = bnxt_hwrm_func_qcaps(bp);
6195         if (rc) {
6196                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
6197                            rc);
6198                 rc = -1;
6199                 goto init_err;
6200         }
6201
6202         rc = bnxt_hwrm_queue_qportcfg(bp);
6203         if (rc) {
6204                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
6205                            rc);
6206                 rc = -1;
6207                 goto init_err;
6208         }
6209
6210         bnxt_set_tpa_flags(bp);
6211         bnxt_set_ring_params(bp);
6212         if (BNXT_PF(bp))
6213                 bp->pf.max_irqs = max_irqs;
6214 #if defined(CONFIG_BNXT_SRIOV)
6215         else
6216                 bp->vf.max_irqs = max_irqs;
6217 #endif
6218         bnxt_set_dflt_rings(bp);
6219
6220         if (BNXT_PF(bp)) {
6221                 dev->hw_features |= NETIF_F_NTUPLE;
6222                 if (bnxt_rfs_capable(bp)) {
6223                         bp->flags |= BNXT_FLAG_RFS;
6224                         dev->features |= NETIF_F_NTUPLE;
6225                 }
6226         }
6227
6228         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
6229                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
6230
6231         rc = bnxt_probe_phy(bp);
6232         if (rc)
6233                 goto init_err;
6234
6235         rc = register_netdev(dev);
6236         if (rc)
6237                 goto init_err;
6238
6239         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
6240                     board_info[ent->driver_data].name,
6241                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
6242
6243         return 0;
6244
6245 init_err:
6246         pci_iounmap(pdev, bp->bar0);
6247         pci_release_regions(pdev);
6248         pci_disable_device(pdev);
6249
6250 init_err_free:
6251         free_netdev(dev);
6252         return rc;
6253 }
6254
6255 /**
6256  * bnxt_io_error_detected - called when PCI error is detected
6257  * @pdev: Pointer to PCI device
6258  * @state: The current pci connection state
6259  *
6260  * This function is called after a PCI bus error affecting
6261  * this device has been detected.
6262  */
6263 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
6264                                                pci_channel_state_t state)
6265 {
6266         struct net_device *netdev = pci_get_drvdata(pdev);
6267
6268         netdev_info(netdev, "PCI I/O error detected\n");
6269
6270         rtnl_lock();
6271         netif_device_detach(netdev);
6272
6273         if (state == pci_channel_io_perm_failure) {
6274                 rtnl_unlock();
6275                 return PCI_ERS_RESULT_DISCONNECT;
6276         }
6277
6278         if (netif_running(netdev))
6279                 bnxt_close(netdev);
6280
6281         pci_disable_device(pdev);
6282         rtnl_unlock();
6283
6284         /* Request a slot slot reset. */
6285         return PCI_ERS_RESULT_NEED_RESET;
6286 }
6287
6288 /**
6289  * bnxt_io_slot_reset - called after the pci bus has been reset.
6290  * @pdev: Pointer to PCI device
6291  *
6292  * Restart the card from scratch, as if from a cold-boot.
6293  * At this point, the card has exprienced a hard reset,
6294  * followed by fixups by BIOS, and has its config space
6295  * set up identically to what it was at cold boot.
6296  */
6297 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
6298 {
6299         struct net_device *netdev = pci_get_drvdata(pdev);
6300         struct bnxt *bp = netdev_priv(netdev);
6301         int err = 0;
6302         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
6303
6304         netdev_info(bp->dev, "PCI Slot Reset\n");
6305
6306         rtnl_lock();
6307
6308         if (pci_enable_device(pdev)) {
6309                 dev_err(&pdev->dev,
6310                         "Cannot re-enable PCI device after reset.\n");
6311         } else {
6312                 pci_set_master(pdev);
6313
6314                 if (netif_running(netdev))
6315                         err = bnxt_open(netdev);
6316
6317                 if (!err)
6318                         result = PCI_ERS_RESULT_RECOVERED;
6319         }
6320
6321         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
6322                 dev_close(netdev);
6323
6324         rtnl_unlock();
6325
6326         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6327         if (err) {
6328                 dev_err(&pdev->dev,
6329                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
6330                          err); /* non-fatal, continue */
6331         }
6332
6333         return PCI_ERS_RESULT_RECOVERED;
6334 }
6335
6336 /**
6337  * bnxt_io_resume - called when traffic can start flowing again.
6338  * @pdev: Pointer to PCI device
6339  *
6340  * This callback is called when the error recovery driver tells
6341  * us that its OK to resume normal operation.
6342  */
6343 static void bnxt_io_resume(struct pci_dev *pdev)
6344 {
6345         struct net_device *netdev = pci_get_drvdata(pdev);
6346
6347         rtnl_lock();
6348
6349         netif_device_attach(netdev);
6350
6351         rtnl_unlock();
6352 }
6353
6354 static const struct pci_error_handlers bnxt_err_handler = {
6355         .error_detected = bnxt_io_error_detected,
6356         .slot_reset     = bnxt_io_slot_reset,
6357         .resume         = bnxt_io_resume
6358 };
6359
6360 static struct pci_driver bnxt_pci_driver = {
6361         .name           = DRV_MODULE_NAME,
6362         .id_table       = bnxt_pci_tbl,
6363         .probe          = bnxt_init_one,
6364         .remove         = bnxt_remove_one,
6365         .err_handler    = &bnxt_err_handler,
6366 #if defined(CONFIG_BNXT_SRIOV)
6367         .sriov_configure = bnxt_sriov_configure,
6368 #endif
6369 };
6370
6371 module_pci_driver(bnxt_pci_driver);