drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static DEFINE_PCI_DEVICE_TABLE(be_dev_ids) = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "AXGMAC0",
  85         "AXGMAC1",
  86         "JTAG",
  87         "MPU_INTPEND"
  88 };
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "HOST8",
 113         "HOST9",
 114         "NETC",
 115         "Unknown",
 116         "Unknown",
 117         "Unknown",
 118         "Unknown",
 119         "Unknown",
 120         "Unknown",
 121         "Unknown",
 122         "Unknown"
 123 };
 124
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190         val |= qid & DB_RQ_RING_ID_MASK;
 191         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 192
 193         wmb();
 194         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 195 }
 196
 197 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 198                           u16 posted)
 199 {
 200         u32 val = 0;
 201         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 202         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 203
 204         wmb();
 205         iowrite32(val, adapter->db + txo->db_offset);
 206 }
 207
 208 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 209                          bool arm, bool clear_int, u16 num_popped)
 210 {
 211         u32 val = 0;
 212         val |= qid & DB_EQ_RING_ID_MASK;
 213         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 214
 215         if (adapter->eeh_error)
 216                 return;
 217
 218         if (arm)
 219                 val |= 1 << DB_EQ_REARM_SHIFT;
 220         if (clear_int)
 221                 val |= 1 << DB_EQ_CLR_SHIFT;
 222         val |= 1 << DB_EQ_EVNT_SHIFT;
 223         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 224         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 225 }
 226
 227 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 228 {
 229         u32 val = 0;
 230         val |= qid & DB_CQ_RING_ID_MASK;
 231         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 232                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 233
 234         if (adapter->eeh_error)
 235                 return;
 236
 237         if (arm)
 238                 val |= 1 << DB_CQ_REARM_SHIFT;
 239         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 240         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 241 }
 242
 243 static int be_mac_addr_set(struct net_device *netdev, void *p)
 244 {
 245         struct be_adapter *adapter = netdev_priv(netdev);
 246         struct device *dev = &adapter->pdev->dev;
 247         struct sockaddr *addr = p;
 248         int status;
 249         u8 mac[ETH_ALEN];
 250         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 251
 252         if (!is_valid_ether_addr(addr->sa_data))
 253                 return -EADDRNOTAVAIL;
 254
 255         /* Proceed further only if, User provided MAC is different
 256          * from active MAC
 257          */
 258         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 259                 return 0;
 260
 261         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 262          * privilege or if PF did not provision the new MAC address.
 263          * On BE3, this cmd will always fail if the VF doesn't have the
 264          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 265          * the MAC for the VF.
 266          */
 267         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 268                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 269         if (!status) {
 270                 curr_pmac_id = adapter->pmac_id[0];
 271
 272                 /* Delete the old programmed MAC. This call may fail if the
 273                  * old MAC was already deleted by the PF driver.
 274                  */
 275                 if (adapter->pmac_id[0] != old_pmac_id)
 276                         be_cmd_pmac_del(adapter, adapter->if_handle,
 277                                         old_pmac_id, 0);
 278         }
 279
 280         /* Decide if the new MAC is successfully activated only after
 281          * querying the FW
 282          */
 283         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 284                                        adapter->if_handle, true, 0);
 285         if (status)
 286                 goto err;
 287
 288         /* The MAC change did not happen, either due to lack of privilege
 289          * or PF didn't pre-provision.
 290          */
 291         if (!ether_addr_equal(addr->sa_data, mac)) {
 292                 status = -EPERM;
 293                 goto err;
 294         }
 295
 296         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 297         dev_info(dev, "MAC address changed to %pM\n", mac);
 298         return 0;
 299 err:
 300         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 301         return status;
 302 }
 303
 304 /* BE2 supports only v0 cmd */
 305 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 306 {
 307         if (BE2_chip(adapter)) {
 308                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 309
 310                 return &cmd->hw_stats;
 311         } else if (BE3_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else {
 316                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         }
 320 }
 321
 322 /* BE2 supports only v0 cmd */
 323 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 324 {
 325         if (BE2_chip(adapter)) {
 326                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 327
 328                 return &hw_stats->erx;
 329         } else if (BE3_chip(adapter)) {
 330                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else {
 334                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         }
 338 }
 339
 340 static void populate_be_v0_stats(struct be_adapter *adapter)
 341 {
 342         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 343         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 344         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 345         struct be_port_rxf_stats_v0 *port_stats =
 346                                         &rxf_stats->port[adapter->port_num];
 347         struct be_drv_stats *drvs = &adapter->drv_stats;
 348
 349         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 350         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 351         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 352         drvs->rx_control_frames = port_stats->rx_control_frames;
 353         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 354         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 355         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 356         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 357         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 358         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 359         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 360         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 361         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 362         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 363         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 364         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 365         drvs->rx_dropped_header_too_small =
 366                 port_stats->rx_dropped_header_too_small;
 367         drvs->rx_address_filtered =
 368                                         port_stats->rx_address_filtered +
 369                                         port_stats->rx_vlan_filtered;
 370         drvs->rx_alignment_symbol_errors =
 371                 port_stats->rx_alignment_symbol_errors;
 372
 373         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 374         drvs->tx_controlframes = port_stats->tx_controlframes;
 375
 376         if (adapter->port_num)
 377                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 378         else
 379                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 380         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 381         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 382         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 383         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 384         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 385         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 386         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 387 }
 388
 389 static void populate_be_v1_stats(struct be_adapter *adapter)
 390 {
 391         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 392         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 393         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 394         struct be_port_rxf_stats_v1 *port_stats =
 395                                         &rxf_stats->port[adapter->port_num];
 396         struct be_drv_stats *drvs = &adapter->drv_stats;
 397
 398         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 399         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 400         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 401         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 402         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 403         drvs->rx_control_frames = port_stats->rx_control_frames;
 404         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 405         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 406         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 407         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 408         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 409         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 410         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 411         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 412         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 413         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 414         drvs->rx_dropped_header_too_small =
 415                 port_stats->rx_dropped_header_too_small;
 416         drvs->rx_input_fifo_overflow_drop =
 417                 port_stats->rx_input_fifo_overflow_drop;
 418         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 419         drvs->rx_alignment_symbol_errors =
 420                 port_stats->rx_alignment_symbol_errors;
 421         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 422         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 423         drvs->tx_controlframes = port_stats->tx_controlframes;
 424         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 425         drvs->jabber_events = port_stats->jabber_events;
 426         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 427         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 428         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 429         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 430         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 431         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 432         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 433 }
 434
 435 static void populate_be_v2_stats(struct be_adapter *adapter)
 436 {
 437         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 438         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 439         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 440         struct be_port_rxf_stats_v2 *port_stats =
 441                                         &rxf_stats->port[adapter->port_num];
 442         struct be_drv_stats *drvs = &adapter->drv_stats;
 443
 444         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 445         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 446         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 447         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 448         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 449         drvs->rx_control_frames = port_stats->rx_control_frames;
 450         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 451         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 452         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 453         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 454         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 455         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 456         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 457         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 458         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 459         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 460         drvs->rx_dropped_header_too_small =
 461                 port_stats->rx_dropped_header_too_small;
 462         drvs->rx_input_fifo_overflow_drop =
 463                 port_stats->rx_input_fifo_overflow_drop;
 464         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 465         drvs->rx_alignment_symbol_errors =
 466                 port_stats->rx_alignment_symbol_errors;
 467         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 468         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 469         drvs->tx_controlframes = port_stats->tx_controlframes;
 470         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 471         drvs->jabber_events = port_stats->jabber_events;
 472         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 473         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 474         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 475         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 476         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 477         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 478         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 479         if (be_roce_supported(adapter)) {
 480                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 481                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 482                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 483                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 484                 drvs->roce_drops_payload_len =
 485                         port_stats->roce_drops_payload_len;
 486         }
 487 }
 488
 489 static void populate_lancer_stats(struct be_adapter *adapter)
 490 {
 491
 492         struct be_drv_stats *drvs = &adapter->drv_stats;
 493         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 494
 495         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 496         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 497         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 498         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 499         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 500         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 501         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 502         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 503         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 504         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 505         drvs->rx_dropped_tcp_length =
 506                                 pport_stats->rx_dropped_invalid_tcp_length;
 507         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 508         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 509         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 510         drvs->rx_dropped_header_too_small =
 511                                 pport_stats->rx_dropped_header_too_small;
 512         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 513         drvs->rx_address_filtered =
 514                                         pport_stats->rx_address_filtered +
 515                                         pport_stats->rx_vlan_filtered;
 516         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 517         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 518         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 519         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 520         drvs->jabber_events = pport_stats->rx_jabbers;
 521         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 522         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 523         drvs->rx_drops_too_many_frags =
 524                                 pport_stats->rx_drops_too_many_frags_lo;
 525 }
 526
 527 static void accumulate_16bit_val(u32 *acc, u16 val)
 528 {
 529 #define lo(x)                   (x & 0xFFFF)
 530 #define hi(x)                   (x & 0xFFFF0000)
 531         bool wrapped = val < lo(*acc);
 532         u32 newacc = hi(*acc) + val;
 533
 534         if (wrapped)
 535                 newacc += 65536;
 536         ACCESS_ONCE(*acc) = newacc;
 537 }
 538
 539 static void populate_erx_stats(struct be_adapter *adapter,
 540                                struct be_rx_obj *rxo, u32 erx_stat)
 541 {
 542         if (!BEx_chip(adapter))
 543                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 544         else
 545                 /* below erx HW counter can actually wrap around after
 546                  * 65535. Driver accumulates a 32-bit value
 547                  */
 548                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 549                                      (u16)erx_stat);
 550 }
 551
 552 void be_parse_stats(struct be_adapter *adapter)
 553 {
 554         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 555         struct be_rx_obj *rxo;
 556         int i;
 557         u32 erx_stat;
 558
 559         if (lancer_chip(adapter)) {
 560                 populate_lancer_stats(adapter);
 561         } else {
 562                 if (BE2_chip(adapter))
 563                         populate_be_v0_stats(adapter);
 564                 else if (BE3_chip(adapter))
 565                         /* for BE3 */
 566                         populate_be_v1_stats(adapter);
 567                 else
 568                         populate_be_v2_stats(adapter);
 569
 570                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 571                 for_all_rx_queues(adapter, rxo, i) {
 572                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 573                         populate_erx_stats(adapter, rxo, erx_stat);
 574                 }
 575         }
 576 }
 577
 578 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 579                                                 struct rtnl_link_stats64 *stats)
 580 {
 581         struct be_adapter *adapter = netdev_priv(netdev);
 582         struct be_drv_stats *drvs = &adapter->drv_stats;
 583         struct be_rx_obj *rxo;
 584         struct be_tx_obj *txo;
 585         u64 pkts, bytes;
 586         unsigned int start;
 587         int i;
 588
 589         for_all_rx_queues(adapter, rxo, i) {
 590                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 591                 do {
 592                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 593                         pkts = rx_stats(rxo)->rx_pkts;
 594                         bytes = rx_stats(rxo)->rx_bytes;
 595                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 596                 stats->rx_packets += pkts;
 597                 stats->rx_bytes += bytes;
 598                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 599                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 600                                         rx_stats(rxo)->rx_drops_no_frags;
 601         }
 602
 603         for_all_tx_queues(adapter, txo, i) {
 604                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 605                 do {
 606                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 607                         pkts = tx_stats(txo)->tx_pkts;
 608                         bytes = tx_stats(txo)->tx_bytes;
 609                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 610                 stats->tx_packets += pkts;
 611                 stats->tx_bytes += bytes;
 612         }
 613
 614         /* bad pkts received */
 615         stats->rx_errors = drvs->rx_crc_errors +
 616                 drvs->rx_alignment_symbol_errors +
 617                 drvs->rx_in_range_errors +
 618                 drvs->rx_out_range_errors +
 619                 drvs->rx_frame_too_long +
 620                 drvs->rx_dropped_too_small +
 621                 drvs->rx_dropped_too_short +
 622                 drvs->rx_dropped_header_too_small +
 623                 drvs->rx_dropped_tcp_length +
 624                 drvs->rx_dropped_runt;
 625
 626         /* detailed rx errors */
 627         stats->rx_length_errors = drvs->rx_in_range_errors +
 628                 drvs->rx_out_range_errors +
 629                 drvs->rx_frame_too_long;
 630
 631         stats->rx_crc_errors = drvs->rx_crc_errors;
 632
 633         /* frame alignment errors */
 634         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 635
 636         /* receiver fifo overrun */
 637         /* drops_no_pbuf is no per i/f, it's per BE card */
 638         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 639                                 drvs->rx_input_fifo_overflow_drop +
 640                                 drvs->rx_drops_no_pbuf;
 641         return stats;
 642 }
 643
 644 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 645 {
 646         struct net_device *netdev = adapter->netdev;
 647
 648         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 649                 netif_carrier_off(netdev);
 650                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 651         }
 652
 653         if (link_status)
 654                 netif_carrier_on(netdev);
 655         else
 656                 netif_carrier_off(netdev);
 657 }
 658
 659 static void be_tx_stats_update(struct be_tx_obj *txo,
 660                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 661                                bool stopped)
 662 {
 663         struct be_tx_stats *stats = tx_stats(txo);
 664
 665         u64_stats_update_begin(&stats->sync);
 666         stats->tx_reqs++;
 667         stats->tx_wrbs += wrb_cnt;
 668         stats->tx_bytes += copied;
 669         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 670         if (stopped)
 671                 stats->tx_stops++;
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Determine number of WRB entries needed to xmit data in an skb */
 676 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 677                            bool *dummy)
 678 {
 679         int cnt = (skb->len > skb->data_len);
 680
 681         cnt += skb_shinfo(skb)->nr_frags;
 682
 683         /* to account for hdr wrb */
 684         cnt++;
 685         if (lancer_chip(adapter) || !(cnt & 1)) {
 686                 *dummy = false;
 687         } else {
 688                 /* add a dummy to make it an even num */
 689                 cnt++;
 690                 *dummy = true;
 691         }
 692         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 693         return cnt;
 694 }
 695
 696 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 697 {
 698         wrb->frag_pa_hi = upper_32_bits(addr);
 699         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 700         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 701         wrb->rsvd0 = 0;
 702 }
 703
 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 705                                      struct sk_buff *skb)
 706 {
 707         u8 vlan_prio;
 708         u16 vlan_tag;
 709
 710         vlan_tag = vlan_tx_tag_get(skb);
 711         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 712         /* If vlan priority provided by OS is NOT in available bmap */
 713         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 714                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 715                                 adapter->recommended_prio;
 716
 717         return vlan_tag;
 718 }
 719
 720 /* Used only for IP tunnel packets */
 721 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 722 {
 723         return (inner_ip_hdr(skb)->version == 4) ?
 724                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 725 }
 726
 727 static u16 skb_ip_proto(struct sk_buff *skb)
 728 {
 729         return (ip_hdr(skb)->version == 4) ?
 730                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 734                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 735                          bool skip_hw_vlan)
 736 {
 737         u16 vlan_tag, proto;
 738
 739         memset(hdr, 0, sizeof(*hdr));
 740
 741         AMAP_SET_BITS(struct amap_eth_hdr_wrb, crc, hdr, 1);
 742
 743         if (skb_is_gso(skb)) {
 744                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso, hdr, 1);
 745                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso_mss,
 746                         hdr, skb_shinfo(skb)->gso_size);
 747                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 748                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso6, hdr, 1);
 749         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 750                 if (skb->encapsulation) {
 751                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, ipcs, hdr, 1);
 752                         proto = skb_inner_ip_proto(skb);
 753                 } else {
 754                         proto = skb_ip_proto(skb);
 755                 }
 756                 if (proto == IPPROTO_TCP)
 757                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, tcpcs, hdr, 1);
 758                 else if (proto == IPPROTO_UDP)
 759                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, udpcs, hdr, 1);
 760         }
 761
 762         if (vlan_tx_tag_present(skb)) {
 763                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan, hdr, 1);
 764                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 765                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan_tag, hdr, vlan_tag);
 766         }
 767
 768         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 769         AMAP_SET_BITS(struct amap_eth_hdr_wrb, complete, hdr, !skip_hw_vlan);
 770         AMAP_SET_BITS(struct amap_eth_hdr_wrb, event, hdr, 1);
 771         AMAP_SET_BITS(struct amap_eth_hdr_wrb, num_wrb, hdr, wrb_cnt);
 772         AMAP_SET_BITS(struct amap_eth_hdr_wrb, len, hdr, len);
 773 }
 774
 775 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 776                           bool unmap_single)
 777 {
 778         dma_addr_t dma;
 779
 780         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 781
 782         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 783         if (wrb->frag_len) {
 784                 if (unmap_single)
 785                         dma_unmap_single(dev, dma, wrb->frag_len,
 786                                          DMA_TO_DEVICE);
 787                 else
 788                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 789         }
 790 }
 791
 792 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 793                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 794                         bool skip_hw_vlan)
 795 {
 796         dma_addr_t busaddr;
 797         int i, copied = 0;
 798         struct device *dev = &adapter->pdev->dev;
 799         struct sk_buff *first_skb = skb;
 800         struct be_eth_wrb *wrb;
 801         struct be_eth_hdr_wrb *hdr;
 802         bool map_single = false;
 803         u16 map_head;
 804
 805         hdr = queue_head_node(txq);
 806         queue_head_inc(txq);
 807         map_head = txq->head;
 808
 809         if (skb->len > skb->data_len) {
 810                 int len = skb_headlen(skb);
 811                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 812                 if (dma_mapping_error(dev, busaddr))
 813                         goto dma_err;
 814                 map_single = true;
 815                 wrb = queue_head_node(txq);
 816                 wrb_fill(wrb, busaddr, len);
 817                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 818                 queue_head_inc(txq);
 819                 copied += len;
 820         }
 821
 822         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 823                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 824                 busaddr = skb_frag_dma_map(dev, frag, 0,
 825                                            skb_frag_size(frag), DMA_TO_DEVICE);
 826                 if (dma_mapping_error(dev, busaddr))
 827                         goto dma_err;
 828                 wrb = queue_head_node(txq);
 829                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 830                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 831                 queue_head_inc(txq);
 832                 copied += skb_frag_size(frag);
 833         }
 834
 835         if (dummy_wrb) {
 836                 wrb = queue_head_node(txq);
 837                 wrb_fill(wrb, 0, 0);
 838                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 839                 queue_head_inc(txq);
 840         }
 841
 842         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 843         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 844
 845         return copied;
 846 dma_err:
 847         txq->head = map_head;
 848         while (copied) {
 849                 wrb = queue_head_node(txq);
 850                 unmap_tx_frag(dev, wrb, map_single);
 851                 map_single = false;
 852                 copied -= wrb->frag_len;
 853                 queue_head_inc(txq);
 854         }
 855         return 0;
 856 }
 857
 858 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 859                                              struct sk_buff *skb,
 860                                              bool *skip_hw_vlan)
 861 {
 862         u16 vlan_tag = 0;
 863
 864         skb = skb_share_check(skb, GFP_ATOMIC);
 865         if (unlikely(!skb))
 866                 return skb;
 867
 868         if (vlan_tx_tag_present(skb))
 869                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 870
 871         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 872                 if (!vlan_tag)
 873                         vlan_tag = adapter->pvid;
 874                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 875                  * skip VLAN insertion
 876                  */
 877                 if (skip_hw_vlan)
 878                         *skip_hw_vlan = true;
 879         }
 880
 881         if (vlan_tag) {
 882                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 if (skip_hw_vlan)
 895                         *skip_hw_vlan = true;
 896         }
 897
 898         return skb;
 899 }
 900
 901 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 902 {
 903         struct ethhdr *eh = (struct ethhdr *)skb->data;
 904         u16 offset = ETH_HLEN;
 905
 906         if (eh->h_proto == htons(ETH_P_IPV6)) {
 907                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 908
 909                 offset += sizeof(struct ipv6hdr);
 910                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 911                     ip6h->nexthdr != NEXTHDR_UDP) {
 912                         struct ipv6_opt_hdr *ehdr =
 913                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 914
 915                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 916                         if (ehdr->hdrlen == 0xff)
 917                                 return true;
 918                 }
 919         }
 920         return false;
 921 }
 922
 923 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 924 {
 925         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 926 }
 927
 928 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 929 {
 930         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 931 }
 932
 933 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 934                                                   struct sk_buff *skb,
 935                                                   bool *skip_hw_vlan)
 936 {
 937         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 938         unsigned int eth_hdr_len;
 939         struct iphdr *ip;
 940
 941         /* For padded packets, BE HW modifies tot_len field in IP header
 942          * incorrecly when VLAN tag is inserted by HW.
 943          * For padded packets, Lancer computes incorrect checksum.
 944          */
 945         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 946                                                 VLAN_ETH_HLEN : ETH_HLEN;
 947         if (skb->len <= 60 &&
 948             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 949             is_ipv4_pkt(skb)) {
 950                 ip = (struct iphdr *)ip_hdr(skb);
 951                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 952         }
 953
 954         /* If vlan tag is already inlined in the packet, skip HW VLAN
 955          * tagging in pvid-tagging mode
 956          */
 957         if (be_pvid_tagging_enabled(adapter) &&
 958             veh->h_vlan_proto == htons(ETH_P_8021Q))
 959                 *skip_hw_vlan = true;
 960
 961         /* HW has a bug wherein it will calculate CSUM for VLAN
 962          * pkts even though it is disabled.
 963          * Manually insert VLAN in pkt.
 964          */
 965         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 966             vlan_tx_tag_present(skb)) {
 967                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 968                 if (unlikely(!skb))
 969                         goto err;
 970         }
 971
 972         /* HW may lockup when VLAN HW tagging is requested on
 973          * certain ipv6 packets. Drop such pkts if the HW workaround to
 974          * skip HW tagging is not enabled by FW.
 975          */
 976         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 977             (adapter->pvid || adapter->qnq_vid) &&
 978             !qnq_async_evt_rcvd(adapter)))
 979                 goto tx_drop;
 980
 981         /* Manual VLAN tag insertion to prevent:
 982          * ASIC lockup when the ASIC inserts VLAN tag into
 983          * certain ipv6 packets. Insert VLAN tags in driver,
 984          * and set event, completion, vlan bits accordingly
 985          * in the Tx WRB.
 986          */
 987         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 988             be_vlan_tag_tx_chk(adapter, skb)) {
 989                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 990                 if (unlikely(!skb))
 991                         goto err;
 992         }
 993
 994         return skb;
 995 tx_drop:
 996         dev_kfree_skb_any(skb);
 997 err:
 998         return NULL;
 999 }
1000
1001 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1002                                            struct sk_buff *skb,
1003                                            bool *skip_hw_vlan)
1004 {
1005         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1006          * less may cause a transmit stall on that port. So the work-around is
1007          * to pad short packets (<= 32 bytes) to a 36-byte length.
1008          */
1009         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1010                 if (skb_padto(skb, 36))
1011                         return NULL;
1012                 skb->len = 36;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1025 {
1026         struct be_adapter *adapter = netdev_priv(netdev);
1027         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1028         struct be_queue_info *txq = &txo->q;
1029         bool dummy_wrb, stopped = false;
1030         u32 wrb_cnt = 0, copied = 0;
1031         bool skip_hw_vlan = false;
1032         u32 start = txq->head;
1033
1034         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1035         if (!skb) {
1036                 tx_stats(txo)->tx_drv_drops++;
1037                 return NETDEV_TX_OK;
1038         }
1039
1040         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1041
1042         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1043                               skip_hw_vlan);
1044         if (copied) {
1045                 int gso_segs = skb_shinfo(skb)->gso_segs;
1046
1047                 /* record the sent skb in the sent_skb table */
1048                 BUG_ON(txo->sent_skb_list[start]);
1049                 txo->sent_skb_list[start] = skb;
1050
1051                 /* Ensure txq has space for the next skb; Else stop the queue
1052                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1053                  * tx compls of the current transmit which'll wake up the queue
1054                  */
1055                 atomic_add(wrb_cnt, &txq->used);
1056                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1057                                                                 txq->len) {
1058                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1059                         stopped = true;
1060                 }
1061
1062                 be_txq_notify(adapter, txo, wrb_cnt);
1063
1064                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1065         } else {
1066                 txq->head = start;
1067                 tx_stats(txo)->tx_drv_drops++;
1068                 dev_kfree_skb_any(skb);
1069         }
1070         return NETDEV_TX_OK;
1071 }
1072
1073 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1074 {
1075         struct be_adapter *adapter = netdev_priv(netdev);
1076         if (new_mtu < BE_MIN_MTU ||
1077             new_mtu > (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN))) {
1078                 dev_info(&adapter->pdev->dev,
1079                          "MTU must be between %d and %d bytes\n",
1080                          BE_MIN_MTU,
1081                          (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN)));
1082                 return -EINVAL;
1083         }
1084         dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n",
1085                  netdev->mtu, new_mtu);
1086         netdev->mtu = new_mtu;
1087         return 0;
1088 }
1089
1090 /*
1091  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1092  * If the user configures more, place BE in vlan promiscuous mode.
1093  */
1094 static int be_vid_config(struct be_adapter *adapter)
1095 {
1096         u16 vids[BE_NUM_VLANS_SUPPORTED];
1097         u16 num = 0, i = 0;
1098         int status = 0;
1099
1100         /* No need to further configure vids if in promiscuous mode */
1101         if (adapter->promiscuous)
1102                 return 0;
1103
1104         if (adapter->vlans_added > be_max_vlans(adapter))
1105                 goto set_vlan_promisc;
1106
1107         /* Construct VLAN Table to give to HW */
1108         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1109                 vids[num++] = cpu_to_le16(i);
1110
1111         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1112         if (status) {
1113                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1114                 if (addl_status(status) ==
1115                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1116                         goto set_vlan_promisc;
1117                 dev_err(&adapter->pdev->dev,
1118                         "Setting HW VLAN filtering failed.\n");
1119         } else {
1120                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1121                         /* hw VLAN filtering re-enabled. */
1122                         status = be_cmd_rx_filter(adapter,
1123                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1124                         if (!status) {
1125                                 dev_info(&adapter->pdev->dev,
1126                                          "Disabling VLAN Promiscuous mode.\n");
1127                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1128                         }
1129                 }
1130         }
1131
1132         return status;
1133
1134 set_vlan_promisc:
1135         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1136                 return 0;
1137
1138         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1139         if (!status) {
1140                 dev_info(&adapter->pdev->dev, "Enable VLAN Promiscuous mode\n");
1141                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1142         } else
1143                 dev_err(&adapter->pdev->dev,
1144                         "Failed to enable VLAN Promiscuous mode.\n");
1145         return status;
1146 }
1147
1148 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1149 {
1150         struct be_adapter *adapter = netdev_priv(netdev);
1151         int status = 0;
1152
1153         /* Packets with VID 0 are always received by Lancer by default */
1154         if (lancer_chip(adapter) && vid == 0)
1155                 return status;
1156
1157         if (test_bit(vid, adapter->vids))
1158                 return status;
1159
1160         set_bit(vid, adapter->vids);
1161         adapter->vlans_added++;
1162
1163         status = be_vid_config(adapter);
1164         if (status) {
1165                 adapter->vlans_added--;
1166                 clear_bit(vid, adapter->vids);
1167         }
1168
1169         return status;
1170 }
1171
1172 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1173 {
1174         struct be_adapter *adapter = netdev_priv(netdev);
1175         int status = 0;
1176
1177         /* Packets with VID 0 are always received by Lancer by default */
1178         if (lancer_chip(adapter) && vid == 0)
1179                 goto ret;
1180
1181         clear_bit(vid, adapter->vids);
1182         status = be_vid_config(adapter);
1183         if (!status)
1184                 adapter->vlans_added--;
1185         else
1186                 set_bit(vid, adapter->vids);
1187 ret:
1188         return status;
1189 }
1190
1191 static void be_clear_promisc(struct be_adapter *adapter)
1192 {
1193         adapter->promiscuous = false;
1194         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1195
1196         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1197 }
1198
1199 static void be_set_rx_mode(struct net_device *netdev)
1200 {
1201         struct be_adapter *adapter = netdev_priv(netdev);
1202         int status;
1203
1204         if (netdev->flags & IFF_PROMISC) {
1205                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1206                 adapter->promiscuous = true;
1207                 goto done;
1208         }
1209
1210         /* BE was previously in promiscuous mode; disable it */
1211         if (adapter->promiscuous) {
1212                 be_clear_promisc(adapter);
1213                 if (adapter->vlans_added)
1214                         be_vid_config(adapter);
1215         }
1216
1217         /* Enable multicast promisc if num configured exceeds what we support */
1218         if (netdev->flags & IFF_ALLMULTI ||
1219             netdev_mc_count(netdev) > be_max_mc(adapter))
1220                 goto set_mcast_promisc;
1221
1222         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1223                 struct netdev_hw_addr *ha;
1224                 int i = 1; /* First slot is claimed by the Primary MAC */
1225
1226                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1227                         be_cmd_pmac_del(adapter, adapter->if_handle,
1228                                         adapter->pmac_id[i], 0);
1229                 }
1230
1231                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1232                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1233                         adapter->promiscuous = true;
1234                         goto done;
1235                 }
1236
1237                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1238                         adapter->uc_macs++; /* First slot is for Primary MAC */
1239                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1240                                         adapter->if_handle,
1241                                         &adapter->pmac_id[adapter->uc_macs], 0);
1242                 }
1243         }
1244
1245         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1246         if (!status) {
1247                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1248                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1249                 goto done;
1250         }
1251
1252 set_mcast_promisc:
1253         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1254                 return;
1255
1256         /* Set to MCAST promisc mode if setting MULTICAST address fails
1257          * or if num configured exceeds what we support
1258          */
1259         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1260         if (!status)
1261                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1262 done:
1263         return;
1264 }
1265
1266 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1267 {
1268         struct be_adapter *adapter = netdev_priv(netdev);
1269         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1270         int status;
1271
1272         if (!sriov_enabled(adapter))
1273                 return -EPERM;
1274
1275         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1276                 return -EINVAL;
1277
1278         if (BEx_chip(adapter)) {
1279                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1280                                 vf + 1);
1281
1282                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1283                                          &vf_cfg->pmac_id, vf + 1);
1284         } else {
1285                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1286                                         vf + 1);
1287         }
1288
1289         if (status)
1290                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed\n",
1291                         mac, vf);
1292         else
1293                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
1294
1295         return status;
1296 }
1297
1298 static int be_get_vf_config(struct net_device *netdev, int vf,
1299                             struct ifla_vf_info *vi)
1300 {
1301         struct be_adapter *adapter = netdev_priv(netdev);
1302         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1303
1304         if (!sriov_enabled(adapter))
1305                 return -EPERM;
1306
1307         if (vf >= adapter->num_vfs)
1308                 return -EINVAL;
1309
1310         vi->vf = vf;
1311         vi->max_tx_rate = vf_cfg->tx_rate;
1312         vi->min_tx_rate = 0;
1313         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1314         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1315         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1316         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1317
1318         return 0;
1319 }
1320
1321 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1322 {
1323         struct be_adapter *adapter = netdev_priv(netdev);
1324         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1325         int status = 0;
1326
1327         if (!sriov_enabled(adapter))
1328                 return -EPERM;
1329
1330         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1331                 return -EINVAL;
1332
1333         if (vlan || qos) {
1334                 vlan |= qos << VLAN_PRIO_SHIFT;
1335                 if (vf_cfg->vlan_tag != vlan)
1336                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1337                                                        vf_cfg->if_handle, 0);
1338         } else {
1339                 /* Reset Transparent Vlan Tagging. */
1340                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1341                                                vf + 1, vf_cfg->if_handle, 0);
1342         }
1343
1344         if (!status)
1345                 vf_cfg->vlan_tag = vlan;
1346         else
1347                 dev_info(&adapter->pdev->dev,
1348                          "VLAN %d config on VF %d failed\n", vlan, vf);
1349         return status;
1350 }
1351
1352 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1353                              int min_tx_rate, int max_tx_rate)
1354 {
1355         struct be_adapter *adapter = netdev_priv(netdev);
1356         struct device *dev = &adapter->pdev->dev;
1357         int percent_rate, status = 0;
1358         u16 link_speed = 0;
1359         u8 link_status;
1360
1361         if (!sriov_enabled(adapter))
1362                 return -EPERM;
1363
1364         if (vf >= adapter->num_vfs)
1365                 return -EINVAL;
1366
1367         if (min_tx_rate)
1368                 return -EINVAL;
1369
1370         if (!max_tx_rate)
1371                 goto config_qos;
1372
1373         status = be_cmd_link_status_query(adapter, &link_speed,
1374                                           &link_status, 0);
1375         if (status)
1376                 goto err;
1377
1378         if (!link_status) {
1379                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1380                 status = -EPERM;
1381                 goto err;
1382         }
1383
1384         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1385                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1386                         link_speed);
1387                 status = -EINVAL;
1388                 goto err;
1389         }
1390
1391         /* On Skyhawk the QOS setting must be done only as a % value */
1392         percent_rate = link_speed / 100;
1393         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1394                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1395                         percent_rate);
1396                 status = -EINVAL;
1397                 goto err;
1398         }
1399
1400 config_qos:
1401         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1402         if (status)
1403                 goto err;
1404
1405         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1406         return 0;
1407
1408 err:
1409         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1410                 max_tx_rate, vf);
1411         return status;
1412 }
1413 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1414                                 int link_state)
1415 {
1416         struct be_adapter *adapter = netdev_priv(netdev);
1417         int status;
1418
1419         if (!sriov_enabled(adapter))
1420                 return -EPERM;
1421
1422         if (vf >= adapter->num_vfs)
1423                 return -EINVAL;
1424
1425         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1426         if (!status)
1427                 adapter->vf_cfg[vf].plink_tracking = link_state;
1428
1429         return status;
1430 }
1431
1432 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1433                           ulong now)
1434 {
1435         aic->rx_pkts_prev = rx_pkts;
1436         aic->tx_reqs_prev = tx_pkts;
1437         aic->jiffies = now;
1438 }
1439
1440 static void be_eqd_update(struct be_adapter *adapter)
1441 {
1442         struct be_set_eqd set_eqd[MAX_EVT_QS];
1443         int eqd, i, num = 0, start;
1444         struct be_aic_obj *aic;
1445         struct be_eq_obj *eqo;
1446         struct be_rx_obj *rxo;
1447         struct be_tx_obj *txo;
1448         u64 rx_pkts, tx_pkts;
1449         ulong now;
1450         u32 pps, delta;
1451
1452         for_all_evt_queues(adapter, eqo, i) {
1453                 aic = &adapter->aic_obj[eqo->idx];
1454                 if (!aic->enable) {
1455                         if (aic->jiffies)
1456                                 aic->jiffies = 0;
1457                         eqd = aic->et_eqd;
1458                         goto modify_eqd;
1459                 }
1460
1461                 rxo = &adapter->rx_obj[eqo->idx];
1462                 do {
1463                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1464                         rx_pkts = rxo->stats.rx_pkts;
1465                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1466
1467                 txo = &adapter->tx_obj[eqo->idx];
1468                 do {
1469                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1470                         tx_pkts = txo->stats.tx_reqs;
1471                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1472
1473
1474                 /* Skip, if wrapped around or first calculation */
1475                 now = jiffies;
1476                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1477                     rx_pkts < aic->rx_pkts_prev ||
1478                     tx_pkts < aic->tx_reqs_prev) {
1479                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1480                         continue;
1481                 }
1482
1483                 delta = jiffies_to_msecs(now - aic->jiffies);
1484                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1485                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1486                 eqd = (pps / 15000) << 2;
1487
1488                 if (eqd < 8)
1489                         eqd = 0;
1490                 eqd = min_t(u32, eqd, aic->max_eqd);
1491                 eqd = max_t(u32, eqd, aic->min_eqd);
1492
1493                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1494 modify_eqd:
1495                 if (eqd != aic->prev_eqd) {
1496                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1497                         set_eqd[num].eq_id = eqo->q.id;
1498                         aic->prev_eqd = eqd;
1499                         num++;
1500                 }
1501         }
1502
1503         if (num)
1504                 be_cmd_modify_eqd(adapter, set_eqd, num);
1505 }
1506
1507 static void be_rx_stats_update(struct be_rx_obj *rxo,
1508                                struct be_rx_compl_info *rxcp)
1509 {
1510         struct be_rx_stats *stats = rx_stats(rxo);
1511
1512         u64_stats_update_begin(&stats->sync);
1513         stats->rx_compl++;
1514         stats->rx_bytes += rxcp->pkt_size;
1515         stats->rx_pkts++;
1516         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1517                 stats->rx_mcast_pkts++;
1518         if (rxcp->err)
1519                 stats->rx_compl_err++;
1520         u64_stats_update_end(&stats->sync);
1521 }
1522
1523 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1524 {
1525         /* L4 checksum is not reliable for non TCP/UDP packets.
1526          * Also ignore ipcksm for ipv6 pkts
1527          */
1528         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1529                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1530 }
1531
1532 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1533 {
1534         struct be_adapter *adapter = rxo->adapter;
1535         struct be_rx_page_info *rx_page_info;
1536         struct be_queue_info *rxq = &rxo->q;
1537         u16 frag_idx = rxq->tail;
1538
1539         rx_page_info = &rxo->page_info_tbl[frag_idx];
1540         BUG_ON(!rx_page_info->page);
1541
1542         if (rx_page_info->last_frag) {
1543                 dma_unmap_page(&adapter->pdev->dev,
1544                                dma_unmap_addr(rx_page_info, bus),
1545                                adapter->big_page_size, DMA_FROM_DEVICE);
1546                 rx_page_info->last_frag = false;
1547         } else {
1548                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1549                                         dma_unmap_addr(rx_page_info, bus),
1550                                         rx_frag_size, DMA_FROM_DEVICE);
1551         }
1552
1553         queue_tail_inc(rxq);
1554         atomic_dec(&rxq->used);
1555         return rx_page_info;
1556 }
1557
1558 /* Throwaway the data in the Rx completion */
1559 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1560                                 struct be_rx_compl_info *rxcp)
1561 {
1562         struct be_rx_page_info *page_info;
1563         u16 i, num_rcvd = rxcp->num_rcvd;
1564
1565         for (i = 0; i < num_rcvd; i++) {
1566                 page_info = get_rx_page_info(rxo);
1567                 put_page(page_info->page);
1568                 memset(page_info, 0, sizeof(*page_info));
1569         }
1570 }
1571
1572 /*
1573  * skb_fill_rx_data forms a complete skb for an ether frame
1574  * indicated by rxcp.
1575  */
1576 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1577                              struct be_rx_compl_info *rxcp)
1578 {
1579         struct be_rx_page_info *page_info;
1580         u16 i, j;
1581         u16 hdr_len, curr_frag_len, remaining;
1582         u8 *start;
1583
1584         page_info = get_rx_page_info(rxo);
1585         start = page_address(page_info->page) + page_info->page_offset;
1586         prefetch(start);
1587
1588         /* Copy data in the first descriptor of this completion */
1589         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1590
1591         skb->len = curr_frag_len;
1592         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1593                 memcpy(skb->data, start, curr_frag_len);
1594                 /* Complete packet has now been moved to data */
1595                 put_page(page_info->page);
1596                 skb->data_len = 0;
1597                 skb->tail += curr_frag_len;
1598         } else {
1599                 hdr_len = ETH_HLEN;
1600                 memcpy(skb->data, start, hdr_len);
1601                 skb_shinfo(skb)->nr_frags = 1;
1602                 skb_frag_set_page(skb, 0, page_info->page);
1603                 skb_shinfo(skb)->frags[0].page_offset =
1604                                         page_info->page_offset + hdr_len;
1605                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1606                                   curr_frag_len - hdr_len);
1607                 skb->data_len = curr_frag_len - hdr_len;
1608                 skb->truesize += rx_frag_size;
1609                 skb->tail += hdr_len;
1610         }
1611         page_info->page = NULL;
1612
1613         if (rxcp->pkt_size <= rx_frag_size) {
1614                 BUG_ON(rxcp->num_rcvd != 1);
1615                 return;
1616         }
1617
1618         /* More frags present for this completion */
1619         remaining = rxcp->pkt_size - curr_frag_len;
1620         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1621                 page_info = get_rx_page_info(rxo);
1622                 curr_frag_len = min(remaining, rx_frag_size);
1623
1624                 /* Coalesce all frags from the same physical page in one slot */
1625                 if (page_info->page_offset == 0) {
1626                         /* Fresh page */
1627                         j++;
1628                         skb_frag_set_page(skb, j, page_info->page);
1629                         skb_shinfo(skb)->frags[j].page_offset =
1630                                                         page_info->page_offset;
1631                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1632                         skb_shinfo(skb)->nr_frags++;
1633                 } else {
1634                         put_page(page_info->page);
1635                 }
1636
1637                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1638                 skb->len += curr_frag_len;
1639                 skb->data_len += curr_frag_len;
1640                 skb->truesize += rx_frag_size;
1641                 remaining -= curr_frag_len;
1642                 page_info->page = NULL;
1643         }
1644         BUG_ON(j > MAX_SKB_FRAGS);
1645 }
1646
1647 /* Process the RX completion indicated by rxcp when GRO is disabled */
1648 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1649                                 struct be_rx_compl_info *rxcp)
1650 {
1651         struct be_adapter *adapter = rxo->adapter;
1652         struct net_device *netdev = adapter->netdev;
1653         struct sk_buff *skb;
1654
1655         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1656         if (unlikely(!skb)) {
1657                 rx_stats(rxo)->rx_drops_no_skbs++;
1658                 be_rx_compl_discard(rxo, rxcp);
1659                 return;
1660         }
1661
1662         skb_fill_rx_data(rxo, skb, rxcp);
1663
1664         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1665                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1666         else
1667                 skb_checksum_none_assert(skb);
1668
1669         skb->protocol = eth_type_trans(skb, netdev);
1670         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1671         if (netdev->features & NETIF_F_RXHASH)
1672                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1673
1674         skb->encapsulation = rxcp->tunneled;
1675         skb_mark_napi_id(skb, napi);
1676
1677         if (rxcp->vlanf)
1678                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1679
1680         netif_receive_skb(skb);
1681 }
1682
1683 /* Process the RX completion indicated by rxcp when GRO is enabled */
1684 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1685                                     struct napi_struct *napi,
1686                                     struct be_rx_compl_info *rxcp)
1687 {
1688         struct be_adapter *adapter = rxo->adapter;
1689         struct be_rx_page_info *page_info;
1690         struct sk_buff *skb = NULL;
1691         u16 remaining, curr_frag_len;
1692         u16 i, j;
1693
1694         skb = napi_get_frags(napi);
1695         if (!skb) {
1696                 be_rx_compl_discard(rxo, rxcp);
1697                 return;
1698         }
1699
1700         remaining = rxcp->pkt_size;
1701         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1702                 page_info = get_rx_page_info(rxo);
1703
1704                 curr_frag_len = min(remaining, rx_frag_size);
1705
1706                 /* Coalesce all frags from the same physical page in one slot */
1707                 if (i == 0 || page_info->page_offset == 0) {
1708                         /* First frag or Fresh page */
1709                         j++;
1710                         skb_frag_set_page(skb, j, page_info->page);
1711                         skb_shinfo(skb)->frags[j].page_offset =
1712                                                         page_info->page_offset;
1713                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1714                 } else {
1715                         put_page(page_info->page);
1716                 }
1717                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1718                 skb->truesize += rx_frag_size;
1719                 remaining -= curr_frag_len;
1720                 memset(page_info, 0, sizeof(*page_info));
1721         }
1722         BUG_ON(j > MAX_SKB_FRAGS);
1723
1724         skb_shinfo(skb)->nr_frags = j + 1;
1725         skb->len = rxcp->pkt_size;
1726         skb->data_len = rxcp->pkt_size;
1727         skb->ip_summed = CHECKSUM_UNNECESSARY;
1728         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1729         if (adapter->netdev->features & NETIF_F_RXHASH)
1730                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1731
1732         skb->encapsulation = rxcp->tunneled;
1733         skb_mark_napi_id(skb, napi);
1734
1735         if (rxcp->vlanf)
1736                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1737
1738         napi_gro_frags(napi);
1739 }
1740
1741 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1742                                  struct be_rx_compl_info *rxcp)
1743 {
1744         rxcp->pkt_size =
1745                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, pktsize, compl);
1746         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtp, compl);
1747         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, err, compl);
1748         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tcpf, compl);
1749         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, udpf, compl);
1750         rxcp->ip_csum =
1751                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ipcksm, compl);
1752         rxcp->l4_csum =
1753                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, l4_cksm, compl);
1754         rxcp->ipv6 =
1755                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ip_version, compl);
1756         rxcp->num_rcvd =
1757                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, numfrags, compl);
1758         rxcp->pkt_type =
1759                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, cast_enc, compl);
1760         rxcp->rss_hash =
1761                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, rsshash, compl);
1762         if (rxcp->vlanf) {
1763                 rxcp->qnq = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, qnq,
1764                                           compl);
1765                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v1,
1766                                                vlan_tag, compl);
1767         }
1768         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, port, compl);
1769         rxcp->tunneled =
1770                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tunneled, compl);
1771 }
1772
1773 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1774                                  struct be_rx_compl_info *rxcp)
1775 {
1776         rxcp->pkt_size =
1777                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, pktsize, compl);
1778         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtp, compl);
1779         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, err, compl);
1780         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, tcpf, compl);
1781         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, udpf, compl);
1782         rxcp->ip_csum =
1783                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ipcksm, compl);
1784         rxcp->l4_csum =
1785                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, l4_cksm, compl);
1786         rxcp->ipv6 =
1787                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ip_version, compl);
1788         rxcp->num_rcvd =
1789                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, numfrags, compl);
1790         rxcp->pkt_type =
1791                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, cast_enc, compl);
1792         rxcp->rss_hash =
1793                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, rsshash, compl);
1794         if (rxcp->vlanf) {
1795                 rxcp->qnq = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, qnq,
1796                                           compl);
1797                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0,
1798                                                vlan_tag, compl);
1799         }
1800         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, port, compl);
1801         rxcp->ip_frag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0,
1802                                       ip_frag, compl);
1803 }
1804
1805 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1806 {
1807         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1808         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1809         struct be_adapter *adapter = rxo->adapter;
1810
1811         /* For checking the valid bit it is Ok to use either definition as the
1812          * valid bit is at the same position in both v0 and v1 Rx compl */
1813         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1814                 return NULL;
1815
1816         rmb();
1817         be_dws_le_to_cpu(compl, sizeof(*compl));
1818
1819         if (adapter->be3_native)
1820                 be_parse_rx_compl_v1(compl, rxcp);
1821         else
1822                 be_parse_rx_compl_v0(compl, rxcp);
1823
1824         if (rxcp->ip_frag)
1825                 rxcp->l4_csum = 0;
1826
1827         if (rxcp->vlanf) {
1828                 /* In QNQ modes, if qnq bit is not set, then the packet was
1829                  * tagged only with the transparent outer vlan-tag and must
1830                  * not be treated as a vlan packet by host
1831                  */
1832                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1833                         rxcp->vlanf = 0;
1834
1835                 if (!lancer_chip(adapter))
1836                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1837
1838                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1839                     !test_bit(rxcp->vlan_tag, adapter->vids))
1840                         rxcp->vlanf = 0;
1841         }
1842
1843         /* As the compl has been parsed, reset it; we wont touch it again */
1844         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1845
1846         queue_tail_inc(&rxo->cq);
1847         return rxcp;
1848 }
1849
1850 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1851 {
1852         u32 order = get_order(size);
1853
1854         if (order > 0)
1855                 gfp |= __GFP_COMP;
1856         return  alloc_pages(gfp, order);
1857 }
1858
1859 /*
1860  * Allocate a page, split it to fragments of size rx_frag_size and post as
1861  * receive buffers to BE
1862  */
1863 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
1864 {
1865         struct be_adapter *adapter = rxo->adapter;
1866         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1867         struct be_queue_info *rxq = &rxo->q;
1868         struct page *pagep = NULL;
1869         struct device *dev = &adapter->pdev->dev;
1870         struct be_eth_rx_d *rxd;
1871         u64 page_dmaaddr = 0, frag_dmaaddr;
1872         u32 posted, page_offset = 0;
1873
1874         page_info = &rxo->page_info_tbl[rxq->head];
1875         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1876                 if (!pagep) {
1877                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1878                         if (unlikely(!pagep)) {
1879                                 rx_stats(rxo)->rx_post_fail++;
1880                                 break;
1881                         }
1882                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1883                                                     adapter->big_page_size,
1884                                                     DMA_FROM_DEVICE);
1885                         if (dma_mapping_error(dev, page_dmaaddr)) {
1886                                 put_page(pagep);
1887                                 pagep = NULL;
1888                                 rx_stats(rxo)->rx_post_fail++;
1889                                 break;
1890                         }
1891                         page_offset = 0;
1892                 } else {
1893                         get_page(pagep);
1894                         page_offset += rx_frag_size;
1895                 }
1896                 page_info->page_offset = page_offset;
1897                 page_info->page = pagep;
1898
1899                 rxd = queue_head_node(rxq);
1900                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1901                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1902                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1903
1904                 /* Any space left in the current big page for another frag? */
1905                 if ((page_offset + rx_frag_size + rx_frag_size) >
1906                                         adapter->big_page_size) {
1907                         pagep = NULL;
1908                         page_info->last_frag = true;
1909                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1910                 } else {
1911                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1912                 }
1913
1914                 prev_page_info = page_info;
1915                 queue_head_inc(rxq);
1916                 page_info = &rxo->page_info_tbl[rxq->head];
1917         }
1918
1919         /* Mark the last frag of a page when we break out of the above loop
1920          * with no more slots available in the RXQ
1921          */
1922         if (pagep) {
1923                 prev_page_info->last_frag = true;
1924                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1925         }
1926
1927         if (posted) {
1928                 atomic_add(posted, &rxq->used);
1929                 if (rxo->rx_post_starved)
1930                         rxo->rx_post_starved = false;
1931                 be_rxq_notify(adapter, rxq->id, posted);
1932         } else if (atomic_read(&rxq->used) == 0) {
1933                 /* Let be_worker replenish when memory is available */
1934                 rxo->rx_post_starved = true;
1935         }
1936 }
1937
1938 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1939 {
1940         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1941
1942         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1943                 return NULL;
1944
1945         rmb();
1946         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1947
1948         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1949
1950         queue_tail_inc(tx_cq);
1951         return txcp;
1952 }
1953
1954 static u16 be_tx_compl_process(struct be_adapter *adapter,
1955                                struct be_tx_obj *txo, u16 last_index)
1956 {
1957         struct be_queue_info *txq = &txo->q;
1958         struct be_eth_wrb *wrb;
1959         struct sk_buff **sent_skbs = txo->sent_skb_list;
1960         struct sk_buff *sent_skb;
1961         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1962         bool unmap_skb_hdr = true;
1963
1964         sent_skb = sent_skbs[txq->tail];
1965         BUG_ON(!sent_skb);
1966         sent_skbs[txq->tail] = NULL;
1967
1968         /* skip header wrb */
1969         queue_tail_inc(txq);
1970
1971         do {
1972                 cur_index = txq->tail;
1973                 wrb = queue_tail_node(txq);
1974                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1975                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1976                 unmap_skb_hdr = false;
1977
1978                 num_wrbs++;
1979                 queue_tail_inc(txq);
1980         } while (cur_index != last_index);
1981
1982         dev_kfree_skb_any(sent_skb);
1983         return num_wrbs;
1984 }
1985
1986 /* Return the number of events in the event queue */
1987 static inline int events_get(struct be_eq_obj *eqo)
1988 {
1989         struct be_eq_entry *eqe;
1990         int num = 0;
1991
1992         do {
1993                 eqe = queue_tail_node(&eqo->q);
1994                 if (eqe->evt == 0)
1995                         break;
1996
1997                 rmb();
1998                 eqe->evt = 0;
1999                 num++;
2000                 queue_tail_inc(&eqo->q);
2001         } while (true);
2002
2003         return num;
2004 }
2005
2006 /* Leaves the EQ is disarmed state */
2007 static void be_eq_clean(struct be_eq_obj *eqo)
2008 {
2009         int num = events_get(eqo);
2010
2011         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2012 }
2013
2014 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2015 {
2016         struct be_rx_page_info *page_info;
2017         struct be_queue_info *rxq = &rxo->q;
2018         struct be_queue_info *rx_cq = &rxo->cq;
2019         struct be_rx_compl_info *rxcp;
2020         struct be_adapter *adapter = rxo->adapter;
2021         int flush_wait = 0;
2022
2023         /* Consume pending rx completions.
2024          * Wait for the flush completion (identified by zero num_rcvd)
2025          * to arrive. Notify CQ even when there are no more CQ entries
2026          * for HW to flush partially coalesced CQ entries.
2027          * In Lancer, there is no need to wait for flush compl.
2028          */
2029         for (;;) {
2030                 rxcp = be_rx_compl_get(rxo);
2031                 if (rxcp == NULL) {
2032                         if (lancer_chip(adapter))
2033                                 break;
2034
2035                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2036                                 dev_warn(&adapter->pdev->dev,
2037                                          "did not receive flush compl\n");
2038                                 break;
2039                         }
2040                         be_cq_notify(adapter, rx_cq->id, true, 0);
2041                         mdelay(1);
2042                 } else {
2043                         be_rx_compl_discard(rxo, rxcp);
2044                         be_cq_notify(adapter, rx_cq->id, false, 1);
2045                         if (rxcp->num_rcvd == 0)
2046                                 break;
2047                 }
2048         }
2049
2050         /* After cleanup, leave the CQ in unarmed state */
2051         be_cq_notify(adapter, rx_cq->id, false, 0);
2052
2053         /* Then free posted rx buffers that were not used */
2054         while (atomic_read(&rxq->used) > 0) {
2055                 page_info = get_rx_page_info(rxo);
2056                 put_page(page_info->page);
2057                 memset(page_info, 0, sizeof(*page_info));
2058         }
2059         BUG_ON(atomic_read(&rxq->used));
2060         rxq->tail = rxq->head = 0;
2061 }
2062
2063 static void be_tx_compl_clean(struct be_adapter *adapter)
2064 {
2065         struct be_tx_obj *txo;
2066         struct be_queue_info *txq;
2067         struct be_eth_tx_compl *txcp;
2068         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2069         struct sk_buff *sent_skb;
2070         bool dummy_wrb;
2071         int i, pending_txqs;
2072
2073         /* Stop polling for compls when HW has been silent for 10ms */
2074         do {
2075                 pending_txqs = adapter->num_tx_qs;
2076
2077                 for_all_tx_queues(adapter, txo, i) {
2078                         cmpl = 0;
2079                         num_wrbs = 0;
2080                         txq = &txo->q;
2081                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2082                                 end_idx =
2083                                         AMAP_GET_BITS(struct amap_eth_tx_compl,
2084                                                       wrb_index, txcp);
2085                                 num_wrbs += be_tx_compl_process(adapter, txo,
2086                                                                 end_idx);
2087                                 cmpl++;
2088                         }
2089                         if (cmpl) {
2090                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2091                                 atomic_sub(num_wrbs, &txq->used);
2092                                 timeo = 0;
2093                         }
2094                         if (atomic_read(&txq->used) == 0)
2095                                 pending_txqs--;
2096                 }
2097
2098                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2099                         break;
2100
2101                 mdelay(1);
2102         } while (true);
2103
2104         for_all_tx_queues(adapter, txo, i) {
2105                 txq = &txo->q;
2106                 if (atomic_read(&txq->used))
2107                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2108                                 atomic_read(&txq->used));
2109
2110                 /* free posted tx for which compls will never arrive */
2111                 while (atomic_read(&txq->used)) {
2112                         sent_skb = txo->sent_skb_list[txq->tail];
2113                         end_idx = txq->tail;
2114                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2115                                                    &dummy_wrb);
2116                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2117                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2118                         atomic_sub(num_wrbs, &txq->used);
2119                 }
2120         }
2121 }
2122
2123 static void be_evt_queues_destroy(struct be_adapter *adapter)
2124 {
2125         struct be_eq_obj *eqo;
2126         int i;
2127
2128         for_all_evt_queues(adapter, eqo, i) {
2129                 if (eqo->q.created) {
2130                         be_eq_clean(eqo);
2131                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2132                         napi_hash_del(&eqo->napi);
2133                         netif_napi_del(&eqo->napi);
2134                 }
2135                 be_queue_free(adapter, &eqo->q);
2136         }
2137 }
2138
2139 static int be_evt_queues_create(struct be_adapter *adapter)
2140 {
2141         struct be_queue_info *eq;
2142         struct be_eq_obj *eqo;
2143         struct be_aic_obj *aic;
2144         int i, rc;
2145
2146         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2147                                     adapter->cfg_num_qs);
2148
2149         for_all_evt_queues(adapter, eqo, i) {
2150                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2151                                BE_NAPI_WEIGHT);
2152                 napi_hash_add(&eqo->napi);
2153                 aic = &adapter->aic_obj[i];
2154                 eqo->adapter = adapter;
2155                 eqo->tx_budget = BE_TX_BUDGET;
2156                 eqo->idx = i;
2157                 aic->max_eqd = BE_MAX_EQD;
2158                 aic->enable = true;
2159
2160                 eq = &eqo->q;
2161                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2162                                     sizeof(struct be_eq_entry));
2163                 if (rc)
2164                         return rc;
2165
2166                 rc = be_cmd_eq_create(adapter, eqo);
2167                 if (rc)
2168                         return rc;
2169         }
2170         return 0;
2171 }
2172
2173 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2174 {
2175         struct be_queue_info *q;
2176
2177         q = &adapter->mcc_obj.q;
2178         if (q->created)
2179                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2180         be_queue_free(adapter, q);
2181
2182         q = &adapter->mcc_obj.cq;
2183         if (q->created)
2184                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2185         be_queue_free(adapter, q);
2186 }
2187
2188 /* Must be called only after TX qs are created as MCC shares TX EQ */
2189 static int be_mcc_queues_create(struct be_adapter *adapter)
2190 {
2191         struct be_queue_info *q, *cq;
2192
2193         cq = &adapter->mcc_obj.cq;
2194         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2195                            sizeof(struct be_mcc_compl)))
2196                 goto err;
2197
2198         /* Use the default EQ for MCC completions */
2199         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2200                 goto mcc_cq_free;
2201
2202         q = &adapter->mcc_obj.q;
2203         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2204                 goto mcc_cq_destroy;
2205
2206         if (be_cmd_mccq_create(adapter, q, cq))
2207                 goto mcc_q_free;
2208
2209         return 0;
2210
2211 mcc_q_free:
2212         be_queue_free(adapter, q);
2213 mcc_cq_destroy:
2214         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2215 mcc_cq_free:
2216         be_queue_free(adapter, cq);
2217 err:
2218         return -1;
2219 }
2220
2221 static void be_tx_queues_destroy(struct be_adapter *adapter)
2222 {
2223         struct be_queue_info *q;
2224         struct be_tx_obj *txo;
2225         u8 i;
2226
2227         for_all_tx_queues(adapter, txo, i) {
2228                 q = &txo->q;
2229                 if (q->created)
2230                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2231                 be_queue_free(adapter, q);
2232
2233                 q = &txo->cq;
2234                 if (q->created)
2235                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2236                 be_queue_free(adapter, q);
2237         }
2238 }
2239
2240 static int be_tx_qs_create(struct be_adapter *adapter)
2241 {
2242         struct be_queue_info *cq, *eq;
2243         struct be_tx_obj *txo;
2244         int status, i;
2245
2246         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2247
2248         for_all_tx_queues(adapter, txo, i) {
2249                 cq = &txo->cq;
2250                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2251                                         sizeof(struct be_eth_tx_compl));
2252                 if (status)
2253                         return status;
2254
2255                 u64_stats_init(&txo->stats.sync);
2256                 u64_stats_init(&txo->stats.sync_compl);
2257
2258                 /* If num_evt_qs is less than num_tx_qs, then more than
2259                  * one txq share an eq
2260                  */
2261                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2262                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2263                 if (status)
2264                         return status;
2265
2266                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2267                                         sizeof(struct be_eth_wrb));
2268                 if (status)
2269                         return status;
2270
2271                 status = be_cmd_txq_create(adapter, txo);
2272                 if (status)
2273                         return status;
2274         }
2275
2276         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2277                  adapter->num_tx_qs);
2278         return 0;
2279 }
2280
2281 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2282 {
2283         struct be_queue_info *q;
2284         struct be_rx_obj *rxo;
2285         int i;
2286
2287         for_all_rx_queues(adapter, rxo, i) {
2288                 q = &rxo->cq;
2289                 if (q->created)
2290                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2291                 be_queue_free(adapter, q);
2292         }
2293 }
2294
2295 static int be_rx_cqs_create(struct be_adapter *adapter)
2296 {
2297         struct be_queue_info *eq, *cq;
2298         struct be_rx_obj *rxo;
2299         int rc, i;
2300
2301         /* We can create as many RSS rings as there are EQs. */
2302         adapter->num_rx_qs = adapter->num_evt_qs;
2303
2304         /* We'll use RSS only if atleast 2 RSS rings are supported.
2305          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2306          */
2307         if (adapter->num_rx_qs > 1)
2308                 adapter->num_rx_qs++;
2309
2310         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2311         for_all_rx_queues(adapter, rxo, i) {
2312                 rxo->adapter = adapter;
2313                 cq = &rxo->cq;
2314                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2315                                     sizeof(struct be_eth_rx_compl));
2316                 if (rc)
2317                         return rc;
2318
2319                 u64_stats_init(&rxo->stats.sync);
2320                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2321                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2322                 if (rc)
2323                         return rc;
2324         }
2325
2326         dev_info(&adapter->pdev->dev,
2327                  "created %d RSS queue(s) and 1 default RX queue\n",
2328                  adapter->num_rx_qs - 1);
2329         return 0;
2330 }
2331
2332 static irqreturn_t be_intx(int irq, void *dev)
2333 {
2334         struct be_eq_obj *eqo = dev;
2335         struct be_adapter *adapter = eqo->adapter;
2336         int num_evts = 0;
2337
2338         /* IRQ is not expected when NAPI is scheduled as the EQ
2339          * will not be armed.
2340          * But, this can happen on Lancer INTx where it takes
2341          * a while to de-assert INTx or in BE2 where occasionaly
2342          * an interrupt may be raised even when EQ is unarmed.
2343          * If NAPI is already scheduled, then counting & notifying
2344          * events will orphan them.
2345          */
2346         if (napi_schedule_prep(&eqo->napi)) {
2347                 num_evts = events_get(eqo);
2348                 __napi_schedule(&eqo->napi);
2349                 if (num_evts)
2350                         eqo->spurious_intr = 0;
2351         }
2352         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2353
2354         /* Return IRQ_HANDLED only for the the first spurious intr
2355          * after a valid intr to stop the kernel from branding
2356          * this irq as a bad one!
2357          */
2358         if (num_evts || eqo->spurious_intr++ == 0)
2359                 return IRQ_HANDLED;
2360         else
2361                 return IRQ_NONE;
2362 }
2363
2364 static irqreturn_t be_msix(int irq, void *dev)
2365 {
2366         struct be_eq_obj *eqo = dev;
2367
2368         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2369         napi_schedule(&eqo->napi);
2370         return IRQ_HANDLED;
2371 }
2372
2373 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2374 {
2375         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2376 }
2377
2378 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2379                          int budget, int polling)
2380 {
2381         struct be_adapter *adapter = rxo->adapter;
2382         struct be_queue_info *rx_cq = &rxo->cq;
2383         struct be_rx_compl_info *rxcp;
2384         u32 work_done;
2385
2386         for (work_done = 0; work_done < budget; work_done++) {
2387                 rxcp = be_rx_compl_get(rxo);
2388                 if (!rxcp)
2389                         break;
2390
2391                 /* Is it a flush compl that has no data */
2392                 if (unlikely(rxcp->num_rcvd == 0))
2393                         goto loop_continue;
2394
2395                 /* Discard compl with partial DMA Lancer B0 */
2396                 if (unlikely(!rxcp->pkt_size)) {
2397                         be_rx_compl_discard(rxo, rxcp);
2398                         goto loop_continue;
2399                 }
2400
2401                 /* On BE drop pkts that arrive due to imperfect filtering in
2402                  * promiscuous mode on some skews
2403                  */
2404                 if (unlikely(rxcp->port != adapter->port_num &&
2405                              !lancer_chip(adapter))) {
2406                         be_rx_compl_discard(rxo, rxcp);
2407                         goto loop_continue;
2408                 }
2409
2410                 /* Don't do gro when we're busy_polling */
2411                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2412                         be_rx_compl_process_gro(rxo, napi, rxcp);
2413                 else
2414                         be_rx_compl_process(rxo, napi, rxcp);
2415
2416 loop_continue:
2417                 be_rx_stats_update(rxo, rxcp);
2418         }
2419
2420         if (work_done) {
2421                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2422
2423                 /* When an rx-obj gets into post_starved state, just
2424                  * let be_worker do the posting.
2425                  */
2426                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2427                     !rxo->rx_post_starved)
2428                         be_post_rx_frags(rxo, GFP_ATOMIC);
2429         }
2430
2431         return work_done;
2432 }
2433
2434 static bool be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2435                           int budget, int idx)
2436 {
2437         struct be_eth_tx_compl *txcp;
2438         int num_wrbs = 0, work_done;
2439
2440         for (work_done = 0; work_done < budget; work_done++) {
2441                 txcp = be_tx_compl_get(&txo->cq);
2442                 if (!txcp)
2443                         break;
2444                 num_wrbs += be_tx_compl_process(adapter, txo,
2445                                                 AMAP_GET_BITS(struct
2446                                                               amap_eth_tx_compl,
2447                                                               wrb_index, txcp));
2448         }
2449
2450         if (work_done) {
2451                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2452                 atomic_sub(num_wrbs, &txo->q.used);
2453
2454                 /* As Tx wrbs have been freed up, wake up netdev queue
2455                  * if it was stopped due to lack of tx wrbs.  */
2456                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2457                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2458                         netif_wake_subqueue(adapter->netdev, idx);
2459                 }
2460
2461                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2462                 tx_stats(txo)->tx_compl += work_done;
2463                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2464         }
2465         return (work_done < budget); /* Done */
2466 }
2467
2468 int be_poll(struct napi_struct *napi, int budget)
2469 {
2470         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2471         struct be_adapter *adapter = eqo->adapter;
2472         int max_work = 0, work, i, num_evts;
2473         struct be_rx_obj *rxo;
2474         bool tx_done;
2475
2476         num_evts = events_get(eqo);
2477
2478         /* Process all TXQs serviced by this EQ */
2479         for (i = eqo->idx; i < adapter->num_tx_qs; i += adapter->num_evt_qs) {
2480                 tx_done = be_process_tx(adapter, &adapter->tx_obj[i],
2481                                         eqo->tx_budget, i);
2482                 if (!tx_done)
2483                         max_work = budget;
2484         }
2485
2486         if (be_lock_napi(eqo)) {
2487                 /* This loop will iterate twice for EQ0 in which
2488                  * completions of the last RXQ (default one) are also processed
2489                  * For other EQs the loop iterates only once
2490                  */
2491                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2492                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2493                         max_work = max(work, max_work);
2494                 }
2495                 be_unlock_napi(eqo);
2496         } else {
2497                 max_work = budget;
2498         }
2499
2500         if (is_mcc_eqo(eqo))
2501                 be_process_mcc(adapter);
2502
2503         if (max_work < budget) {
2504                 napi_complete(napi);
2505                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2506         } else {
2507                 /* As we'll continue in polling mode, count and clear events */
2508                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2509         }
2510         return max_work;
2511 }
2512
2513 #ifdef CONFIG_NET_RX_BUSY_POLL
2514 static int be_busy_poll(struct napi_struct *napi)
2515 {
2516         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2517         struct be_adapter *adapter = eqo->adapter;
2518         struct be_rx_obj *rxo;
2519         int i, work = 0;
2520
2521         if (!be_lock_busy_poll(eqo))
2522                 return LL_FLUSH_BUSY;
2523
2524         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2525                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2526                 if (work)
2527                         break;
2528         }
2529
2530         be_unlock_busy_poll(eqo);
2531         return work;
2532 }
2533 #endif
2534
2535 void be_detect_error(struct be_adapter *adapter)
2536 {
2537         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2538         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2539         u32 i;
2540         bool error_detected = false;
2541         struct device *dev = &adapter->pdev->dev;
2542         struct net_device *netdev = adapter->netdev;
2543
2544         if (be_hw_error(adapter))
2545                 return;
2546
2547         if (lancer_chip(adapter)) {
2548                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2549                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2550                         sliport_err1 = ioread32(adapter->db +
2551                                                 SLIPORT_ERROR1_OFFSET);
2552                         sliport_err2 = ioread32(adapter->db +
2553                                                 SLIPORT_ERROR2_OFFSET);
2554                         adapter->hw_error = true;
2555                         /* Do not log error messages if its a FW reset */
2556                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2557                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2558                                 dev_info(dev, "Firmware update in progress\n");
2559                         } else {
2560                                 error_detected = true;
2561                                 dev_err(dev, "Error detected in the card\n");
2562                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2563                                         sliport_status);
2564                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2565                                         sliport_err1);
2566                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2567                                         sliport_err2);
2568                         }
2569                 }
2570         } else {
2571                 pci_read_config_dword(adapter->pdev,
2572                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2573                 pci_read_config_dword(adapter->pdev,
2574                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2575                 pci_read_config_dword(adapter->pdev,
2576                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2577                 pci_read_config_dword(adapter->pdev,
2578                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2579
2580                 ue_lo = (ue_lo & ~ue_lo_mask);
2581                 ue_hi = (ue_hi & ~ue_hi_mask);
2582
2583                 /* On certain platforms BE hardware can indicate spurious UEs.
2584                  * Allow HW to stop working completely in case of a real UE.
2585                  * Hence not setting the hw_error for UE detection.
2586                  */
2587
2588                 if (ue_lo || ue_hi) {
2589                         error_detected = true;
2590                         dev_err(dev,
2591                                 "Unrecoverable Error detected in the adapter");
2592                         dev_err(dev, "Please reboot server to recover");
2593                         if (skyhawk_chip(adapter))
2594                                 adapter->hw_error = true;
2595                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2596                                 if (ue_lo & 1)
2597                                         dev_err(dev, "UE: %s bit set\n",
2598                                                 ue_status_low_desc[i]);
2599                         }
2600                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2601                                 if (ue_hi & 1)
2602                                         dev_err(dev, "UE: %s bit set\n",
2603                                                 ue_status_hi_desc[i]);
2604                         }
2605                 }
2606         }
2607         if (error_detected)
2608                 netif_carrier_off(netdev);
2609 }
2610
2611 static void be_msix_disable(struct be_adapter *adapter)
2612 {
2613         if (msix_enabled(adapter)) {
2614                 pci_disable_msix(adapter->pdev);
2615                 adapter->num_msix_vec = 0;
2616                 adapter->num_msix_roce_vec = 0;
2617         }
2618 }
2619
2620 static int be_msix_enable(struct be_adapter *adapter)
2621 {
2622         int i, num_vec;
2623         struct device *dev = &adapter->pdev->dev;
2624
2625         /* If RoCE is supported, program the max number of NIC vectors that
2626          * may be configured via set-channels, along with vectors needed for
2627          * RoCe. Else, just program the number we'll use initially.
2628          */
2629         if (be_roce_supported(adapter))
2630                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2631                                 2 * num_online_cpus());
2632         else
2633                 num_vec = adapter->cfg_num_qs;
2634
2635         for (i = 0; i < num_vec; i++)
2636                 adapter->msix_entries[i].entry = i;
2637
2638         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2639                                         MIN_MSIX_VECTORS, num_vec);
2640         if (num_vec < 0)
2641                 goto fail;
2642
2643         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2644                 adapter->num_msix_roce_vec = num_vec / 2;
2645                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2646                          adapter->num_msix_roce_vec);
2647         }
2648
2649         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2650
2651         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2652                  adapter->num_msix_vec);
2653         return 0;
2654
2655 fail:
2656         dev_warn(dev, "MSIx enable failed\n");
2657
2658         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2659         if (!be_physfn(adapter))
2660                 return num_vec;
2661         return 0;
2662 }
2663
2664 static inline int be_msix_vec_get(struct be_adapter *adapter,
2665                                   struct be_eq_obj *eqo)
2666 {
2667         return adapter->msix_entries[eqo->msix_idx].vector;
2668 }
2669
2670 static int be_msix_register(struct be_adapter *adapter)
2671 {
2672         struct net_device *netdev = adapter->netdev;
2673         struct be_eq_obj *eqo;
2674         int status, i, vec;
2675
2676         for_all_evt_queues(adapter, eqo, i) {
2677                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2678                 vec = be_msix_vec_get(adapter, eqo);
2679                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2680                 if (status)
2681                         goto err_msix;
2682         }
2683
2684         return 0;
2685 err_msix:
2686         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2687                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2688         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2689                  status);
2690         be_msix_disable(adapter);
2691         return status;
2692 }
2693
2694 static int be_irq_register(struct be_adapter *adapter)
2695 {
2696         struct net_device *netdev = adapter->netdev;
2697         int status;
2698
2699         if (msix_enabled(adapter)) {
2700                 status = be_msix_register(adapter);
2701                 if (status == 0)
2702                         goto done;
2703                 /* INTx is not supported for VF */
2704                 if (!be_physfn(adapter))
2705                         return status;
2706         }
2707
2708         /* INTx: only the first EQ is used */
2709         netdev->irq = adapter->pdev->irq;
2710         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2711                              &adapter->eq_obj[0]);
2712         if (status) {
2713                 dev_err(&adapter->pdev->dev,
2714                         "INTx request IRQ failed - err %d\n", status);
2715                 return status;
2716         }
2717 done:
2718         adapter->isr_registered = true;
2719         return 0;
2720 }
2721
2722 static void be_irq_unregister(struct be_adapter *adapter)
2723 {
2724         struct net_device *netdev = adapter->netdev;
2725         struct be_eq_obj *eqo;
2726         int i;
2727
2728         if (!adapter->isr_registered)
2729                 return;
2730
2731         /* INTx */
2732         if (!msix_enabled(adapter)) {
2733                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2734                 goto done;
2735         }
2736
2737         /* MSIx */
2738         for_all_evt_queues(adapter, eqo, i)
2739                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2740
2741 done:
2742         adapter->isr_registered = false;
2743 }
2744
2745 static void be_rx_qs_destroy(struct be_adapter *adapter)
2746 {
2747         struct be_queue_info *q;
2748         struct be_rx_obj *rxo;
2749         int i;
2750
2751         for_all_rx_queues(adapter, rxo, i) {
2752                 q = &rxo->q;
2753                 if (q->created) {
2754                         be_cmd_rxq_destroy(adapter, q);
2755                         be_rx_cq_clean(rxo);
2756                 }
2757                 be_queue_free(adapter, q);
2758         }
2759 }
2760
2761 static int be_close(struct net_device *netdev)
2762 {
2763         struct be_adapter *adapter = netdev_priv(netdev);
2764         struct be_eq_obj *eqo;
2765         int i;
2766
2767         /* This protection is needed as be_close() may be called even when the
2768          * adapter is in cleared state (after eeh perm failure)
2769          */
2770         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2771                 return 0;
2772
2773         be_roce_dev_close(adapter);
2774
2775         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2776                 for_all_evt_queues(adapter, eqo, i) {
2777                         napi_disable(&eqo->napi);
2778                         be_disable_busy_poll(eqo);
2779                 }
2780                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2781         }
2782
2783         be_async_mcc_disable(adapter);
2784
2785         /* Wait for all pending tx completions to arrive so that
2786          * all tx skbs are freed.
2787          */
2788         netif_tx_disable(netdev);
2789         be_tx_compl_clean(adapter);
2790
2791         be_rx_qs_destroy(adapter);
2792
2793         for (i = 1; i < (adapter->uc_macs + 1); i++)
2794                 be_cmd_pmac_del(adapter, adapter->if_handle,
2795                                 adapter->pmac_id[i], 0);
2796         adapter->uc_macs = 0;
2797
2798         for_all_evt_queues(adapter, eqo, i) {
2799                 if (msix_enabled(adapter))
2800                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2801                 else
2802                         synchronize_irq(netdev->irq);
2803                 be_eq_clean(eqo);
2804         }
2805
2806         be_irq_unregister(adapter);
2807
2808         return 0;
2809 }
2810
2811 static int be_rx_qs_create(struct be_adapter *adapter)
2812 {
2813         struct be_rx_obj *rxo;
2814         int rc, i, j;
2815         u8 rss_hkey[RSS_HASH_KEY_LEN];
2816         struct rss_info *rss = &adapter->rss_info;
2817
2818         for_all_rx_queues(adapter, rxo, i) {
2819                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2820                                     sizeof(struct be_eth_rx_d));
2821                 if (rc)
2822                         return rc;
2823         }
2824
2825         /* The FW would like the default RXQ to be created first */
2826         rxo = default_rxo(adapter);
2827         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2828                                adapter->if_handle, false, &rxo->rss_id);
2829         if (rc)
2830                 return rc;
2831
2832         for_all_rss_queues(adapter, rxo, i) {
2833                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2834                                        rx_frag_size, adapter->if_handle,
2835                                        true, &rxo->rss_id);
2836                 if (rc)
2837                         return rc;
2838         }
2839
2840         if (be_multi_rxq(adapter)) {
2841                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2842                         j += adapter->num_rx_qs - 1) {
2843                         for_all_rss_queues(adapter, rxo, i) {
2844                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2845                                         break;
2846                                 rss->rsstable[j + i] = rxo->rss_id;
2847                                 rss->rss_queue[j + i] = i;
2848                         }
2849                 }
2850                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2851                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2852
2853                 if (!BEx_chip(adapter))
2854                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2855                                 RSS_ENABLE_UDP_IPV6;
2856         } else {
2857                 /* Disable RSS, if only default RX Q is created */
2858                 rss->rss_flags = RSS_ENABLE_NONE;
2859         }
2860
2861         get_random_bytes(rss_hkey, RSS_HASH_KEY_LEN);
2862         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2863                                128, rss_hkey);
2864         if (rc) {
2865                 rss->rss_flags = RSS_ENABLE_NONE;
2866                 return rc;
2867         }
2868
2869         memcpy(rss->rss_hkey, rss_hkey, RSS_HASH_KEY_LEN);
2870
2871         /* First time posting */
2872         for_all_rx_queues(adapter, rxo, i)
2873                 be_post_rx_frags(rxo, GFP_KERNEL);
2874         return 0;
2875 }
2876
2877 static int be_open(struct net_device *netdev)
2878 {
2879         struct be_adapter *adapter = netdev_priv(netdev);
2880         struct be_eq_obj *eqo;
2881         struct be_rx_obj *rxo;
2882         struct be_tx_obj *txo;
2883         u8 link_status;
2884         int status, i;
2885
2886         status = be_rx_qs_create(adapter);
2887         if (status)
2888                 goto err;
2889
2890         status = be_irq_register(adapter);
2891         if (status)
2892                 goto err;
2893
2894         for_all_rx_queues(adapter, rxo, i)
2895                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2896
2897         for_all_tx_queues(adapter, txo, i)
2898                 be_cq_notify(adapter, txo->cq.id, true, 0);
2899
2900         be_async_mcc_enable(adapter);
2901
2902         for_all_evt_queues(adapter, eqo, i) {
2903                 napi_enable(&eqo->napi);
2904                 be_enable_busy_poll(eqo);
2905                 be_eq_notify(adapter, eqo->q.id, true, false, 0);
2906         }
2907         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2908
2909         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2910         if (!status)
2911                 be_link_status_update(adapter, link_status);
2912
2913         netif_tx_start_all_queues(netdev);
2914         be_roce_dev_open(adapter);
2915
2916 #ifdef CONFIG_BE2NET_VXLAN
2917         if (skyhawk_chip(adapter))
2918                 vxlan_get_rx_port(netdev);
2919 #endif
2920
2921         return 0;
2922 err:
2923         be_close(adapter->netdev);
2924         return -EIO;
2925 }
2926
2927 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2928 {
2929         struct be_dma_mem cmd;
2930         int status = 0;
2931         u8 mac[ETH_ALEN];
2932
2933         memset(mac, 0, ETH_ALEN);
2934
2935         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2936         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2937                                      GFP_KERNEL);
2938         if (cmd.va == NULL)
2939                 return -1;
2940
2941         if (enable) {
2942                 status = pci_write_config_dword(adapter->pdev,
2943                                                 PCICFG_PM_CONTROL_OFFSET,
2944                                                 PCICFG_PM_CONTROL_MASK);
2945                 if (status) {
2946                         dev_err(&adapter->pdev->dev,
2947                                 "Could not enable Wake-on-lan\n");
2948                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2949                                           cmd.dma);
2950                         return status;
2951                 }
2952                 status = be_cmd_enable_magic_wol(adapter,
2953                                                  adapter->netdev->dev_addr,
2954                                                  &cmd);
2955                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2956                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2957         } else {
2958                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2959                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2960                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2961         }
2962
2963         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
2964         return status;
2965 }
2966
2967 /*
2968  * Generate a seed MAC address from the PF MAC Address using jhash.
2969  * MAC Address for VFs are assigned incrementally starting from the seed.
2970  * These addresses are programmed in the ASIC by the PF and the VF driver
2971  * queries for the MAC address during its probe.
2972  */
2973 static int be_vf_eth_addr_config(struct be_adapter *adapter)
2974 {
2975         u32 vf;
2976         int status = 0;
2977         u8 mac[ETH_ALEN];
2978         struct be_vf_cfg *vf_cfg;
2979
2980         be_vf_eth_addr_generate(adapter, mac);
2981
2982         for_all_vfs(adapter, vf_cfg, vf) {
2983                 if (BEx_chip(adapter))
2984                         status = be_cmd_pmac_add(adapter, mac,
2985                                                  vf_cfg->if_handle,
2986                                                  &vf_cfg->pmac_id, vf + 1);
2987                 else
2988                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
2989                                                 vf + 1);
2990
2991                 if (status)
2992                         dev_err(&adapter->pdev->dev,
2993                                 "Mac address assignment failed for VF %d\n",
2994                                 vf);
2995                 else
2996                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
2997
2998                 mac[5] += 1;
2999         }
3000         return status;
3001 }
3002
3003 static int be_vfs_mac_query(struct be_adapter *adapter)
3004 {
3005         int status, vf;
3006         u8 mac[ETH_ALEN];
3007         struct be_vf_cfg *vf_cfg;
3008
3009         for_all_vfs(adapter, vf_cfg, vf) {
3010                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3011                                                mac, vf_cfg->if_handle,
3012                                                false, vf+1);
3013                 if (status)
3014                         return status;
3015                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3016         }
3017         return 0;
3018 }
3019
3020 static void be_vf_clear(struct be_adapter *adapter)
3021 {
3022         struct be_vf_cfg *vf_cfg;
3023         u32 vf;
3024
3025         if (pci_vfs_assigned(adapter->pdev)) {
3026                 dev_warn(&adapter->pdev->dev,
3027                          "VFs are assigned to VMs: not disabling VFs\n");
3028                 goto done;
3029         }
3030
3031         pci_disable_sriov(adapter->pdev);
3032
3033         for_all_vfs(adapter, vf_cfg, vf) {
3034                 if (BEx_chip(adapter))
3035                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3036                                         vf_cfg->pmac_id, vf + 1);
3037                 else
3038                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3039                                        vf + 1);
3040
3041                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3042         }
3043 done:
3044         kfree(adapter->vf_cfg);
3045         adapter->num_vfs = 0;
3046 }
3047
3048 static void be_clear_queues(struct be_adapter *adapter)
3049 {
3050         be_mcc_queues_destroy(adapter);
3051         be_rx_cqs_destroy(adapter);
3052         be_tx_queues_destroy(adapter);
3053         be_evt_queues_destroy(adapter);
3054 }
3055
3056 static void be_cancel_worker(struct be_adapter *adapter)
3057 {
3058         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3059                 cancel_delayed_work_sync(&adapter->work);
3060                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3061         }
3062 }
3063
3064 static void be_mac_clear(struct be_adapter *adapter)
3065 {
3066         int i;
3067
3068         if (adapter->pmac_id) {
3069                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3070                         be_cmd_pmac_del(adapter, adapter->if_handle,
3071                                         adapter->pmac_id[i], 0);
3072                 adapter->uc_macs = 0;
3073
3074                 kfree(adapter->pmac_id);
3075                 adapter->pmac_id = NULL;
3076         }
3077 }
3078
3079 #ifdef CONFIG_BE2NET_VXLAN
3080 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3081 {
3082         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3083                 be_cmd_manage_iface(adapter, adapter->if_handle,
3084                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3085
3086         if (adapter->vxlan_port)
3087                 be_cmd_set_vxlan_port(adapter, 0);
3088
3089         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3090         adapter->vxlan_port = 0;
3091 }
3092 #endif
3093
3094 static int be_clear(struct be_adapter *adapter)
3095 {
3096         be_cancel_worker(adapter);
3097
3098         if (sriov_enabled(adapter))
3099                 be_vf_clear(adapter);
3100
3101 #ifdef CONFIG_BE2NET_VXLAN
3102         be_disable_vxlan_offloads(adapter);
3103 #endif
3104         /* delete the primary mac along with the uc-mac list */
3105         be_mac_clear(adapter);
3106
3107         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3108
3109         be_clear_queues(adapter);
3110
3111         be_msix_disable(adapter);
3112         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3113         return 0;
3114 }
3115
3116 static int be_vfs_if_create(struct be_adapter *adapter)
3117 {
3118         struct be_resources res = {0};
3119         struct be_vf_cfg *vf_cfg;
3120         u32 cap_flags, en_flags, vf;
3121         int status = 0;
3122
3123         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3124                     BE_IF_FLAGS_MULTICAST;
3125
3126         for_all_vfs(adapter, vf_cfg, vf) {
3127                 if (!BE3_chip(adapter)) {
3128                         status = be_cmd_get_profile_config(adapter, &res,
3129                                                            vf + 1);
3130                         if (!status)
3131                                 cap_flags = res.if_cap_flags;
3132                 }
3133
3134                 /* If a FW profile exists, then cap_flags are updated */
3135                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3136                                         BE_IF_FLAGS_BROADCAST |
3137                                         BE_IF_FLAGS_MULTICAST);
3138                 status =
3139                     be_cmd_if_create(adapter, cap_flags, en_flags,
3140                                      &vf_cfg->if_handle, vf + 1);
3141                 if (status)
3142                         goto err;
3143         }
3144 err:
3145         return status;
3146 }
3147
3148 static int be_vf_setup_init(struct be_adapter *adapter)
3149 {
3150         struct be_vf_cfg *vf_cfg;
3151         int vf;
3152
3153         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3154                                   GFP_KERNEL);
3155         if (!adapter->vf_cfg)
3156                 return -ENOMEM;
3157
3158         for_all_vfs(adapter, vf_cfg, vf) {
3159                 vf_cfg->if_handle = -1;
3160                 vf_cfg->pmac_id = -1;
3161         }
3162         return 0;
3163 }
3164
3165 static int be_vf_setup(struct be_adapter *adapter)
3166 {
3167         struct device *dev = &adapter->pdev->dev;
3168         struct be_vf_cfg *vf_cfg;
3169         int status, old_vfs, vf;
3170         u32 privileges;
3171
3172         old_vfs = pci_num_vf(adapter->pdev);
3173         if (old_vfs) {
3174                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3175                 if (old_vfs != num_vfs)
3176                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3177                 adapter->num_vfs = old_vfs;
3178         } else {
3179                 if (num_vfs > be_max_vfs(adapter))
3180                         dev_info(dev, "Device supports %d VFs and not %d\n",
3181                                  be_max_vfs(adapter), num_vfs);
3182                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3183                 if (!adapter->num_vfs)
3184                         return 0;
3185         }
3186
3187         status = be_vf_setup_init(adapter);
3188         if (status)
3189                 goto err;
3190
3191         if (old_vfs) {
3192                 for_all_vfs(adapter, vf_cfg, vf) {
3193                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3194                         if (status)
3195                                 goto err;
3196                 }
3197         } else {
3198                 status = be_vfs_if_create(adapter);
3199                 if (status)
3200                         goto err;
3201         }
3202
3203         if (old_vfs) {
3204                 status = be_vfs_mac_query(adapter);
3205                 if (status)
3206                         goto err;
3207         } else {
3208                 status = be_vf_eth_addr_config(adapter);
3209                 if (status)
3210                         goto err;
3211         }
3212
3213         for_all_vfs(adapter, vf_cfg, vf) {
3214                 /* Allow VFs to programs MAC/VLAN filters */
3215                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3216                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3217                         status = be_cmd_set_fn_privileges(adapter,
3218                                                           privileges |
3219                                                           BE_PRIV_FILTMGMT,
3220                                                           vf + 1);
3221                         if (!status)
3222                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3223                                          vf);
3224                 }
3225
3226                 /* Allow full available bandwidth */
3227                 if (!old_vfs)
3228                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3229
3230                 if (!old_vfs) {
3231                         be_cmd_enable_vf(adapter, vf + 1);
3232                         be_cmd_set_logical_link_config(adapter,
3233                                                        IFLA_VF_LINK_STATE_AUTO,
3234                                                        vf+1);
3235                 }
3236         }
3237
3238         if (!old_vfs) {
3239                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3240                 if (status) {
3241                         dev_err(dev, "SRIOV enable failed\n");
3242                         adapter->num_vfs = 0;
3243                         goto err;
3244                 }
3245         }
3246         return 0;
3247 err:
3248         dev_err(dev, "VF setup failed\n");
3249         be_vf_clear(adapter);
3250         return status;
3251 }
3252
3253 /* Converting function_mode bits on BE3 to SH mc_type enums */
3254
3255 static u8 be_convert_mc_type(u32 function_mode)
3256 {
3257         if (function_mode & VNIC_MODE && function_mode & FLEX10_MODE)
3258                 return vNIC1;
3259         else if (function_mode & FLEX10_MODE)
3260                 return FLEX10;
3261         else if (function_mode & VNIC_MODE)
3262                 return vNIC2;
3263         else if (function_mode & UMC_ENABLED)
3264                 return UMC;
3265         else
3266                 return MC_NONE;
3267 }
3268
3269 /* On BE2/BE3 FW does not suggest the supported limits */
3270 static void BEx_get_resources(struct be_adapter *adapter,
3271                               struct be_resources *res)
3272 {
3273         struct pci_dev *pdev = adapter->pdev;
3274         bool use_sriov = false;
3275         int max_vfs = 0;
3276
3277         if (be_physfn(adapter) && BE3_chip(adapter)) {
3278                 be_cmd_get_profile_config(adapter, res, 0);
3279                 /* Some old versions of BE3 FW don't report max_vfs value */
3280                 if (res->max_vfs == 0) {
3281                         max_vfs = pci_sriov_get_totalvfs(pdev);
3282                         res->max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3283                 }
3284                 use_sriov = res->max_vfs && sriov_want(adapter);
3285         }
3286
3287         if (be_physfn(adapter))
3288                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3289         else
3290                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3291
3292         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3293
3294         if (be_is_mc(adapter)) {
3295                 /* Assuming that there are 4 channels per port,
3296                  * when multi-channel is enabled
3297                  */
3298                 if (be_is_qnq_mode(adapter))
3299                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3300                 else
3301                         /* In a non-qnq multichannel mode, the pvid
3302                          * takes up one vlan entry
3303                          */
3304                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3305         } else {
3306                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3307         }
3308
3309         res->max_mcast_mac = BE_MAX_MC;
3310
3311         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3312          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3313          *    *only* if it is RSS-capable.
3314          */
3315         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3316             !be_physfn(adapter) || (be_is_mc(adapter) &&
3317             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS)))
3318                 res->max_tx_qs = 1;
3319         else
3320                 res->max_tx_qs = BE3_MAX_TX_QS;
3321
3322         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3323             !use_sriov && be_physfn(adapter))
3324                 res->max_rss_qs = (adapter->be3_native) ?
3325                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3326         res->max_rx_qs = res->max_rss_qs + 1;
3327
3328         if (be_physfn(adapter))
3329                 res->max_evt_qs = (res->max_vfs > 0) ?
3330                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3331         else
3332                 res->max_evt_qs = 1;
3333
3334         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3335         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3336                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3337 }
3338
3339 static void be_setup_init(struct be_adapter *adapter)
3340 {
3341         adapter->vlan_prio_bmap = 0xff;
3342         adapter->phy.link_speed = -1;
3343         adapter->if_handle = -1;
3344         adapter->be3_native = false;
3345         adapter->promiscuous = false;
3346         if (be_physfn(adapter))
3347                 adapter->cmd_privileges = MAX_PRIVILEGES;
3348         else
3349                 adapter->cmd_privileges = MIN_PRIVILEGES;
3350 }
3351
3352 static int be_get_resources(struct be_adapter *adapter)
3353 {
3354         struct device *dev = &adapter->pdev->dev;
3355         struct be_resources res = {0};
3356         int status;
3357
3358         if (BEx_chip(adapter)) {
3359                 BEx_get_resources(adapter, &res);
3360                 adapter->res = res;
3361         }
3362
3363         /* For Lancer, SH etc read per-function resource limits from FW.
3364          * GET_FUNC_CONFIG returns per function guaranteed limits.
3365          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3366          */
3367         if (!BEx_chip(adapter)) {
3368                 status = be_cmd_get_func_config(adapter, &res);
3369                 if (status)
3370                         return status;
3371
3372                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3373                 if (be_roce_supported(adapter))
3374                         res.max_evt_qs /= 2;
3375                 adapter->res = res;
3376
3377                 if (be_physfn(adapter)) {
3378                         status = be_cmd_get_profile_config(adapter, &res, 0);
3379                         if (status)
3380                                 return status;
3381                         adapter->res.max_vfs = res.max_vfs;
3382                 }
3383
3384                 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3385                          be_max_txqs(adapter), be_max_rxqs(adapter),
3386                          be_max_rss(adapter), be_max_eqs(adapter),
3387                          be_max_vfs(adapter));
3388                 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3389                          be_max_uc(adapter), be_max_mc(adapter),
3390                          be_max_vlans(adapter));
3391         }
3392
3393         return 0;
3394 }
3395
3396 /* Routine to query per function resource limits */
3397 static int be_get_config(struct be_adapter *adapter)
3398 {
3399         u16 profile_id;
3400         int status;
3401
3402         status = be_cmd_query_fw_cfg(adapter, &adapter->port_num,
3403                                      &adapter->function_mode,
3404                                      &adapter->function_caps,
3405                                      &adapter->asic_rev);
3406         if (status)
3407                 return status;
3408
3409          if (be_physfn(adapter)) {
3410                 status = be_cmd_get_active_profile(adapter, &profile_id);
3411                 if (!status)
3412                         dev_info(&adapter->pdev->dev,
3413                                  "Using profile 0x%x\n", profile_id);
3414         }
3415
3416         status = be_get_resources(adapter);
3417         if (status)
3418                 return status;
3419
3420         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3421                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3422         if (!adapter->pmac_id)
3423                 return -ENOMEM;
3424
3425         /* Sanitize cfg_num_qs based on HW and platform limits */
3426         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3427
3428         return 0;
3429 }
3430
3431 static int be_mac_setup(struct be_adapter *adapter)
3432 {
3433         u8 mac[ETH_ALEN];
3434         int status;
3435
3436         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3437                 status = be_cmd_get_perm_mac(adapter, mac);
3438                 if (status)
3439                         return status;
3440
3441                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3442                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3443         } else {
3444                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3445                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3446         }
3447
3448         /* For BE3-R VFs, the PF programs the initial MAC address */
3449         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3450                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3451                                 &adapter->pmac_id[0], 0);
3452         return 0;
3453 }
3454
3455 static void be_schedule_worker(struct be_adapter *adapter)
3456 {
3457         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3458         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3459 }
3460
3461 static int be_setup_queues(struct be_adapter *adapter)
3462 {
3463         struct net_device *netdev = adapter->netdev;
3464         int status;
3465
3466         status = be_evt_queues_create(adapter);
3467         if (status)
3468                 goto err;
3469
3470         status = be_tx_qs_create(adapter);
3471         if (status)
3472                 goto err;
3473
3474         status = be_rx_cqs_create(adapter);
3475         if (status)
3476                 goto err;
3477
3478         status = be_mcc_queues_create(adapter);
3479         if (status)
3480                 goto err;
3481
3482         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3483         if (status)
3484                 goto err;
3485
3486         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3487         if (status)
3488                 goto err;
3489
3490         return 0;
3491 err:
3492         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3493         return status;
3494 }
3495
3496 int be_update_queues(struct be_adapter *adapter)
3497 {
3498         struct net_device *netdev = adapter->netdev;
3499         int status;
3500
3501         if (netif_running(netdev))
3502                 be_close(netdev);
3503
3504         be_cancel_worker(adapter);
3505
3506         /* If any vectors have been shared with RoCE we cannot re-program
3507          * the MSIx table.
3508          */
3509         if (!adapter->num_msix_roce_vec)
3510                 be_msix_disable(adapter);
3511
3512         be_clear_queues(adapter);
3513
3514         if (!msix_enabled(adapter)) {
3515                 status = be_msix_enable(adapter);
3516                 if (status)
3517                         return status;
3518         }
3519
3520         status = be_setup_queues(adapter);
3521         if (status)
3522                 return status;
3523
3524         be_schedule_worker(adapter);
3525
3526         if (netif_running(netdev))
3527                 status = be_open(netdev);
3528
3529         return status;
3530 }
3531
3532 static int be_setup(struct be_adapter *adapter)
3533 {
3534         struct device *dev = &adapter->pdev->dev;
3535         u32 tx_fc, rx_fc, en_flags;
3536         int status;
3537
3538         be_setup_init(adapter);
3539
3540         if (!lancer_chip(adapter))
3541                 be_cmd_req_native_mode(adapter);
3542
3543         status = be_get_config(adapter);
3544         if (status)
3545                 goto err;
3546
3547         status = be_msix_enable(adapter);
3548         if (status)
3549                 goto err;
3550
3551         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3552                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3553         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3554                 en_flags |= BE_IF_FLAGS_RSS;
3555         en_flags = en_flags & be_if_cap_flags(adapter);
3556         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3557                                   &adapter->if_handle, 0);
3558         if (status)
3559                 goto err;
3560
3561         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3562         rtnl_lock();
3563         status = be_setup_queues(adapter);
3564         rtnl_unlock();
3565         if (status)
3566                 goto err;
3567
3568         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3569
3570         status = be_mac_setup(adapter);
3571         if (status)
3572                 goto err;
3573
3574         be_cmd_get_fw_ver(adapter, adapter->fw_ver, adapter->fw_on_flash);
3575
3576         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3577                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work.",
3578                         adapter->fw_ver);
3579                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3580         }
3581
3582         if (adapter->vlans_added)
3583                 be_vid_config(adapter);
3584
3585         be_set_rx_mode(adapter->netdev);
3586
3587         be_cmd_get_acpi_wol_cap(adapter);
3588
3589         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3590
3591         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3592                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3593                                         adapter->rx_fc);
3594
3595         if (be_physfn(adapter))
3596                 be_cmd_set_logical_link_config(adapter,
3597                                                IFLA_VF_LINK_STATE_AUTO, 0);
3598
3599         if (sriov_want(adapter)) {
3600                 if (be_max_vfs(adapter))
3601                         be_vf_setup(adapter);
3602                 else
3603                         dev_warn(dev, "device doesn't support SRIOV\n");
3604         }
3605
3606         status = be_cmd_get_phy_info(adapter);
3607         if (!status && be_pause_supported(adapter))
3608                 adapter->phy.fc_autoneg = 1;
3609
3610         be_schedule_worker(adapter);
3611         adapter->flags |= BE_FLAGS_SETUP_DONE;
3612         return 0;
3613 err:
3614         be_clear(adapter);
3615         return status;
3616 }
3617
3618 #ifdef CONFIG_NET_POLL_CONTROLLER
3619 static void be_netpoll(struct net_device *netdev)
3620 {
3621         struct be_adapter *adapter = netdev_priv(netdev);
3622         struct be_eq_obj *eqo;
3623         int i;
3624
3625         for_all_evt_queues(adapter, eqo, i) {
3626                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3627                 napi_schedule(&eqo->napi);
3628         }
3629
3630         return;
3631 }
3632 #endif
3633
3634 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3635
3636 static bool phy_flashing_required(struct be_adapter *adapter)
3637 {
3638         return (adapter->phy.phy_type == TN_8022 &&
3639                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3640 }
3641
3642 static bool is_comp_in_ufi(struct be_adapter *adapter,
3643                            struct flash_section_info *fsec, int type)
3644 {
3645         int i = 0, img_type = 0;
3646         struct flash_section_info_g2 *fsec_g2 = NULL;
3647
3648         if (BE2_chip(adapter))
3649                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3650
3651         for (i = 0; i < MAX_FLASH_COMP; i++) {
3652                 if (fsec_g2)
3653                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3654                 else
3655                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3656
3657                 if (img_type == type)
3658                         return true;
3659         }
3660         return false;
3661
3662 }
3663
3664 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3665                                                 int header_size,
3666                                                 const struct firmware *fw)
3667 {
3668         struct flash_section_info *fsec = NULL;
3669         const u8 *p = fw->data;
3670
3671         p += header_size;
3672         while (p < (fw->data + fw->size)) {
3673                 fsec = (struct flash_section_info *)p;
3674                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3675                         return fsec;
3676                 p += 32;
3677         }
3678         return NULL;
3679 }
3680
3681 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3682                               u32 img_offset, u32 img_size, int hdr_size,
3683                               u16 img_optype, bool *crc_match)
3684 {
3685         u32 crc_offset;
3686         int status;
3687         u8 crc[4];
3688
3689         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3690         if (status)
3691                 return status;
3692
3693         crc_offset = hdr_size + img_offset + img_size - 4;
3694
3695         /* Skip flashing, if crc of flashed region matches */
3696         if (!memcmp(crc, p + crc_offset, 4))
3697                 *crc_match = true;
3698         else
3699                 *crc_match = false;
3700
3701         return status;
3702 }
3703
3704 static int be_flash(struct be_adapter *adapter, const u8 *img,
3705                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3706 {
3707         struct be_cmd_write_flashrom *req = flash_cmd->va;
3708         u32 total_bytes, flash_op, num_bytes;
3709         int status;
3710
3711         total_bytes = img_size;
3712         while (total_bytes) {
3713                 num_bytes = min_t(u32, 32*1024, total_bytes);
3714
3715                 total_bytes -= num_bytes;
3716
3717                 if (!total_bytes) {
3718                         if (optype == OPTYPE_PHY_FW)
3719                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3720                         else
3721                                 flash_op = FLASHROM_OPER_FLASH;
3722                 } else {
3723                         if (optype == OPTYPE_PHY_FW)
3724                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3725                         else
3726                                 flash_op = FLASHROM_OPER_SAVE;
3727                 }
3728
3729                 memcpy(req->data_buf, img, num_bytes);
3730                 img += num_bytes;
3731                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3732                                                flash_op, num_bytes);
3733                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3734                     optype == OPTYPE_PHY_FW)
3735                         break;
3736                 else if (status)
3737                         return status;
3738         }
3739         return 0;
3740 }
3741
3742 /* For BE2, BE3 and BE3-R */
3743 static int be_flash_BEx(struct be_adapter *adapter,
3744                         const struct firmware *fw,
3745                         struct be_dma_mem *flash_cmd, int num_of_images)
3746 {
3747         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3748         struct device *dev = &adapter->pdev->dev;
3749         struct flash_section_info *fsec = NULL;
3750         int status, i, filehdr_size, num_comp;
3751         const struct flash_comp *pflashcomp;
3752         bool crc_match;
3753         const u8 *p;
3754
3755         struct flash_comp gen3_flash_types[] = {
3756                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3757                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3758                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3759                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3760                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3761                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3762                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3763                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3764                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3765                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3766                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3767                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3768                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3769                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3770                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3771                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3772                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3773                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3774                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3775                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3776         };
3777
3778         struct flash_comp gen2_flash_types[] = {
3779                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3780                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3781                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3782                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3783                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3784                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3785                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3786                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3787                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3788                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3789                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3790                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3791                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3792                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3793                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3794                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3795         };
3796
3797         if (BE3_chip(adapter)) {
3798                 pflashcomp = gen3_flash_types;
3799                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3800                 num_comp = ARRAY_SIZE(gen3_flash_types);
3801         } else {
3802                 pflashcomp = gen2_flash_types;
3803                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3804                 num_comp = ARRAY_SIZE(gen2_flash_types);
3805         }
3806
3807         /* Get flash section info*/
3808         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3809         if (!fsec) {
3810                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3811                 return -1;
3812         }
3813         for (i = 0; i < num_comp; i++) {
3814                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3815                         continue;
3816
3817                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3818                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3819                         continue;
3820
3821                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3822                     !phy_flashing_required(adapter))
3823                                 continue;
3824
3825                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3826                         status = be_check_flash_crc(adapter, fw->data,
3827                                                     pflashcomp[i].offset,
3828                                                     pflashcomp[i].size,
3829                                                     filehdr_size +
3830                                                     img_hdrs_size,
3831                                                     OPTYPE_REDBOOT, &crc_match);
3832                         if (status) {
3833                                 dev_err(dev,
3834                                         "Could not get CRC for 0x%x region\n",
3835                                         pflashcomp[i].optype);
3836                                 continue;
3837                         }
3838
3839                         if (crc_match)
3840                                 continue;
3841                 }
3842
3843                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3844                         img_hdrs_size;
3845                 if (p + pflashcomp[i].size > fw->data + fw->size)
3846                         return -1;
3847
3848                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3849                                   pflashcomp[i].size);
3850                 if (status) {
3851                         dev_err(dev, "Flashing section type 0x%x failed\n",
3852                                 pflashcomp[i].img_type);
3853                         return status;
3854                 }
3855         }
3856         return 0;
3857 }
3858
3859 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3860 {
3861         u32 img_type = le32_to_cpu(fsec_entry.type);
3862         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3863
3864         if (img_optype != 0xFFFF)
3865                 return img_optype;
3866
3867         switch (img_type) {
3868         case IMAGE_FIRMWARE_iSCSI:
3869                 img_optype = OPTYPE_ISCSI_ACTIVE;
3870                 break;
3871         case IMAGE_BOOT_CODE:
3872                 img_optype = OPTYPE_REDBOOT;
3873                 break;
3874         case IMAGE_OPTION_ROM_ISCSI:
3875                 img_optype = OPTYPE_BIOS;
3876                 break;
3877         case IMAGE_OPTION_ROM_PXE:
3878                 img_optype = OPTYPE_PXE_BIOS;
3879                 break;
3880         case IMAGE_OPTION_ROM_FCoE:
3881                 img_optype = OPTYPE_FCOE_BIOS;
3882                 break;
3883         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3884                 img_optype = OPTYPE_ISCSI_BACKUP;
3885                 break;
3886         case IMAGE_NCSI:
3887                 img_optype = OPTYPE_NCSI_FW;
3888                 break;
3889         case IMAGE_FLASHISM_JUMPVECTOR:
3890                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3891                 break;
3892         case IMAGE_FIRMWARE_PHY:
3893                 img_optype = OPTYPE_SH_PHY_FW;
3894                 break;
3895         case IMAGE_REDBOOT_DIR:
3896                 img_optype = OPTYPE_REDBOOT_DIR;
3897                 break;
3898         case IMAGE_REDBOOT_CONFIG:
3899                 img_optype = OPTYPE_REDBOOT_CONFIG;
3900                 break;
3901         case IMAGE_UFI_DIR:
3902                 img_optype = OPTYPE_UFI_DIR;
3903                 break;
3904         default:
3905                 break;
3906         }
3907
3908         return img_optype;
3909 }
3910
3911 static int be_flash_skyhawk(struct be_adapter *adapter,
3912                             const struct firmware *fw,
3913                             struct be_dma_mem *flash_cmd, int num_of_images)
3914 {
3915         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
3916         struct device *dev = &adapter->pdev->dev;
3917         struct flash_section_info *fsec = NULL;
3918         u32 img_offset, img_size, img_type;
3919         int status, i, filehdr_size;
3920         bool crc_match, old_fw_img;
3921         u16 img_optype;
3922         const u8 *p;
3923
3924         filehdr_size = sizeof(struct flash_file_hdr_g3);
3925         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3926         if (!fsec) {
3927                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3928                 return -1;
3929         }
3930
3931         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
3932                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
3933                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
3934                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
3935                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
3936                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
3937
3938                 if (img_optype == 0xFFFF)
3939                         continue;
3940                 /* Don't bother verifying CRC if an old FW image is being
3941                  * flashed
3942                  */
3943                 if (old_fw_img)
3944                         goto flash;
3945
3946                 status = be_check_flash_crc(adapter, fw->data, img_offset,
3947                                             img_size, filehdr_size +
3948                                             img_hdrs_size, img_optype,
3949                                             &crc_match);
3950                 /* The current FW image on the card does not recognize the new
3951                  * FLASH op_type. The FW download is partially complete.
3952                  * Reboot the server now to enable FW image to recognize the
3953                  * new FLASH op_type. To complete the remaining process,
3954                  * download the same FW again after the reboot.
3955                  */
3956                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
3957                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
3958                         dev_err(dev, "Flash incomplete. Reset the server\n");
3959                         dev_err(dev, "Download FW image again after reset\n");
3960                         return -EAGAIN;
3961                 } else if (status) {
3962                         dev_err(dev, "Could not get CRC for 0x%x region\n",
3963                                 img_optype);
3964                         return -EFAULT;
3965                 }
3966
3967                 if (crc_match)
3968                         continue;
3969
3970 flash:
3971                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
3972                 if (p + img_size > fw->data + fw->size)
3973                         return -1;
3974
3975                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
3976                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
3977                  * UFI_DIR region
3978                  */
3979                 if (old_fw_img &&
3980                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
3981                      (img_optype == OPTYPE_UFI_DIR &&
3982                       base_status(status) == MCC_STATUS_FAILED))) {
3983                         continue;
3984                 } else if (status) {
3985                         dev_err(dev, "Flashing section type 0x%x failed\n",
3986                                 img_type);
3987                         return -EFAULT;
3988                 }
3989         }
3990         return 0;
3991 }
3992
3993 static int lancer_fw_download(struct be_adapter *adapter,
3994                               const struct firmware *fw)
3995 {
3996 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
3997 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
3998         struct be_dma_mem flash_cmd;
3999         const u8 *data_ptr = NULL;
4000         u8 *dest_image_ptr = NULL;
4001         size_t image_size = 0;
4002         u32 chunk_size = 0;
4003         u32 data_written = 0;
4004         u32 offset = 0;
4005         int status = 0;
4006         u8 add_status = 0;
4007         u8 change_status;
4008
4009         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4010                 dev_err(&adapter->pdev->dev,
4011                         "FW Image not properly aligned. "
4012                         "Length must be 4 byte aligned.\n");
4013                 status = -EINVAL;
4014                 goto lancer_fw_exit;
4015         }
4016
4017         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4018                                 + LANCER_FW_DOWNLOAD_CHUNK;
4019         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4020                                           &flash_cmd.dma, GFP_KERNEL);
4021         if (!flash_cmd.va) {
4022                 status = -ENOMEM;
4023                 goto lancer_fw_exit;
4024         }
4025
4026         dest_image_ptr = flash_cmd.va +
4027                                 sizeof(struct lancer_cmd_req_write_object);
4028         image_size = fw->size;
4029         data_ptr = fw->data;
4030
4031         while (image_size) {
4032                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4033
4034                 /* Copy the image chunk content. */
4035                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4036
4037                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4038                                                  chunk_size, offset,
4039                                                  LANCER_FW_DOWNLOAD_LOCATION,
4040                                                  &data_written, &change_status,
4041                                                  &add_status);
4042                 if (status)
4043                         break;
4044
4045                 offset += data_written;
4046                 data_ptr += data_written;
4047                 image_size -= data_written;
4048         }
4049
4050         if (!status) {
4051                 /* Commit the FW written */
4052                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4053                                                  0, offset,
4054                                                  LANCER_FW_DOWNLOAD_LOCATION,
4055                                                  &data_written, &change_status,
4056                                                  &add_status);
4057         }
4058
4059         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4060                           flash_cmd.dma);
4061         if (status) {
4062                 dev_err(&adapter->pdev->dev,
4063                         "Firmware load error. "
4064                         "Status code: 0x%x Additional Status: 0x%x\n",
4065                         status, add_status);
4066                 goto lancer_fw_exit;
4067         }
4068
4069         if (change_status == LANCER_FW_RESET_NEEDED) {
4070                 dev_info(&adapter->pdev->dev,
4071                          "Resetting adapter to activate new FW\n");
4072                 status = lancer_physdev_ctrl(adapter,
4073                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4074                 if (status) {
4075                         dev_err(&adapter->pdev->dev,
4076                                 "Adapter busy for FW reset.\n"
4077                                 "New FW will not be active.\n");
4078                         goto lancer_fw_exit;
4079                 }
4080         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4081                 dev_err(&adapter->pdev->dev,
4082                         "System reboot required for new FW to be active\n");
4083         }
4084
4085         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4086 lancer_fw_exit:
4087         return status;
4088 }
4089
4090 #define UFI_TYPE2               2
4091 #define UFI_TYPE3               3
4092 #define UFI_TYPE3R              10
4093 #define UFI_TYPE4               4
4094 static int be_get_ufi_type(struct be_adapter *adapter,
4095                            struct flash_file_hdr_g3 *fhdr)
4096 {
4097         if (fhdr == NULL)
4098                 goto be_get_ufi_exit;
4099
4100         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4101                 return UFI_TYPE4;
4102         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4103                 if (fhdr->asic_type_rev == 0x10)
4104                         return UFI_TYPE3R;
4105                 else
4106                         return UFI_TYPE3;
4107         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4108                 return UFI_TYPE2;
4109
4110 be_get_ufi_exit:
4111         dev_err(&adapter->pdev->dev,
4112                 "UFI and Interface are not compatible for flashing\n");
4113         return -1;
4114 }
4115
4116 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4117 {
4118         struct flash_file_hdr_g3 *fhdr3;
4119         struct image_hdr *img_hdr_ptr = NULL;
4120         struct be_dma_mem flash_cmd;
4121         const u8 *p;
4122         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4123
4124         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4125         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4126                                           &flash_cmd.dma, GFP_KERNEL);
4127         if (!flash_cmd.va) {
4128                 status = -ENOMEM;
4129                 goto be_fw_exit;
4130         }
4131
4132         p = fw->data;
4133         fhdr3 = (struct flash_file_hdr_g3 *)p;
4134
4135         ufi_type = be_get_ufi_type(adapter, fhdr3);
4136
4137         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4138         for (i = 0; i < num_imgs; i++) {
4139                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4140                                 (sizeof(struct flash_file_hdr_g3) +
4141                                  i * sizeof(struct image_hdr)));
4142                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4143                         switch (ufi_type) {
4144                         case UFI_TYPE4:
4145                                 status = be_flash_skyhawk(adapter, fw,
4146                                                           &flash_cmd, num_imgs);
4147                                 break;
4148                         case UFI_TYPE3R:
4149                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4150                                                       num_imgs);
4151                                 break;
4152                         case UFI_TYPE3:
4153                                 /* Do not flash this ufi on BE3-R cards */
4154                                 if (adapter->asic_rev < 0x10)
4155                                         status = be_flash_BEx(adapter, fw,
4156                                                               &flash_cmd,
4157                                                               num_imgs);
4158                                 else {
4159                                         status = -1;
4160                                         dev_err(&adapter->pdev->dev,
4161                                                 "Can't load BE3 UFI on BE3R\n");
4162                                 }
4163                         }
4164                 }
4165         }
4166
4167         if (ufi_type == UFI_TYPE2)
4168                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4169         else if (ufi_type == -1)
4170                 status = -1;
4171
4172         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4173                           flash_cmd.dma);
4174         if (status) {
4175                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4176                 goto be_fw_exit;
4177         }
4178
4179         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4180
4181 be_fw_exit:
4182         return status;
4183 }
4184
4185 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4186 {
4187         const struct firmware *fw;
4188         int status;
4189
4190         if (!netif_running(adapter->netdev)) {
4191                 dev_err(&adapter->pdev->dev,
4192                         "Firmware load not allowed (interface is down)\n");
4193                 return -1;
4194         }
4195
4196         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4197         if (status)
4198                 goto fw_exit;
4199
4200         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4201
4202         if (lancer_chip(adapter))
4203                 status = lancer_fw_download(adapter, fw);
4204         else
4205                 status = be_fw_download(adapter, fw);
4206
4207         if (!status)
4208                 be_cmd_get_fw_ver(adapter, adapter->fw_ver,
4209                                   adapter->fw_on_flash);
4210
4211 fw_exit:
4212         release_firmware(fw);
4213         return status;
4214 }
4215
4216 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4217 {
4218         struct be_adapter *adapter = netdev_priv(dev);
4219         struct nlattr *attr, *br_spec;
4220         int rem;
4221         int status = 0;
4222         u16 mode = 0;
4223
4224         if (!sriov_enabled(adapter))
4225                 return -EOPNOTSUPP;
4226
4227         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4228
4229         nla_for_each_nested(attr, br_spec, rem) {
4230                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4231                         continue;
4232
4233                 mode = nla_get_u16(attr);
4234                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4235                         return -EINVAL;
4236
4237                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4238                                                adapter->if_handle,
4239                                                mode == BRIDGE_MODE_VEPA ?
4240                                                PORT_FWD_TYPE_VEPA :
4241                                                PORT_FWD_TYPE_VEB);
4242                 if (status)
4243                         goto err;
4244
4245                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4246                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4247
4248                 return status;
4249         }
4250 err:
4251         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4252                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4253
4254         return status;
4255 }
4256
4257 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4258                                  struct net_device *dev, u32 filter_mask)
4259 {
4260         struct be_adapter *adapter = netdev_priv(dev);
4261         int status = 0;
4262         u8 hsw_mode;
4263
4264         if (!sriov_enabled(adapter))
4265                 return 0;
4266
4267         /* BE and Lancer chips support VEB mode only */
4268         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4269                 hsw_mode = PORT_FWD_TYPE_VEB;
4270         } else {
4271                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4272                                                adapter->if_handle, &hsw_mode);
4273                 if (status)
4274                         return 0;
4275         }
4276
4277         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4278                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4279                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4280 }
4281
4282 #ifdef CONFIG_BE2NET_VXLAN
4283 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4284                               __be16 port)
4285 {
4286         struct be_adapter *adapter = netdev_priv(netdev);
4287         struct device *dev = &adapter->pdev->dev;
4288         int status;
4289
4290         if (lancer_chip(adapter) || BEx_chip(adapter))
4291                 return;
4292
4293         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4294                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4295                          be16_to_cpu(port));
4296                 dev_info(dev,
4297                          "Only one UDP port supported for VxLAN offloads\n");
4298                 return;
4299         }
4300
4301         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4302                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4303         if (status) {
4304                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4305                 goto err;
4306         }
4307
4308         status = be_cmd_set_vxlan_port(adapter, port);
4309         if (status) {
4310                 dev_warn(dev, "Failed to add VxLAN port\n");
4311                 goto err;
4312         }
4313         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4314         adapter->vxlan_port = port;
4315
4316         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4317                  be16_to_cpu(port));
4318         return;
4319 err:
4320         be_disable_vxlan_offloads(adapter);
4321         return;
4322 }
4323
4324 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4325                               __be16 port)
4326 {
4327         struct be_adapter *adapter = netdev_priv(netdev);
4328
4329         if (lancer_chip(adapter) || BEx_chip(adapter))
4330                 return;
4331
4332         if (adapter->vxlan_port != port)
4333                 return;
4334
4335         be_disable_vxlan_offloads(adapter);
4336
4337         dev_info(&adapter->pdev->dev,
4338                  "Disabled VxLAN offloads for UDP port %d\n",
4339                  be16_to_cpu(port));
4340 }
4341 #endif
4342
4343 static const struct net_device_ops be_netdev_ops = {
4344         .ndo_open               = be_open,
4345         .ndo_stop               = be_close,
4346         .ndo_start_xmit         = be_xmit,
4347         .ndo_set_rx_mode        = be_set_rx_mode,
4348         .ndo_set_mac_address    = be_mac_addr_set,
4349         .ndo_change_mtu         = be_change_mtu,
4350         .ndo_get_stats64        = be_get_stats64,
4351         .ndo_validate_addr      = eth_validate_addr,
4352         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4353         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4354         .ndo_set_vf_mac         = be_set_vf_mac,
4355         .ndo_set_vf_vlan        = be_set_vf_vlan,
4356         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4357         .ndo_get_vf_config      = be_get_vf_config,
4358         .ndo_set_vf_link_state  = be_set_vf_link_state,
4359 #ifdef CONFIG_NET_POLL_CONTROLLER
4360         .ndo_poll_controller    = be_netpoll,
4361 #endif
4362         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4363         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4364 #ifdef CONFIG_NET_RX_BUSY_POLL
4365         .ndo_busy_poll          = be_busy_poll,
4366 #endif
4367 #ifdef CONFIG_BE2NET_VXLAN
4368         .ndo_add_vxlan_port     = be_add_vxlan_port,
4369         .ndo_del_vxlan_port     = be_del_vxlan_port,
4370 #endif
4371 };
4372
4373 static void be_netdev_init(struct net_device *netdev)
4374 {
4375         struct be_adapter *adapter = netdev_priv(netdev);
4376
4377         if (skyhawk_chip(adapter)) {
4378                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4379                                            NETIF_F_TSO | NETIF_F_TSO6 |
4380                                            NETIF_F_GSO_UDP_TUNNEL;
4381                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4382         }
4383         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4384                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4385                 NETIF_F_HW_VLAN_CTAG_TX;
4386         if (be_multi_rxq(adapter))
4387                 netdev->hw_features |= NETIF_F_RXHASH;
4388
4389         netdev->features |= netdev->hw_features |
4390                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4391
4392         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4393                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4394
4395         netdev->priv_flags |= IFF_UNICAST_FLT;
4396
4397         netdev->flags |= IFF_MULTICAST;
4398
4399         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4400
4401         netdev->netdev_ops = &be_netdev_ops;
4402
4403         netdev->ethtool_ops = &be_ethtool_ops;
4404 }
4405
4406 static void be_unmap_pci_bars(struct be_adapter *adapter)
4407 {
4408         if (adapter->csr)
4409                 pci_iounmap(adapter->pdev, adapter->csr);
4410         if (adapter->db)
4411                 pci_iounmap(adapter->pdev, adapter->db);
4412 }
4413
4414 static int db_bar(struct be_adapter *adapter)
4415 {
4416         if (lancer_chip(adapter) || !be_physfn(adapter))
4417                 return 0;
4418         else
4419                 return 4;
4420 }
4421
4422 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4423 {
4424         if (skyhawk_chip(adapter)) {
4425                 adapter->roce_db.size = 4096;
4426                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4427                                                               db_bar(adapter));
4428                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4429                                                                db_bar(adapter));
4430         }
4431         return 0;
4432 }
4433
4434 static int be_map_pci_bars(struct be_adapter *adapter)
4435 {
4436         u8 __iomem *addr;
4437
4438         if (BEx_chip(adapter) && be_physfn(adapter)) {
4439                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4440                 if (adapter->csr == NULL)
4441                         return -ENOMEM;
4442         }
4443
4444         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4445         if (addr == NULL)
4446                 goto pci_map_err;
4447         adapter->db = addr;
4448
4449         be_roce_map_pci_bars(adapter);
4450         return 0;
4451
4452 pci_map_err:
4453         be_unmap_pci_bars(adapter);
4454         return -ENOMEM;
4455 }
4456
4457 static void be_ctrl_cleanup(struct be_adapter *adapter)
4458 {
4459         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4460
4461         be_unmap_pci_bars(adapter);
4462
4463         if (mem->va)
4464                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4465                                   mem->dma);
4466
4467         mem = &adapter->rx_filter;
4468         if (mem->va)
4469                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4470                                   mem->dma);
4471 }
4472
4473 static int be_ctrl_init(struct be_adapter *adapter)
4474 {
4475         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4476         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4477         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4478         u32 sli_intf;
4479         int status;
4480
4481         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4482         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4483                                  SLI_INTF_FAMILY_SHIFT;
4484         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4485
4486         status = be_map_pci_bars(adapter);
4487         if (status)
4488                 goto done;
4489
4490         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4491         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4492                                                 mbox_mem_alloc->size,
4493                                                 &mbox_mem_alloc->dma,
4494                                                 GFP_KERNEL);
4495         if (!mbox_mem_alloc->va) {
4496                 status = -ENOMEM;
4497                 goto unmap_pci_bars;
4498         }
4499         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4500         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4501         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4502         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4503
4504         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4505         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4506                                             rx_filter->size, &rx_filter->dma,
4507                                             GFP_KERNEL);
4508         if (rx_filter->va == NULL) {
4509                 status = -ENOMEM;
4510                 goto free_mbox;
4511         }
4512
4513         mutex_init(&adapter->mbox_lock);
4514         spin_lock_init(&adapter->mcc_lock);
4515         spin_lock_init(&adapter->mcc_cq_lock);
4516
4517         init_completion(&adapter->et_cmd_compl);
4518         pci_save_state(adapter->pdev);
4519         return 0;
4520
4521 free_mbox:
4522         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4523                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4524
4525 unmap_pci_bars:
4526         be_unmap_pci_bars(adapter);
4527
4528 done:
4529         return status;
4530 }
4531
4532 static void be_stats_cleanup(struct be_adapter *adapter)
4533 {
4534         struct be_dma_mem *cmd = &adapter->stats_cmd;
4535
4536         if (cmd->va)
4537                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4538                                   cmd->va, cmd->dma);
4539 }
4540
4541 static int be_stats_init(struct be_adapter *adapter)
4542 {
4543         struct be_dma_mem *cmd = &adapter->stats_cmd;
4544
4545         if (lancer_chip(adapter))
4546                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4547         else if (BE2_chip(adapter))
4548                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4549         else if (BE3_chip(adapter))
4550                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4551         else
4552                 /* ALL non-BE ASICs */
4553                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4554
4555         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4556                                       GFP_KERNEL);
4557         if (cmd->va == NULL)
4558                 return -1;
4559         return 0;
4560 }
4561
4562 static void be_remove(struct pci_dev *pdev)
4563 {
4564         struct be_adapter *adapter = pci_get_drvdata(pdev);
4565
4566         if (!adapter)
4567                 return;
4568
4569         be_roce_dev_remove(adapter);
4570         be_intr_set(adapter, false);
4571
4572         cancel_delayed_work_sync(&adapter->func_recovery_work);
4573
4574         unregister_netdev(adapter->netdev);
4575
4576         be_clear(adapter);
4577
4578         /* tell fw we're done with firing cmds */
4579         be_cmd_fw_clean(adapter);
4580
4581         be_stats_cleanup(adapter);
4582
4583         be_ctrl_cleanup(adapter);
4584
4585         pci_disable_pcie_error_reporting(pdev);
4586
4587         pci_release_regions(pdev);
4588         pci_disable_device(pdev);
4589
4590         free_netdev(adapter->netdev);
4591 }
4592
4593 static int be_get_initial_config(struct be_adapter *adapter)
4594 {
4595         int status, level;
4596
4597         status = be_cmd_get_cntl_attributes(adapter);
4598         if (status)
4599                 return status;
4600
4601         /* Must be a power of 2 or else MODULO will BUG_ON */
4602         adapter->be_get_temp_freq = 64;
4603
4604         if (BEx_chip(adapter)) {
4605                 level = be_cmd_get_fw_log_level(adapter);
4606                 adapter->msg_enable =
4607                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4608         }
4609
4610         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4611         return 0;
4612 }
4613
4614 static int lancer_recover_func(struct be_adapter *adapter)
4615 {
4616         struct device *dev = &adapter->pdev->dev;
4617         int status;
4618
4619         status = lancer_test_and_set_rdy_state(adapter);
4620         if (status)
4621                 goto err;
4622
4623         if (netif_running(adapter->netdev))
4624                 be_close(adapter->netdev);
4625
4626         be_clear(adapter);
4627
4628         be_clear_all_error(adapter);
4629
4630         status = be_setup(adapter);
4631         if (status)
4632                 goto err;
4633
4634         if (netif_running(adapter->netdev)) {
4635                 status = be_open(adapter->netdev);
4636                 if (status)
4637                         goto err;
4638         }
4639
4640         dev_err(dev, "Adapter recovery successful\n");
4641         return 0;
4642 err:
4643         if (status == -EAGAIN)
4644                 dev_err(dev, "Waiting for resource provisioning\n");
4645         else
4646                 dev_err(dev, "Adapter recovery failed\n");
4647
4648         return status;
4649 }
4650
4651 static void be_func_recovery_task(struct work_struct *work)
4652 {
4653         struct be_adapter *adapter =
4654                 container_of(work, struct be_adapter,  func_recovery_work.work);
4655         int status = 0;
4656
4657         be_detect_error(adapter);
4658
4659         if (adapter->hw_error && lancer_chip(adapter)) {
4660
4661                 rtnl_lock();
4662                 netif_device_detach(adapter->netdev);
4663                 rtnl_unlock();
4664
4665                 status = lancer_recover_func(adapter);
4666                 if (!status)
4667                         netif_device_attach(adapter->netdev);
4668         }
4669
4670         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4671          * no need to attempt further recovery.
4672          */
4673         if (!status || status == -EAGAIN)
4674                 schedule_delayed_work(&adapter->func_recovery_work,
4675                                       msecs_to_jiffies(1000));
4676 }
4677
4678 static void be_worker(struct work_struct *work)
4679 {
4680         struct be_adapter *adapter =
4681                 container_of(work, struct be_adapter, work.work);
4682         struct be_rx_obj *rxo;
4683         int i;
4684
4685         /* when interrupts are not yet enabled, just reap any pending
4686         * mcc completions */
4687         if (!netif_running(adapter->netdev)) {
4688                 local_bh_disable();
4689                 be_process_mcc(adapter);
4690                 local_bh_enable();
4691                 goto reschedule;
4692         }
4693
4694         if (!adapter->stats_cmd_sent) {
4695                 if (lancer_chip(adapter))
4696                         lancer_cmd_get_pport_stats(adapter,
4697                                                 &adapter->stats_cmd);
4698                 else
4699                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4700         }
4701
4702         if (be_physfn(adapter) &&
4703             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4704                 be_cmd_get_die_temperature(adapter);
4705
4706         for_all_rx_queues(adapter, rxo, i) {
4707                 /* Replenish RX-queues starved due to memory
4708                  * allocation failures.
4709                  */
4710                 if (rxo->rx_post_starved)
4711                         be_post_rx_frags(rxo, GFP_KERNEL);
4712         }
4713
4714         be_eqd_update(adapter);
4715
4716 reschedule:
4717         adapter->work_counter++;
4718         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4719 }
4720
4721 /* If any VFs are already enabled don't FLR the PF */
4722 static bool be_reset_required(struct be_adapter *adapter)
4723 {
4724         return pci_num_vf(adapter->pdev) ? false : true;
4725 }
4726
4727 static char *mc_name(struct be_adapter *adapter)
4728 {
4729         char *str = ""; /* default */
4730
4731         switch (adapter->mc_type) {
4732         case UMC:
4733                 str = "UMC";
4734                 break;
4735         case FLEX10:
4736                 str = "FLEX10";
4737                 break;
4738         case vNIC1:
4739                 str = "vNIC-1";
4740                 break;
4741         case nPAR:
4742                 str = "nPAR";
4743                 break;
4744         case UFP:
4745                 str = "UFP";
4746                 break;
4747         case vNIC2:
4748                 str = "vNIC-2";
4749                 break;
4750         default:
4751                 str = "";
4752         }
4753
4754         return str;
4755 }
4756
4757 static inline char *func_name(struct be_adapter *adapter)
4758 {
4759         return be_physfn(adapter) ? "PF" : "VF";
4760 }
4761
4762 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4763 {
4764         int status = 0;
4765         struct be_adapter *adapter;
4766         struct net_device *netdev;
4767         char port_name;
4768
4769         status = pci_enable_device(pdev);
4770         if (status)
4771                 goto do_none;
4772
4773         status = pci_request_regions(pdev, DRV_NAME);
4774         if (status)
4775                 goto disable_dev;
4776         pci_set_master(pdev);
4777
4778         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4779         if (netdev == NULL) {
4780                 status = -ENOMEM;
4781                 goto rel_reg;
4782         }
4783         adapter = netdev_priv(netdev);
4784         adapter->pdev = pdev;
4785         pci_set_drvdata(pdev, adapter);
4786         adapter->netdev = netdev;
4787         SET_NETDEV_DEV(netdev, &pdev->dev);
4788
4789         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4790         if (!status) {
4791                 netdev->features |= NETIF_F_HIGHDMA;
4792         } else {
4793                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4794                 if (status) {
4795                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4796                         goto free_netdev;
4797                 }
4798         }
4799
4800         if (be_physfn(adapter)) {
4801                 status = pci_enable_pcie_error_reporting(pdev);
4802                 if (!status)
4803                         dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4804         }
4805
4806         status = be_ctrl_init(adapter);
4807         if (status)
4808                 goto free_netdev;
4809
4810         /* sync up with fw's ready state */
4811         if (be_physfn(adapter)) {
4812                 status = be_fw_wait_ready(adapter);
4813                 if (status)
4814                         goto ctrl_clean;
4815         }
4816
4817         if (be_reset_required(adapter)) {
4818                 status = be_cmd_reset_function(adapter);
4819                 if (status)
4820                         goto ctrl_clean;
4821
4822                 /* Wait for interrupts to quiesce after an FLR */
4823                 msleep(100);
4824         }
4825
4826         /* Allow interrupts for other ULPs running on NIC function */
4827         be_intr_set(adapter, true);
4828
4829         /* tell fw we're ready to fire cmds */
4830         status = be_cmd_fw_init(adapter);
4831         if (status)
4832                 goto ctrl_clean;
4833
4834         status = be_stats_init(adapter);
4835         if (status)
4836                 goto ctrl_clean;
4837
4838         status = be_get_initial_config(adapter);
4839         if (status)
4840                 goto stats_clean;
4841
4842         INIT_DELAYED_WORK(&adapter->work, be_worker);
4843         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4844         adapter->rx_fc = adapter->tx_fc = true;
4845
4846         status = be_setup(adapter);
4847         if (status)
4848                 goto stats_clean;
4849
4850         be_netdev_init(netdev);
4851         status = register_netdev(netdev);
4852         if (status != 0)
4853                 goto unsetup;
4854
4855         be_roce_dev_add(adapter);
4856
4857         schedule_delayed_work(&adapter->func_recovery_work,
4858                               msecs_to_jiffies(1000));
4859
4860         be_cmd_query_port_name(adapter, &port_name);
4861
4862         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4863                  func_name(adapter), mc_name(adapter), port_name);
4864
4865         return 0;
4866
4867 unsetup:
4868         be_clear(adapter);
4869 stats_clean:
4870         be_stats_cleanup(adapter);
4871 ctrl_clean:
4872         be_ctrl_cleanup(adapter);
4873 free_netdev:
4874         free_netdev(netdev);
4875 rel_reg:
4876         pci_release_regions(pdev);
4877 disable_dev:
4878         pci_disable_device(pdev);
4879 do_none:
4880         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4881         return status;
4882 }
4883
4884 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4885 {
4886         struct be_adapter *adapter = pci_get_drvdata(pdev);
4887         struct net_device *netdev =  adapter->netdev;
4888
4889         if (adapter->wol_en)
4890                 be_setup_wol(adapter, true);
4891
4892         be_intr_set(adapter, false);
4893         cancel_delayed_work_sync(&adapter->func_recovery_work);
4894
4895         netif_device_detach(netdev);
4896         if (netif_running(netdev)) {
4897                 rtnl_lock();
4898                 be_close(netdev);
4899                 rtnl_unlock();
4900         }
4901         be_clear(adapter);
4902
4903         pci_save_state(pdev);
4904         pci_disable_device(pdev);
4905         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4906         return 0;
4907 }
4908
4909 static int be_resume(struct pci_dev *pdev)
4910 {
4911         int status = 0;
4912         struct be_adapter *adapter = pci_get_drvdata(pdev);
4913         struct net_device *netdev =  adapter->netdev;
4914
4915         netif_device_detach(netdev);
4916
4917         status = pci_enable_device(pdev);
4918         if (status)
4919                 return status;
4920
4921         pci_set_power_state(pdev, PCI_D0);
4922         pci_restore_state(pdev);
4923
4924         status = be_fw_wait_ready(adapter);
4925         if (status)
4926                 return status;
4927
4928         be_intr_set(adapter, true);
4929         /* tell fw we're ready to fire cmds */
4930         status = be_cmd_fw_init(adapter);
4931         if (status)
4932                 return status;
4933
4934         be_setup(adapter);
4935         if (netif_running(netdev)) {
4936                 rtnl_lock();
4937                 be_open(netdev);
4938                 rtnl_unlock();
4939         }
4940
4941         schedule_delayed_work(&adapter->func_recovery_work,
4942                               msecs_to_jiffies(1000));
4943         netif_device_attach(netdev);
4944
4945         if (adapter->wol_en)
4946                 be_setup_wol(adapter, false);
4947
4948         return 0;
4949 }
4950
4951 /*
4952  * An FLR will stop BE from DMAing any data.
4953  */
4954 static void be_shutdown(struct pci_dev *pdev)
4955 {
4956         struct be_adapter *adapter = pci_get_drvdata(pdev);
4957
4958         if (!adapter)
4959                 return;
4960
4961         cancel_delayed_work_sync(&adapter->work);
4962         cancel_delayed_work_sync(&adapter->func_recovery_work);
4963
4964         netif_device_detach(adapter->netdev);
4965
4966         be_cmd_reset_function(adapter);
4967
4968         pci_disable_device(pdev);
4969 }
4970
4971 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
4972                                             pci_channel_state_t state)
4973 {
4974         struct be_adapter *adapter = pci_get_drvdata(pdev);
4975         struct net_device *netdev =  adapter->netdev;
4976
4977         dev_err(&adapter->pdev->dev, "EEH error detected\n");
4978
4979         if (!adapter->eeh_error) {
4980                 adapter->eeh_error = true;
4981
4982                 cancel_delayed_work_sync(&adapter->func_recovery_work);
4983
4984                 rtnl_lock();
4985                 netif_device_detach(netdev);
4986                 if (netif_running(netdev))
4987                         be_close(netdev);
4988                 rtnl_unlock();
4989
4990                 be_clear(adapter);
4991         }
4992
4993         if (state == pci_channel_io_perm_failure)
4994                 return PCI_ERS_RESULT_DISCONNECT;
4995
4996         pci_disable_device(pdev);
4997
4998         /* The error could cause the FW to trigger a flash debug dump.
4999          * Resetting the card while flash dump is in progress
5000          * can cause it not to recover; wait for it to finish.
5001          * Wait only for first function as it is needed only once per
5002          * adapter.
5003          */
5004         if (pdev->devfn == 0)
5005                 ssleep(30);
5006
5007         return PCI_ERS_RESULT_NEED_RESET;
5008 }
5009
5010 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5011 {
5012         struct be_adapter *adapter = pci_get_drvdata(pdev);
5013         int status;
5014
5015         dev_info(&adapter->pdev->dev, "EEH reset\n");
5016
5017         status = pci_enable_device(pdev);
5018         if (status)
5019                 return PCI_ERS_RESULT_DISCONNECT;
5020
5021         pci_set_master(pdev);
5022         pci_set_power_state(pdev, PCI_D0);
5023         pci_restore_state(pdev);
5024
5025         /* Check if card is ok and fw is ready */
5026         dev_info(&adapter->pdev->dev,
5027                  "Waiting for FW to be ready after EEH reset\n");
5028         status = be_fw_wait_ready(adapter);
5029         if (status)
5030                 return PCI_ERS_RESULT_DISCONNECT;
5031
5032         pci_cleanup_aer_uncorrect_error_status(pdev);
5033         be_clear_all_error(adapter);
5034         return PCI_ERS_RESULT_RECOVERED;
5035 }
5036
5037 static void be_eeh_resume(struct pci_dev *pdev)
5038 {
5039         int status = 0;
5040         struct be_adapter *adapter = pci_get_drvdata(pdev);
5041         struct net_device *netdev =  adapter->netdev;
5042
5043         dev_info(&adapter->pdev->dev, "EEH resume\n");
5044
5045         pci_save_state(pdev);
5046
5047         status = be_cmd_reset_function(adapter);
5048         if (status)
5049                 goto err;
5050
5051         /* On some BE3 FW versions, after a HW reset,
5052          * interrupts will remain disabled for each function.
5053          * So, explicitly enable interrupts
5054          */
5055         be_intr_set(adapter, true);
5056
5057         /* tell fw we're ready to fire cmds */
5058         status = be_cmd_fw_init(adapter);
5059         if (status)
5060                 goto err;
5061
5062         status = be_setup(adapter);
5063         if (status)
5064                 goto err;
5065
5066         if (netif_running(netdev)) {
5067                 status = be_open(netdev);
5068                 if (status)
5069                         goto err;
5070         }
5071
5072         schedule_delayed_work(&adapter->func_recovery_work,
5073                               msecs_to_jiffies(1000));
5074         netif_device_attach(netdev);
5075         return;
5076 err:
5077         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5078 }
5079
5080 static const struct pci_error_handlers be_eeh_handlers = {
5081         .error_detected = be_eeh_err_detected,
5082         .slot_reset = be_eeh_reset,
5083         .resume = be_eeh_resume,
5084 };
5085
5086 static struct pci_driver be_driver = {
5087         .name = DRV_NAME,
5088         .id_table = be_dev_ids,
5089         .probe = be_probe,
5090         .remove = be_remove,
5091         .suspend = be_suspend,
5092         .resume = be_resume,
5093         .shutdown = be_shutdown,
5094         .err_handler = &be_eeh_handlers
5095 };
5096
5097 static int __init be_init_module(void)
5098 {
5099         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5100             rx_frag_size != 2048) {
5101                 printk(KERN_WARNING DRV_NAME
5102                         " : Module param rx_frag_size must be 2048/4096/8192."
5103                         " Using 2048\n");
5104                 rx_frag_size = 2048;
5105         }
5106
5107         return pci_register_driver(&be_driver);
5108 }
5109 module_init(be_init_module);
5110
5111 static void __exit be_exit_module(void)
5112 {
5113         pci_unregister_driver(&be_driver);
5114 }
5115 module_exit(be_exit_module);