drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2013 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26
  27 MODULE_VERSION(DRV_VER);
  28 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 static unsigned int num_vfs;
  34 module_param(num_vfs, uint, S_IRUGO);
  35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  36
  37 static ushort rx_frag_size = 2048;
  38 module_param(rx_frag_size, ushort, S_IRUGO);
  39 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  40
  41 static DEFINE_PCI_DEVICE_TABLE(be_dev_ids) = {
  42         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  44         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  46         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  50         { 0 }
  51 };
  52 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  53 /* UE Status Low CSR */
  54 static const char * const ue_status_low_desc[] = {
  55         "CEV",
  56         "CTX",
  57         "DBUF",
  58         "ERX",
  59         "Host",
  60         "MPU",
  61         "NDMA",
  62         "PTC ",
  63         "RDMA ",
  64         "RXF ",
  65         "RXIPS ",
  66         "RXULP0 ",
  67         "RXULP1 ",
  68         "RXULP2 ",
  69         "TIM ",
  70         "TPOST ",
  71         "TPRE ",
  72         "TXIPS ",
  73         "TXULP0 ",
  74         "TXULP1 ",
  75         "UC ",
  76         "WDMA ",
  77         "TXULP2 ",
  78         "HOST1 ",
  79         "P0_OB_LINK ",
  80         "P1_OB_LINK ",
  81         "HOST_GPIO ",
  82         "MBOX ",
  83         "AXGMAC0",
  84         "AXGMAC1",
  85         "JTAG",
  86         "MPU_INTPEND"
  87 };
  88 /* UE Status High CSR */
  89 static const char * const ue_status_hi_desc[] = {
  90         "LPCMEMHOST",
  91         "MGMT_MAC",
  92         "PCS0ONLINE",
  93         "MPU_IRAM",
  94         "PCS1ONLINE",
  95         "PCTL0",
  96         "PCTL1",
  97         "PMEM",
  98         "RR",
  99         "TXPB",
 100         "RXPP",
 101         "XAUI",
 102         "TXP",
 103         "ARM",
 104         "IPC",
 105         "HOST2",
 106         "HOST3",
 107         "HOST4",
 108         "HOST5",
 109         "HOST6",
 110         "HOST7",
 111         "HOST8",
 112         "HOST9",
 113         "NETC",
 114         "Unknown",
 115         "Unknown",
 116         "Unknown",
 117         "Unknown",
 118         "Unknown",
 119         "Unknown",
 120         "Unknown",
 121         "Unknown"
 122 };
 123
 124
 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 126 {
 127         struct be_dma_mem *mem = &q->dma_mem;
 128         if (mem->va) {
 129                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 130                                   mem->dma);
 131                 mem->va = NULL;
 132         }
 133 }
 134
 135 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 136                 u16 len, u16 entry_size)
 137 {
 138         struct be_dma_mem *mem = &q->dma_mem;
 139
 140         memset(q, 0, sizeof(*q));
 141         q->len = len;
 142         q->entry_size = entry_size;
 143         mem->size = len * entry_size;
 144         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 145                                       GFP_KERNEL);
 146         if (!mem->va)
 147                 return -ENOMEM;
 148         return 0;
 149 }
 150
 151 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 152 {
 153         u32 reg, enabled;
 154
 155         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 156                                 &reg);
 157         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 158
 159         if (!enabled && enable)
 160                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 161         else if (enabled && !enable)
 162                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else
 164                 return;
 165
 166         pci_write_config_dword(adapter->pdev,
 167                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 168 }
 169
 170 static void be_intr_set(struct be_adapter *adapter, bool enable)
 171 {
 172         int status = 0;
 173
 174         /* On lancer interrupts can't be controlled via this register */
 175         if (lancer_chip(adapter))
 176                 return;
 177
 178         if (adapter->eeh_error)
 179                 return;
 180
 181         status = be_cmd_intr_set(adapter, enable);
 182         if (status)
 183                 be_reg_intr_set(adapter, enable);
 184 }
 185
 186 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 187 {
 188         u32 val = 0;
 189         val |= qid & DB_RQ_RING_ID_MASK;
 190         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 191
 192         wmb();
 193         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 194 }
 195
 196 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 197                           u16 posted)
 198 {
 199         u32 val = 0;
 200         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 201         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 202
 203         wmb();
 204         iowrite32(val, adapter->db + txo->db_offset);
 205 }
 206
 207 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 208                 bool arm, bool clear_int, u16 num_popped)
 209 {
 210         u32 val = 0;
 211         val |= qid & DB_EQ_RING_ID_MASK;
 212         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) <<
 213                         DB_EQ_RING_ID_EXT_MASK_SHIFT);
 214
 215         if (adapter->eeh_error)
 216                 return;
 217
 218         if (arm)
 219                 val |= 1 << DB_EQ_REARM_SHIFT;
 220         if (clear_int)
 221                 val |= 1 << DB_EQ_CLR_SHIFT;
 222         val |= 1 << DB_EQ_EVNT_SHIFT;
 223         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 224         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 225 }
 226
 227 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 228 {
 229         u32 val = 0;
 230         val |= qid & DB_CQ_RING_ID_MASK;
 231         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 232                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 233
 234         if (adapter->eeh_error)
 235                 return;
 236
 237         if (arm)
 238                 val |= 1 << DB_CQ_REARM_SHIFT;
 239         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 240         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 241 }
 242
 243 static int be_mac_addr_set(struct net_device *netdev, void *p)
 244 {
 245         struct be_adapter *adapter = netdev_priv(netdev);
 246         struct device *dev = &adapter->pdev->dev;
 247         struct sockaddr *addr = p;
 248         int status;
 249         u8 mac[ETH_ALEN];
 250         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 251
 252         if (!is_valid_ether_addr(addr->sa_data))
 253                 return -EADDRNOTAVAIL;
 254
 255         /* Proceed further only if, User provided MAC is different
 256          * from active MAC
 257          */
 258         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 259                 return 0;
 260
 261         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 262          * privilege or if PF did not provision the new MAC address.
 263          * On BE3, this cmd will always fail if the VF doesn't have the
 264          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 265          * the MAC for the VF.
 266          */
 267         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 268                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 269         if (!status) {
 270                 curr_pmac_id = adapter->pmac_id[0];
 271
 272                 /* Delete the old programmed MAC. This call may fail if the
 273                  * old MAC was already deleted by the PF driver.
 274                  */
 275                 if (adapter->pmac_id[0] != old_pmac_id)
 276                         be_cmd_pmac_del(adapter, adapter->if_handle,
 277                                         old_pmac_id, 0);
 278         }
 279
 280         /* Decide if the new MAC is successfully activated only after
 281          * querying the FW
 282          */
 283         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 284                                        adapter->if_handle, true, 0);
 285         if (status)
 286                 goto err;
 287
 288         /* The MAC change did not happen, either due to lack of privilege
 289          * or PF didn't pre-provision.
 290          */
 291         if (!ether_addr_equal(addr->sa_data, mac)) {
 292                 status = -EPERM;
 293                 goto err;
 294         }
 295
 296         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 297         dev_info(dev, "MAC address changed to %pM\n", mac);
 298         return 0;
 299 err:
 300         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 301         return status;
 302 }
 303
 304 /* BE2 supports only v0 cmd */
 305 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 306 {
 307         if (BE2_chip(adapter)) {
 308                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 309
 310                 return &cmd->hw_stats;
 311         } else if (BE3_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else {
 316                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         }
 320 }
 321
 322 /* BE2 supports only v0 cmd */
 323 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 324 {
 325         if (BE2_chip(adapter)) {
 326                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 327
 328                 return &hw_stats->erx;
 329         } else if (BE3_chip(adapter)) {
 330                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else {
 334                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         }
 338 }
 339
 340 static void populate_be_v0_stats(struct be_adapter *adapter)
 341 {
 342         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 343         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 344         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 345         struct be_port_rxf_stats_v0 *port_stats =
 346                                         &rxf_stats->port[adapter->port_num];
 347         struct be_drv_stats *drvs = &adapter->drv_stats;
 348
 349         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 350         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 351         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 352         drvs->rx_control_frames = port_stats->rx_control_frames;
 353         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 354         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 355         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 356         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 357         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 358         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 359         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 360         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 361         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 362         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 363         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 364         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 365         drvs->rx_dropped_header_too_small =
 366                 port_stats->rx_dropped_header_too_small;
 367         drvs->rx_address_filtered =
 368                                         port_stats->rx_address_filtered +
 369                                         port_stats->rx_vlan_filtered;
 370         drvs->rx_alignment_symbol_errors =
 371                 port_stats->rx_alignment_symbol_errors;
 372
 373         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 374         drvs->tx_controlframes = port_stats->tx_controlframes;
 375
 376         if (adapter->port_num)
 377                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 378         else
 379                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 380         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 381         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 382         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 383         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 384         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 385         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 386         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 387 }
 388
 389 static void populate_be_v1_stats(struct be_adapter *adapter)
 390 {
 391         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 392         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 393         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 394         struct be_port_rxf_stats_v1 *port_stats =
 395                                         &rxf_stats->port[adapter->port_num];
 396         struct be_drv_stats *drvs = &adapter->drv_stats;
 397
 398         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 399         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 400         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 401         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 402         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 403         drvs->rx_control_frames = port_stats->rx_control_frames;
 404         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 405         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 406         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 407         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 408         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 409         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 410         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 411         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 412         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 413         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 414         drvs->rx_dropped_header_too_small =
 415                 port_stats->rx_dropped_header_too_small;
 416         drvs->rx_input_fifo_overflow_drop =
 417                 port_stats->rx_input_fifo_overflow_drop;
 418         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 419         drvs->rx_alignment_symbol_errors =
 420                 port_stats->rx_alignment_symbol_errors;
 421         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 422         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 423         drvs->tx_controlframes = port_stats->tx_controlframes;
 424         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 425         drvs->jabber_events = port_stats->jabber_events;
 426         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 427         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 428         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 429         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 430         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 431         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 432         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 433 }
 434
 435 static void populate_be_v2_stats(struct be_adapter *adapter)
 436 {
 437         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 438         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 439         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 440         struct be_port_rxf_stats_v2 *port_stats =
 441                                         &rxf_stats->port[adapter->port_num];
 442         struct be_drv_stats *drvs = &adapter->drv_stats;
 443
 444         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 445         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 446         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 447         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 448         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 449         drvs->rx_control_frames = port_stats->rx_control_frames;
 450         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 451         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 452         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 453         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 454         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 455         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 456         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 457         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 458         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 459         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 460         drvs->rx_dropped_header_too_small =
 461                 port_stats->rx_dropped_header_too_small;
 462         drvs->rx_input_fifo_overflow_drop =
 463                 port_stats->rx_input_fifo_overflow_drop;
 464         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 465         drvs->rx_alignment_symbol_errors =
 466                 port_stats->rx_alignment_symbol_errors;
 467         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 468         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 469         drvs->tx_controlframes = port_stats->tx_controlframes;
 470         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 471         drvs->jabber_events = port_stats->jabber_events;
 472         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 473         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 474         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 475         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 476         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 477         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 478         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 479         if (be_roce_supported(adapter))  {
 480                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 481                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 482                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 483                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 484                 drvs->roce_drops_payload_len =
 485                         port_stats->roce_drops_payload_len;
 486         }
 487 }
 488
 489 static void populate_lancer_stats(struct be_adapter *adapter)
 490 {
 491
 492         struct be_drv_stats *drvs = &adapter->drv_stats;
 493         struct lancer_pport_stats *pport_stats =
 494                                         pport_stats_from_cmd(adapter);
 495
 496         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 497         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 498         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 499         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 500         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 501         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 502         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 503         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 504         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 505         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 506         drvs->rx_dropped_tcp_length =
 507                                 pport_stats->rx_dropped_invalid_tcp_length;
 508         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 509         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 510         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 511         drvs->rx_dropped_header_too_small =
 512                                 pport_stats->rx_dropped_header_too_small;
 513         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 514         drvs->rx_address_filtered =
 515                                         pport_stats->rx_address_filtered +
 516                                         pport_stats->rx_vlan_filtered;
 517         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 518         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 519         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 520         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 521         drvs->jabber_events = pport_stats->rx_jabbers;
 522         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 523         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 524         drvs->rx_drops_too_many_frags =
 525                                 pport_stats->rx_drops_too_many_frags_lo;
 526 }
 527
 528 static void accumulate_16bit_val(u32 *acc, u16 val)
 529 {
 530 #define lo(x)                   (x & 0xFFFF)
 531 #define hi(x)                   (x & 0xFFFF0000)
 532         bool wrapped = val < lo(*acc);
 533         u32 newacc = hi(*acc) + val;
 534
 535         if (wrapped)
 536                 newacc += 65536;
 537         ACCESS_ONCE(*acc) = newacc;
 538 }
 539
 540 static void populate_erx_stats(struct be_adapter *adapter,
 541                         struct be_rx_obj *rxo,
 542                         u32 erx_stat)
 543 {
 544         if (!BEx_chip(adapter))
 545                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 546         else
 547                 /* below erx HW counter can actually wrap around after
 548                  * 65535. Driver accumulates a 32-bit value
 549                  */
 550                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 551                                      (u16)erx_stat);
 552 }
 553
 554 void be_parse_stats(struct be_adapter *adapter)
 555 {
 556         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 557         struct be_rx_obj *rxo;
 558         int i;
 559         u32 erx_stat;
 560
 561         if (lancer_chip(adapter)) {
 562                 populate_lancer_stats(adapter);
 563         } else {
 564                 if (BE2_chip(adapter))
 565                         populate_be_v0_stats(adapter);
 566                 else if (BE3_chip(adapter))
 567                         /* for BE3 */
 568                         populate_be_v1_stats(adapter);
 569                 else
 570                         populate_be_v2_stats(adapter);
 571
 572                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 573                 for_all_rx_queues(adapter, rxo, i) {
 574                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 575                         populate_erx_stats(adapter, rxo, erx_stat);
 576                 }
 577         }
 578 }
 579
 580 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 581                                         struct rtnl_link_stats64 *stats)
 582 {
 583         struct be_adapter *adapter = netdev_priv(netdev);
 584         struct be_drv_stats *drvs = &adapter->drv_stats;
 585         struct be_rx_obj *rxo;
 586         struct be_tx_obj *txo;
 587         u64 pkts, bytes;
 588         unsigned int start;
 589         int i;
 590
 591         for_all_rx_queues(adapter, rxo, i) {
 592                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 593                 do {
 594                         start = u64_stats_fetch_begin_bh(&rx_stats->sync);
 595                         pkts = rx_stats(rxo)->rx_pkts;
 596                         bytes = rx_stats(rxo)->rx_bytes;
 597                 } while (u64_stats_fetch_retry_bh(&rx_stats->sync, start));
 598                 stats->rx_packets += pkts;
 599                 stats->rx_bytes += bytes;
 600                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 601                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 602                                         rx_stats(rxo)->rx_drops_no_frags;
 603         }
 604
 605         for_all_tx_queues(adapter, txo, i) {
 606                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 607                 do {
 608                         start = u64_stats_fetch_begin_bh(&tx_stats->sync);
 609                         pkts = tx_stats(txo)->tx_pkts;
 610                         bytes = tx_stats(txo)->tx_bytes;
 611                 } while (u64_stats_fetch_retry_bh(&tx_stats->sync, start));
 612                 stats->tx_packets += pkts;
 613                 stats->tx_bytes += bytes;
 614         }
 615
 616         /* bad pkts received */
 617         stats->rx_errors = drvs->rx_crc_errors +
 618                 drvs->rx_alignment_symbol_errors +
 619                 drvs->rx_in_range_errors +
 620                 drvs->rx_out_range_errors +
 621                 drvs->rx_frame_too_long +
 622                 drvs->rx_dropped_too_small +
 623                 drvs->rx_dropped_too_short +
 624                 drvs->rx_dropped_header_too_small +
 625                 drvs->rx_dropped_tcp_length +
 626                 drvs->rx_dropped_runt;
 627
 628         /* detailed rx errors */
 629         stats->rx_length_errors = drvs->rx_in_range_errors +
 630                 drvs->rx_out_range_errors +
 631                 drvs->rx_frame_too_long;
 632
 633         stats->rx_crc_errors = drvs->rx_crc_errors;
 634
 635         /* frame alignment errors */
 636         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 637
 638         /* receiver fifo overrun */
 639         /* drops_no_pbuf is no per i/f, it's per BE card */
 640         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 641                                 drvs->rx_input_fifo_overflow_drop +
 642                                 drvs->rx_drops_no_pbuf;
 643         return stats;
 644 }
 645
 646 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 647 {
 648         struct net_device *netdev = adapter->netdev;
 649
 650         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 651                 netif_carrier_off(netdev);
 652                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 653         }
 654
 655         if ((link_status & LINK_STATUS_MASK) == LINK_UP)
 656                 netif_carrier_on(netdev);
 657         else
 658                 netif_carrier_off(netdev);
 659 }
 660
 661 static void be_tx_stats_update(struct be_tx_obj *txo,
 662                         u32 wrb_cnt, u32 copied, u32 gso_segs, bool stopped)
 663 {
 664         struct be_tx_stats *stats = tx_stats(txo);
 665
 666         u64_stats_update_begin(&stats->sync);
 667         stats->tx_reqs++;
 668         stats->tx_wrbs += wrb_cnt;
 669         stats->tx_bytes += copied;
 670         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 671         if (stopped)
 672                 stats->tx_stops++;
 673         u64_stats_update_end(&stats->sync);
 674 }
 675
 676 /* Determine number of WRB entries needed to xmit data in an skb */
 677 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 678                                                                 bool *dummy)
 679 {
 680         int cnt = (skb->len > skb->data_len);
 681
 682         cnt += skb_shinfo(skb)->nr_frags;
 683
 684         /* to account for hdr wrb */
 685         cnt++;
 686         if (lancer_chip(adapter) || !(cnt & 1)) {
 687                 *dummy = false;
 688         } else {
 689                 /* add a dummy to make it an even num */
 690                 cnt++;
 691                 *dummy = true;
 692         }
 693         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 694         return cnt;
 695 }
 696
 697 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 698 {
 699         wrb->frag_pa_hi = upper_32_bits(addr);
 700         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 701         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 702         wrb->rsvd0 = 0;
 703 }
 704
 705 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 706                                         struct sk_buff *skb)
 707 {
 708         u8 vlan_prio;
 709         u16 vlan_tag;
 710
 711         vlan_tag = vlan_tx_tag_get(skb);
 712         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 713         /* If vlan priority provided by OS is NOT in available bmap */
 714         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 715                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 716                                 adapter->recommended_prio;
 717
 718         return vlan_tag;
 719 }
 720
 721 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 722                 struct sk_buff *skb, u32 wrb_cnt, u32 len, bool skip_hw_vlan)
 723 {
 724         u16 vlan_tag;
 725
 726         memset(hdr, 0, sizeof(*hdr));
 727
 728         AMAP_SET_BITS(struct amap_eth_hdr_wrb, crc, hdr, 1);
 729
 730         if (skb_is_gso(skb)) {
 731                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso, hdr, 1);
 732                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso_mss,
 733                         hdr, skb_shinfo(skb)->gso_size);
 734                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 735                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso6, hdr, 1);
 736         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 737                 if (is_tcp_pkt(skb))
 738                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, tcpcs, hdr, 1);
 739                 else if (is_udp_pkt(skb))
 740                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, udpcs, hdr, 1);
 741         }
 742
 743         if (vlan_tx_tag_present(skb)) {
 744                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan, hdr, 1);
 745                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 746                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan_tag, hdr, vlan_tag);
 747         }
 748
 749         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 750         AMAP_SET_BITS(struct amap_eth_hdr_wrb, complete, hdr, !skip_hw_vlan);
 751         AMAP_SET_BITS(struct amap_eth_hdr_wrb, event, hdr, 1);
 752         AMAP_SET_BITS(struct amap_eth_hdr_wrb, num_wrb, hdr, wrb_cnt);
 753         AMAP_SET_BITS(struct amap_eth_hdr_wrb, len, hdr, len);
 754 }
 755
 756 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 757                 bool unmap_single)
 758 {
 759         dma_addr_t dma;
 760
 761         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 762
 763         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 764         if (wrb->frag_len) {
 765                 if (unmap_single)
 766                         dma_unmap_single(dev, dma, wrb->frag_len,
 767                                          DMA_TO_DEVICE);
 768                 else
 769                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 770         }
 771 }
 772
 773 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 774                 struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 775                 bool skip_hw_vlan)
 776 {
 777         dma_addr_t busaddr;
 778         int i, copied = 0;
 779         struct device *dev = &adapter->pdev->dev;
 780         struct sk_buff *first_skb = skb;
 781         struct be_eth_wrb *wrb;
 782         struct be_eth_hdr_wrb *hdr;
 783         bool map_single = false;
 784         u16 map_head;
 785
 786         hdr = queue_head_node(txq);
 787         queue_head_inc(txq);
 788         map_head = txq->head;
 789
 790         if (skb->len > skb->data_len) {
 791                 int len = skb_headlen(skb);
 792                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 793                 if (dma_mapping_error(dev, busaddr))
 794                         goto dma_err;
 795                 map_single = true;
 796                 wrb = queue_head_node(txq);
 797                 wrb_fill(wrb, busaddr, len);
 798                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 799                 queue_head_inc(txq);
 800                 copied += len;
 801         }
 802
 803         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 804                 const struct skb_frag_struct *frag =
 805                         &skb_shinfo(skb)->frags[i];
 806                 busaddr = skb_frag_dma_map(dev, frag, 0,
 807                                            skb_frag_size(frag), DMA_TO_DEVICE);
 808                 if (dma_mapping_error(dev, busaddr))
 809                         goto dma_err;
 810                 wrb = queue_head_node(txq);
 811                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 812                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 813                 queue_head_inc(txq);
 814                 copied += skb_frag_size(frag);
 815         }
 816
 817         if (dummy_wrb) {
 818                 wrb = queue_head_node(txq);
 819                 wrb_fill(wrb, 0, 0);
 820                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 821                 queue_head_inc(txq);
 822         }
 823
 824         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 825         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 826
 827         return copied;
 828 dma_err:
 829         txq->head = map_head;
 830         while (copied) {
 831                 wrb = queue_head_node(txq);
 832                 unmap_tx_frag(dev, wrb, map_single);
 833                 map_single = false;
 834                 copied -= wrb->frag_len;
 835                 queue_head_inc(txq);
 836         }
 837         return 0;
 838 }
 839
 840 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 841                                              struct sk_buff *skb,
 842                                              bool *skip_hw_vlan)
 843 {
 844         u16 vlan_tag = 0;
 845
 846         skb = skb_share_check(skb, GFP_ATOMIC);
 847         if (unlikely(!skb))
 848                 return skb;
 849
 850         if (vlan_tx_tag_present(skb))
 851                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 852
 853         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 854                 if (!vlan_tag)
 855                         vlan_tag = adapter->pvid;
 856                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 857                  * skip VLAN insertion
 858                  */
 859                 if (skip_hw_vlan)
 860                         *skip_hw_vlan = true;
 861         }
 862
 863         if (vlan_tag) {
 864                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 865                 if (unlikely(!skb))
 866                         return skb;
 867                 skb->vlan_tci = 0;
 868         }
 869
 870         /* Insert the outer VLAN, if any */
 871         if (adapter->qnq_vid) {
 872                 vlan_tag = adapter->qnq_vid;
 873                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 874                 if (unlikely(!skb))
 875                         return skb;
 876                 if (skip_hw_vlan)
 877                         *skip_hw_vlan = true;
 878         }
 879
 880         return skb;
 881 }
 882
 883 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 884 {
 885         struct ethhdr *eh = (struct ethhdr *)skb->data;
 886         u16 offset = ETH_HLEN;
 887
 888         if (eh->h_proto == htons(ETH_P_IPV6)) {
 889                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 890
 891                 offset += sizeof(struct ipv6hdr);
 892                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 893                     ip6h->nexthdr != NEXTHDR_UDP) {
 894                         struct ipv6_opt_hdr *ehdr =
 895                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 896
 897                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 898                         if (ehdr->hdrlen == 0xff)
 899                                 return true;
 900                 }
 901         }
 902         return false;
 903 }
 904
 905 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 906 {
 907         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 908 }
 909
 910 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter,
 911                                 struct sk_buff *skb)
 912 {
 913         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 914 }
 915
 916 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 917                                                   struct sk_buff *skb,
 918                                                   bool *skip_hw_vlan)
 919 {
 920         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 921         unsigned int eth_hdr_len;
 922         struct iphdr *ip;
 923
 924         /* For padded packets, BE HW modifies tot_len field in IP header
 925          * incorrecly when VLAN tag is inserted by HW.
 926          * For padded packets, Lancer computes incorrect checksum.
 927          */
 928         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 929                                                 VLAN_ETH_HLEN : ETH_HLEN;
 930         if (skb->len <= 60 &&
 931             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 932             is_ipv4_pkt(skb)) {
 933                 ip = (struct iphdr *)ip_hdr(skb);
 934                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 935         }
 936
 937         /* If vlan tag is already inlined in the packet, skip HW VLAN
 938          * tagging in UMC mode
 939          */
 940         if ((adapter->function_mode & UMC_ENABLED) &&
 941             veh->h_vlan_proto == htons(ETH_P_8021Q))
 942                         *skip_hw_vlan = true;
 943
 944         /* HW has a bug wherein it will calculate CSUM for VLAN
 945          * pkts even though it is disabled.
 946          * Manually insert VLAN in pkt.
 947          */
 948         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 949             vlan_tx_tag_present(skb)) {
 950                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 951                 if (unlikely(!skb))
 952                         goto err;
 953         }
 954
 955         /* HW may lockup when VLAN HW tagging is requested on
 956          * certain ipv6 packets. Drop such pkts if the HW workaround to
 957          * skip HW tagging is not enabled by FW.
 958          */
 959         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 960             (adapter->pvid || adapter->qnq_vid) &&
 961             !qnq_async_evt_rcvd(adapter)))
 962                 goto tx_drop;
 963
 964         /* Manual VLAN tag insertion to prevent:
 965          * ASIC lockup when the ASIC inserts VLAN tag into
 966          * certain ipv6 packets. Insert VLAN tags in driver,
 967          * and set event, completion, vlan bits accordingly
 968          * in the Tx WRB.
 969          */
 970         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 971             be_vlan_tag_tx_chk(adapter, skb)) {
 972                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 973                 if (unlikely(!skb))
 974                         goto err;
 975         }
 976
 977         return skb;
 978 tx_drop:
 979         dev_kfree_skb_any(skb);
 980 err:
 981         return NULL;
 982 }
 983
 984 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
 985                                            struct sk_buff *skb,
 986                                            bool *skip_hw_vlan)
 987 {
 988         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
 989          * less may cause a transmit stall on that port. So the work-around is
 990          * to pad short packets (<= 32 bytes) to a 36-byte length.
 991          */
 992         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
 993                 if (skb_padto(skb, 36))
 994                         return NULL;
 995                 skb->len = 36;
 996         }
 997
 998         if (BEx_chip(adapter) || lancer_chip(adapter)) {
 999                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1000                 if (!skb)
1001                         return NULL;
1002         }
1003
1004         return skb;
1005 }
1006
1007 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1008 {
1009         struct be_adapter *adapter = netdev_priv(netdev);
1010         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1011         struct be_queue_info *txq = &txo->q;
1012         bool dummy_wrb, stopped = false;
1013         u32 wrb_cnt = 0, copied = 0;
1014         bool skip_hw_vlan = false;
1015         u32 start = txq->head;
1016
1017         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1018         if (!skb) {
1019                 tx_stats(txo)->tx_drv_drops++;
1020                 return NETDEV_TX_OK;
1021         }
1022
1023         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1024
1025         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1026                               skip_hw_vlan);
1027         if (copied) {
1028                 int gso_segs = skb_shinfo(skb)->gso_segs;
1029
1030                 /* record the sent skb in the sent_skb table */
1031                 BUG_ON(txo->sent_skb_list[start]);
1032                 txo->sent_skb_list[start] = skb;
1033
1034                 /* Ensure txq has space for the next skb; Else stop the queue
1035                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1036                  * tx compls of the current transmit which'll wake up the queue
1037                  */
1038                 atomic_add(wrb_cnt, &txq->used);
1039                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1040                                                                 txq->len) {
1041                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1042                         stopped = true;
1043                 }
1044
1045                 be_txq_notify(adapter, txo, wrb_cnt);
1046
1047                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1048         } else {
1049                 txq->head = start;
1050                 tx_stats(txo)->tx_drv_drops++;
1051                 dev_kfree_skb_any(skb);
1052         }
1053         return NETDEV_TX_OK;
1054 }
1055
1056 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1057 {
1058         struct be_adapter *adapter = netdev_priv(netdev);
1059         if (new_mtu < BE_MIN_MTU ||
1060                         new_mtu > (BE_MAX_JUMBO_FRAME_SIZE -
1061                                         (ETH_HLEN + ETH_FCS_LEN))) {
1062                 dev_info(&adapter->pdev->dev,
1063                         "MTU must be between %d and %d bytes\n",
1064                         BE_MIN_MTU,
1065                         (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN)));
1066                 return -EINVAL;
1067         }
1068         dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n",
1069                         netdev->mtu, new_mtu);
1070         netdev->mtu = new_mtu;
1071         return 0;
1072 }
1073
1074 /*
1075  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1076  * If the user configures more, place BE in vlan promiscuous mode.
1077  */
1078 static int be_vid_config(struct be_adapter *adapter)
1079 {
1080         u16 vids[BE_NUM_VLANS_SUPPORTED];
1081         u16 num = 0, i;
1082         int status = 0;
1083
1084         /* No need to further configure vids if in promiscuous mode */
1085         if (adapter->promiscuous)
1086                 return 0;
1087
1088         if (adapter->vlans_added > be_max_vlans(adapter))
1089                 goto set_vlan_promisc;
1090
1091         /* Construct VLAN Table to give to HW */
1092         for (i = 0; i < VLAN_N_VID; i++)
1093                 if (adapter->vlan_tag[i])
1094                         vids[num++] = cpu_to_le16(i);
1095
1096         status = be_cmd_vlan_config(adapter, adapter->if_handle,
1097                                     vids, num, 0);
1098
1099         if (status) {
1100                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1101                 if (status == MCC_ADDL_STS_INSUFFICIENT_RESOURCES)
1102                         goto set_vlan_promisc;
1103                 dev_err(&adapter->pdev->dev,
1104                         "Setting HW VLAN filtering failed.\n");
1105         } else {
1106                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1107                         /* hw VLAN filtering re-enabled. */
1108                         status = be_cmd_rx_filter(adapter,
1109                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1110                         if (!status) {
1111                                 dev_info(&adapter->pdev->dev,
1112                                          "Disabling VLAN Promiscuous mode.\n");
1113                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1114                         }
1115                 }
1116         }
1117
1118         return status;
1119
1120 set_vlan_promisc:
1121         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1122                 return 0;
1123
1124         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1125         if (!status) {
1126                 dev_info(&adapter->pdev->dev, "Enable VLAN Promiscuous mode\n");
1127                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1128         } else
1129                 dev_err(&adapter->pdev->dev,
1130                         "Failed to enable VLAN Promiscuous mode.\n");
1131         return status;
1132 }
1133
1134 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1135 {
1136         struct be_adapter *adapter = netdev_priv(netdev);
1137         int status = 0;
1138
1139         /* Packets with VID 0 are always received by Lancer by default */
1140         if (lancer_chip(adapter) && vid == 0)
1141                 goto ret;
1142
1143         adapter->vlan_tag[vid] = 1;
1144         adapter->vlans_added++;
1145
1146         status = be_vid_config(adapter);
1147         if (status) {
1148                 adapter->vlans_added--;
1149                 adapter->vlan_tag[vid] = 0;
1150         }
1151 ret:
1152         return status;
1153 }
1154
1155 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1156 {
1157         struct be_adapter *adapter = netdev_priv(netdev);
1158         int status = 0;
1159
1160         /* Packets with VID 0 are always received by Lancer by default */
1161         if (lancer_chip(adapter) && vid == 0)
1162                 goto ret;
1163
1164         adapter->vlan_tag[vid] = 0;
1165         status = be_vid_config(adapter);
1166         if (!status)
1167                 adapter->vlans_added--;
1168         else
1169                 adapter->vlan_tag[vid] = 1;
1170 ret:
1171         return status;
1172 }
1173
1174 static void be_clear_promisc(struct be_adapter *adapter)
1175 {
1176         adapter->promiscuous = false;
1177         adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1178
1179         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1180 }
1181
1182 static void be_set_rx_mode(struct net_device *netdev)
1183 {
1184         struct be_adapter *adapter = netdev_priv(netdev);
1185         int status;
1186
1187         if (netdev->flags & IFF_PROMISC) {
1188                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1189                 adapter->promiscuous = true;
1190                 goto done;
1191         }
1192
1193         /* BE was previously in promiscuous mode; disable it */
1194         if (adapter->promiscuous) {
1195                 be_clear_promisc(adapter);
1196                 if (adapter->vlans_added)
1197                         be_vid_config(adapter);
1198         }
1199
1200         /* Enable multicast promisc if num configured exceeds what we support */
1201         if (netdev->flags & IFF_ALLMULTI ||
1202             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1203                 be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1204                 goto done;
1205         }
1206
1207         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1208                 struct netdev_hw_addr *ha;
1209                 int i = 1; /* First slot is claimed by the Primary MAC */
1210
1211                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1212                         be_cmd_pmac_del(adapter, adapter->if_handle,
1213                                         adapter->pmac_id[i], 0);
1214                 }
1215
1216                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1217                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1218                         adapter->promiscuous = true;
1219                         goto done;
1220                 }
1221
1222                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1223                         adapter->uc_macs++; /* First slot is for Primary MAC */
1224                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1225                                         adapter->if_handle,
1226                                         &adapter->pmac_id[adapter->uc_macs], 0);
1227                 }
1228         }
1229
1230         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1231
1232         /* Set to MCAST promisc mode if setting MULTICAST address fails */
1233         if (status) {
1234                 dev_info(&adapter->pdev->dev, "Exhausted multicast HW filters.\n");
1235                 dev_info(&adapter->pdev->dev, "Disabling HW multicast filtering.\n");
1236                 be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1237         }
1238 done:
1239         return;
1240 }
1241
1242 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1243 {
1244         struct be_adapter *adapter = netdev_priv(netdev);
1245         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1246         int status;
1247
1248         if (!sriov_enabled(adapter))
1249                 return -EPERM;
1250
1251         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1252                 return -EINVAL;
1253
1254         if (BEx_chip(adapter)) {
1255                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1256                                 vf + 1);
1257
1258                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1259                                          &vf_cfg->pmac_id, vf + 1);
1260         } else {
1261                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1262                                         vf + 1);
1263         }
1264
1265         if (status)
1266                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed\n",
1267                                 mac, vf);
1268         else
1269                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
1270
1271         return status;
1272 }
1273
1274 static int be_get_vf_config(struct net_device *netdev, int vf,
1275                         struct ifla_vf_info *vi)
1276 {
1277         struct be_adapter *adapter = netdev_priv(netdev);
1278         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1279
1280         if (!sriov_enabled(adapter))
1281                 return -EPERM;
1282
1283         if (vf >= adapter->num_vfs)
1284                 return -EINVAL;
1285
1286         vi->vf = vf;
1287         vi->tx_rate = vf_cfg->tx_rate;
1288         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1289         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1290         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1291
1292         return 0;
1293 }
1294
1295 static int be_set_vf_vlan(struct net_device *netdev,
1296                         int vf, u16 vlan, u8 qos)
1297 {
1298         struct be_adapter *adapter = netdev_priv(netdev);
1299         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1300         int status = 0;
1301
1302         if (!sriov_enabled(adapter))
1303                 return -EPERM;
1304
1305         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1306                 return -EINVAL;
1307
1308         if (vlan || qos) {
1309                 vlan |= qos << VLAN_PRIO_SHIFT;
1310                 if (vf_cfg->vlan_tag != vlan)
1311                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1312                                                        vf_cfg->if_handle, 0);
1313         } else {
1314                 /* Reset Transparent Vlan Tagging. */
1315                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1316                                                vf + 1, vf_cfg->if_handle, 0);
1317         }
1318
1319         if (!status)
1320                 vf_cfg->vlan_tag = vlan;
1321         else
1322                 dev_info(&adapter->pdev->dev,
1323                          "VLAN %d config on VF %d failed\n", vlan, vf);
1324         return status;
1325 }
1326
1327 static int be_set_vf_tx_rate(struct net_device *netdev,
1328                         int vf, int rate)
1329 {
1330         struct be_adapter *adapter = netdev_priv(netdev);
1331         int status = 0;
1332
1333         if (!sriov_enabled(adapter))
1334                 return -EPERM;
1335
1336         if (vf >= adapter->num_vfs)
1337                 return -EINVAL;
1338
1339         if (rate < 100 || rate > 10000) {
1340                 dev_err(&adapter->pdev->dev,
1341                         "tx rate must be between 100 and 10000 Mbps\n");
1342                 return -EINVAL;
1343         }
1344
1345         if (lancer_chip(adapter))
1346                 status = be_cmd_set_profile_config(adapter, rate / 10, vf + 1);
1347         else
1348                 status = be_cmd_set_qos(adapter, rate / 10, vf + 1);
1349
1350         if (status)
1351                 dev_err(&adapter->pdev->dev,
1352                                 "tx rate %d on VF %d failed\n", rate, vf);
1353         else
1354                 adapter->vf_cfg[vf].tx_rate = rate;
1355         return status;
1356 }
1357
1358 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1359                           ulong now)
1360 {
1361         aic->rx_pkts_prev = rx_pkts;
1362         aic->tx_reqs_prev = tx_pkts;
1363         aic->jiffies = now;
1364 }
1365
1366 static void be_eqd_update(struct be_adapter *adapter)
1367 {
1368         struct be_set_eqd set_eqd[MAX_EVT_QS];
1369         int eqd, i, num = 0, start;
1370         struct be_aic_obj *aic;
1371         struct be_eq_obj *eqo;
1372         struct be_rx_obj *rxo;
1373         struct be_tx_obj *txo;
1374         u64 rx_pkts, tx_pkts;
1375         ulong now;
1376         u32 pps, delta;
1377
1378         for_all_evt_queues(adapter, eqo, i) {
1379                 aic = &adapter->aic_obj[eqo->idx];
1380                 if (!aic->enable) {
1381                         if (aic->jiffies)
1382                                 aic->jiffies = 0;
1383                         eqd = aic->et_eqd;
1384                         goto modify_eqd;
1385                 }
1386
1387                 rxo = &adapter->rx_obj[eqo->idx];
1388                 do {
1389                         start = u64_stats_fetch_begin_bh(&rxo->stats.sync);
1390                         rx_pkts = rxo->stats.rx_pkts;
1391                 } while (u64_stats_fetch_retry_bh(&rxo->stats.sync, start));
1392
1393                 txo = &adapter->tx_obj[eqo->idx];
1394                 do {
1395                         start = u64_stats_fetch_begin_bh(&txo->stats.sync);
1396                         tx_pkts = txo->stats.tx_reqs;
1397                 } while (u64_stats_fetch_retry_bh(&txo->stats.sync, start));
1398
1399
1400                 /* Skip, if wrapped around or first calculation */
1401                 now = jiffies;
1402                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1403                     rx_pkts < aic->rx_pkts_prev ||
1404                     tx_pkts < aic->tx_reqs_prev) {
1405                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1406                         continue;
1407                 }
1408
1409                 delta = jiffies_to_msecs(now - aic->jiffies);
1410                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1411                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1412                 eqd = (pps / 15000) << 2;
1413
1414                 if (eqd < 8)
1415                         eqd = 0;
1416                 eqd = min_t(u32, eqd, aic->max_eqd);
1417                 eqd = max_t(u32, eqd, aic->min_eqd);
1418
1419                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1420 modify_eqd:
1421                 if (eqd != aic->prev_eqd) {
1422                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1423                         set_eqd[num].eq_id = eqo->q.id;
1424                         aic->prev_eqd = eqd;
1425                         num++;
1426                 }
1427         }
1428
1429         if (num)
1430                 be_cmd_modify_eqd(adapter, set_eqd, num);
1431 }
1432
1433 static void be_rx_stats_update(struct be_rx_obj *rxo,
1434                 struct be_rx_compl_info *rxcp)
1435 {
1436         struct be_rx_stats *stats = rx_stats(rxo);
1437
1438         u64_stats_update_begin(&stats->sync);
1439         stats->rx_compl++;
1440         stats->rx_bytes += rxcp->pkt_size;
1441         stats->rx_pkts++;
1442         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1443                 stats->rx_mcast_pkts++;
1444         if (rxcp->err)
1445                 stats->rx_compl_err++;
1446         u64_stats_update_end(&stats->sync);
1447 }
1448
1449 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1450 {
1451         /* L4 checksum is not reliable for non TCP/UDP packets.
1452          * Also ignore ipcksm for ipv6 pkts */
1453         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1454                                 (rxcp->ip_csum || rxcp->ipv6);
1455 }
1456
1457 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1458 {
1459         struct be_adapter *adapter = rxo->adapter;
1460         struct be_rx_page_info *rx_page_info;
1461         struct be_queue_info *rxq = &rxo->q;
1462         u16 frag_idx = rxq->tail;
1463
1464         rx_page_info = &rxo->page_info_tbl[frag_idx];
1465         BUG_ON(!rx_page_info->page);
1466
1467         if (rx_page_info->last_page_user) {
1468                 dma_unmap_page(&adapter->pdev->dev,
1469                                dma_unmap_addr(rx_page_info, bus),
1470                                adapter->big_page_size, DMA_FROM_DEVICE);
1471                 rx_page_info->last_page_user = false;
1472         }
1473
1474         queue_tail_inc(rxq);
1475         atomic_dec(&rxq->used);
1476         return rx_page_info;
1477 }
1478
1479 /* Throwaway the data in the Rx completion */
1480 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1481                                 struct be_rx_compl_info *rxcp)
1482 {
1483         struct be_rx_page_info *page_info;
1484         u16 i, num_rcvd = rxcp->num_rcvd;
1485
1486         for (i = 0; i < num_rcvd; i++) {
1487                 page_info = get_rx_page_info(rxo);
1488                 put_page(page_info->page);
1489                 memset(page_info, 0, sizeof(*page_info));
1490         }
1491 }
1492
1493 /*
1494  * skb_fill_rx_data forms a complete skb for an ether frame
1495  * indicated by rxcp.
1496  */
1497 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1498                              struct be_rx_compl_info *rxcp)
1499 {
1500         struct be_rx_page_info *page_info;
1501         u16 i, j;
1502         u16 hdr_len, curr_frag_len, remaining;
1503         u8 *start;
1504
1505         page_info = get_rx_page_info(rxo);
1506         start = page_address(page_info->page) + page_info->page_offset;
1507         prefetch(start);
1508
1509         /* Copy data in the first descriptor of this completion */
1510         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1511
1512         skb->len = curr_frag_len;
1513         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1514                 memcpy(skb->data, start, curr_frag_len);
1515                 /* Complete packet has now been moved to data */
1516                 put_page(page_info->page);
1517                 skb->data_len = 0;
1518                 skb->tail += curr_frag_len;
1519         } else {
1520                 hdr_len = ETH_HLEN;
1521                 memcpy(skb->data, start, hdr_len);
1522                 skb_shinfo(skb)->nr_frags = 1;
1523                 skb_frag_set_page(skb, 0, page_info->page);
1524                 skb_shinfo(skb)->frags[0].page_offset =
1525                                         page_info->page_offset + hdr_len;
1526                 skb_frag_size_set(&skb_shinfo(skb)->frags[0], curr_frag_len - hdr_len);
1527                 skb->data_len = curr_frag_len - hdr_len;
1528                 skb->truesize += rx_frag_size;
1529                 skb->tail += hdr_len;
1530         }
1531         page_info->page = NULL;
1532
1533         if (rxcp->pkt_size <= rx_frag_size) {
1534                 BUG_ON(rxcp->num_rcvd != 1);
1535                 return;
1536         }
1537
1538         /* More frags present for this completion */
1539         remaining = rxcp->pkt_size - curr_frag_len;
1540         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1541                 page_info = get_rx_page_info(rxo);
1542                 curr_frag_len = min(remaining, rx_frag_size);
1543
1544                 /* Coalesce all frags from the same physical page in one slot */
1545                 if (page_info->page_offset == 0) {
1546                         /* Fresh page */
1547                         j++;
1548                         skb_frag_set_page(skb, j, page_info->page);
1549                         skb_shinfo(skb)->frags[j].page_offset =
1550                                                         page_info->page_offset;
1551                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1552                         skb_shinfo(skb)->nr_frags++;
1553                 } else {
1554                         put_page(page_info->page);
1555                 }
1556
1557                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1558                 skb->len += curr_frag_len;
1559                 skb->data_len += curr_frag_len;
1560                 skb->truesize += rx_frag_size;
1561                 remaining -= curr_frag_len;
1562                 page_info->page = NULL;
1563         }
1564         BUG_ON(j > MAX_SKB_FRAGS);
1565 }
1566
1567 /* Process the RX completion indicated by rxcp when GRO is disabled */
1568 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1569                                 struct be_rx_compl_info *rxcp)
1570 {
1571         struct be_adapter *adapter = rxo->adapter;
1572         struct net_device *netdev = adapter->netdev;
1573         struct sk_buff *skb;
1574
1575         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1576         if (unlikely(!skb)) {
1577                 rx_stats(rxo)->rx_drops_no_skbs++;
1578                 be_rx_compl_discard(rxo, rxcp);
1579                 return;
1580         }
1581
1582         skb_fill_rx_data(rxo, skb, rxcp);
1583
1584         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1585                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1586         else
1587                 skb_checksum_none_assert(skb);
1588
1589         skb->protocol = eth_type_trans(skb, netdev);
1590         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1591         if (netdev->features & NETIF_F_RXHASH)
1592                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1593         skb_mark_napi_id(skb, napi);
1594
1595         if (rxcp->vlanf)
1596                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1597
1598         netif_receive_skb(skb);
1599 }
1600
1601 /* Process the RX completion indicated by rxcp when GRO is enabled */
1602 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1603                                     struct napi_struct *napi,
1604                                     struct be_rx_compl_info *rxcp)
1605 {
1606         struct be_adapter *adapter = rxo->adapter;
1607         struct be_rx_page_info *page_info;
1608         struct sk_buff *skb = NULL;
1609         u16 remaining, curr_frag_len;
1610         u16 i, j;
1611
1612         skb = napi_get_frags(napi);
1613         if (!skb) {
1614                 be_rx_compl_discard(rxo, rxcp);
1615                 return;
1616         }
1617
1618         remaining = rxcp->pkt_size;
1619         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1620                 page_info = get_rx_page_info(rxo);
1621
1622                 curr_frag_len = min(remaining, rx_frag_size);
1623
1624                 /* Coalesce all frags from the same physical page in one slot */
1625                 if (i == 0 || page_info->page_offset == 0) {
1626                         /* First frag or Fresh page */
1627                         j++;
1628                         skb_frag_set_page(skb, j, page_info->page);
1629                         skb_shinfo(skb)->frags[j].page_offset =
1630                                                         page_info->page_offset;
1631                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1632                 } else {
1633                         put_page(page_info->page);
1634                 }
1635                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1636                 skb->truesize += rx_frag_size;
1637                 remaining -= curr_frag_len;
1638                 memset(page_info, 0, sizeof(*page_info));
1639         }
1640         BUG_ON(j > MAX_SKB_FRAGS);
1641
1642         skb_shinfo(skb)->nr_frags = j + 1;
1643         skb->len = rxcp->pkt_size;
1644         skb->data_len = rxcp->pkt_size;
1645         skb->ip_summed = CHECKSUM_UNNECESSARY;
1646         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1647         if (adapter->netdev->features & NETIF_F_RXHASH)
1648                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1649         skb_mark_napi_id(skb, napi);
1650
1651         if (rxcp->vlanf)
1652                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1653
1654         napi_gro_frags(napi);
1655 }
1656
1657 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1658                                  struct be_rx_compl_info *rxcp)
1659 {
1660         rxcp->pkt_size =
1661                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, pktsize, compl);
1662         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtp, compl);
1663         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, err, compl);
1664         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tcpf, compl);
1665         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, udpf, compl);
1666         rxcp->ip_csum =
1667                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ipcksm, compl);
1668         rxcp->l4_csum =
1669                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, l4_cksm, compl);
1670         rxcp->ipv6 =
1671                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ip_version, compl);
1672         rxcp->num_rcvd =
1673                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, numfrags, compl);
1674         rxcp->pkt_type =
1675                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, cast_enc, compl);
1676         rxcp->rss_hash =
1677                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, rsshash, compl);
1678         if (rxcp->vlanf) {
1679                 rxcp->vtm = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtm,
1680                                           compl);
1681                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vlan_tag,
1682                                                compl);
1683         }
1684         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, port, compl);
1685 }
1686
1687 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1688                                  struct be_rx_compl_info *rxcp)
1689 {
1690         rxcp->pkt_size =
1691                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, pktsize, compl);
1692         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtp, compl);
1693         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, err, compl);
1694         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, tcpf, compl);
1695         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, udpf, compl);
1696         rxcp->ip_csum =
1697                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ipcksm, compl);
1698         rxcp->l4_csum =
1699                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, l4_cksm, compl);
1700         rxcp->ipv6 =
1701                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ip_version, compl);
1702         rxcp->num_rcvd =
1703                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, numfrags, compl);
1704         rxcp->pkt_type =
1705                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, cast_enc, compl);
1706         rxcp->rss_hash =
1707                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, rsshash, compl);
1708         if (rxcp->vlanf) {
1709                 rxcp->vtm = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtm,
1710                                           compl);
1711                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vlan_tag,
1712                                                compl);
1713         }
1714         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, port, compl);
1715         rxcp->ip_frag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0,
1716                                       ip_frag, compl);
1717 }
1718
1719 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1720 {
1721         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1722         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1723         struct be_adapter *adapter = rxo->adapter;
1724
1725         /* For checking the valid bit it is Ok to use either definition as the
1726          * valid bit is at the same position in both v0 and v1 Rx compl */
1727         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1728                 return NULL;
1729
1730         rmb();
1731         be_dws_le_to_cpu(compl, sizeof(*compl));
1732
1733         if (adapter->be3_native)
1734                 be_parse_rx_compl_v1(compl, rxcp);
1735         else
1736                 be_parse_rx_compl_v0(compl, rxcp);
1737
1738         if (rxcp->ip_frag)
1739                 rxcp->l4_csum = 0;
1740
1741         if (rxcp->vlanf) {
1742                 /* vlanf could be wrongly set in some cards.
1743                  * ignore if vtm is not set */
1744                 if ((adapter->function_mode & FLEX10_MODE) && !rxcp->vtm)
1745                         rxcp->vlanf = 0;
1746
1747                 if (!lancer_chip(adapter))
1748                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1749
1750                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1751                     !adapter->vlan_tag[rxcp->vlan_tag])
1752                         rxcp->vlanf = 0;
1753         }
1754
1755         /* As the compl has been parsed, reset it; we wont touch it again */
1756         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1757
1758         queue_tail_inc(&rxo->cq);
1759         return rxcp;
1760 }
1761
1762 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1763 {
1764         u32 order = get_order(size);
1765
1766         if (order > 0)
1767                 gfp |= __GFP_COMP;
1768         return  alloc_pages(gfp, order);
1769 }
1770
1771 /*
1772  * Allocate a page, split it to fragments of size rx_frag_size and post as
1773  * receive buffers to BE
1774  */
1775 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
1776 {
1777         struct be_adapter *adapter = rxo->adapter;
1778         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1779         struct be_queue_info *rxq = &rxo->q;
1780         struct page *pagep = NULL;
1781         struct device *dev = &adapter->pdev->dev;
1782         struct be_eth_rx_d *rxd;
1783         u64 page_dmaaddr = 0, frag_dmaaddr;
1784         u32 posted, page_offset = 0;
1785
1786         page_info = &rxo->page_info_tbl[rxq->head];
1787         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1788                 if (!pagep) {
1789                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1790                         if (unlikely(!pagep)) {
1791                                 rx_stats(rxo)->rx_post_fail++;
1792                                 break;
1793                         }
1794                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1795                                                     adapter->big_page_size,
1796                                                     DMA_FROM_DEVICE);
1797                         if (dma_mapping_error(dev, page_dmaaddr)) {
1798                                 put_page(pagep);
1799                                 pagep = NULL;
1800                                 rx_stats(rxo)->rx_post_fail++;
1801                                 break;
1802                         }
1803                         page_info->page_offset = 0;
1804                 } else {
1805                         get_page(pagep);
1806                         page_info->page_offset = page_offset + rx_frag_size;
1807                 }
1808                 page_offset = page_info->page_offset;
1809                 page_info->page = pagep;
1810                 dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1811                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1812
1813                 rxd = queue_head_node(rxq);
1814                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1815                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1816
1817                 /* Any space left in the current big page for another frag? */
1818                 if ((page_offset + rx_frag_size + rx_frag_size) >
1819                                         adapter->big_page_size) {
1820                         pagep = NULL;
1821                         page_info->last_page_user = true;
1822                 }
1823
1824                 prev_page_info = page_info;
1825                 queue_head_inc(rxq);
1826                 page_info = &rxo->page_info_tbl[rxq->head];
1827         }
1828         if (pagep)
1829                 prev_page_info->last_page_user = true;
1830
1831         if (posted) {
1832                 atomic_add(posted, &rxq->used);
1833                 if (rxo->rx_post_starved)
1834                         rxo->rx_post_starved = false;
1835                 be_rxq_notify(adapter, rxq->id, posted);
1836         } else if (atomic_read(&rxq->used) == 0) {
1837                 /* Let be_worker replenish when memory is available */
1838                 rxo->rx_post_starved = true;
1839         }
1840 }
1841
1842 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1843 {
1844         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1845
1846         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1847                 return NULL;
1848
1849         rmb();
1850         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1851
1852         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1853
1854         queue_tail_inc(tx_cq);
1855         return txcp;
1856 }
1857
1858 static u16 be_tx_compl_process(struct be_adapter *adapter,
1859                 struct be_tx_obj *txo, u16 last_index)
1860 {
1861         struct be_queue_info *txq = &txo->q;
1862         struct be_eth_wrb *wrb;
1863         struct sk_buff **sent_skbs = txo->sent_skb_list;
1864         struct sk_buff *sent_skb;
1865         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1866         bool unmap_skb_hdr = true;
1867
1868         sent_skb = sent_skbs[txq->tail];
1869         BUG_ON(!sent_skb);
1870         sent_skbs[txq->tail] = NULL;
1871
1872         /* skip header wrb */
1873         queue_tail_inc(txq);
1874
1875         do {
1876                 cur_index = txq->tail;
1877                 wrb = queue_tail_node(txq);
1878                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1879                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1880                 unmap_skb_hdr = false;
1881
1882                 num_wrbs++;
1883                 queue_tail_inc(txq);
1884         } while (cur_index != last_index);
1885
1886         kfree_skb(sent_skb);
1887         return num_wrbs;
1888 }
1889
1890 /* Return the number of events in the event queue */
1891 static inline int events_get(struct be_eq_obj *eqo)
1892 {
1893         struct be_eq_entry *eqe;
1894         int num = 0;
1895
1896         do {
1897                 eqe = queue_tail_node(&eqo->q);
1898                 if (eqe->evt == 0)
1899                         break;
1900
1901                 rmb();
1902                 eqe->evt = 0;
1903                 num++;
1904                 queue_tail_inc(&eqo->q);
1905         } while (true);
1906
1907         return num;
1908 }
1909
1910 /* Leaves the EQ is disarmed state */
1911 static void be_eq_clean(struct be_eq_obj *eqo)
1912 {
1913         int num = events_get(eqo);
1914
1915         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
1916 }
1917
1918 static void be_rx_cq_clean(struct be_rx_obj *rxo)
1919 {
1920         struct be_rx_page_info *page_info;
1921         struct be_queue_info *rxq = &rxo->q;
1922         struct be_queue_info *rx_cq = &rxo->cq;
1923         struct be_rx_compl_info *rxcp;
1924         struct be_adapter *adapter = rxo->adapter;
1925         int flush_wait = 0;
1926
1927         /* Consume pending rx completions.
1928          * Wait for the flush completion (identified by zero num_rcvd)
1929          * to arrive. Notify CQ even when there are no more CQ entries
1930          * for HW to flush partially coalesced CQ entries.
1931          * In Lancer, there is no need to wait for flush compl.
1932          */
1933         for (;;) {
1934                 rxcp = be_rx_compl_get(rxo);
1935                 if (rxcp == NULL) {
1936                         if (lancer_chip(adapter))
1937                                 break;
1938
1939                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
1940                                 dev_warn(&adapter->pdev->dev,
1941                                          "did not receive flush compl\n");
1942                                 break;
1943                         }
1944                         be_cq_notify(adapter, rx_cq->id, true, 0);
1945                         mdelay(1);
1946                 } else {
1947                         be_rx_compl_discard(rxo, rxcp);
1948                         be_cq_notify(adapter, rx_cq->id, false, 1);
1949                         if (rxcp->num_rcvd == 0)
1950                                 break;
1951                 }
1952         }
1953
1954         /* After cleanup, leave the CQ in unarmed state */
1955         be_cq_notify(adapter, rx_cq->id, false, 0);
1956
1957         /* Then free posted rx buffers that were not used */
1958         while (atomic_read(&rxq->used) > 0) {
1959                 page_info = get_rx_page_info(rxo);
1960                 put_page(page_info->page);
1961                 memset(page_info, 0, sizeof(*page_info));
1962         }
1963         BUG_ON(atomic_read(&rxq->used));
1964         rxq->tail = rxq->head = 0;
1965 }
1966
1967 static void be_tx_compl_clean(struct be_adapter *adapter)
1968 {
1969         struct be_tx_obj *txo;
1970         struct be_queue_info *txq;
1971         struct be_eth_tx_compl *txcp;
1972         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
1973         struct sk_buff *sent_skb;
1974         bool dummy_wrb;
1975         int i, pending_txqs;
1976
1977         /* Wait for a max of 200ms for all the tx-completions to arrive. */
1978         do {
1979                 pending_txqs = adapter->num_tx_qs;
1980
1981                 for_all_tx_queues(adapter, txo, i) {
1982                         txq = &txo->q;
1983                         while ((txcp = be_tx_compl_get(&txo->cq))) {
1984                                 end_idx =
1985                                         AMAP_GET_BITS(struct amap_eth_tx_compl,
1986                                                       wrb_index, txcp);
1987                                 num_wrbs += be_tx_compl_process(adapter, txo,
1988                                                                 end_idx);
1989                                 cmpl++;
1990                         }
1991                         if (cmpl) {
1992                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
1993                                 atomic_sub(num_wrbs, &txq->used);
1994                                 cmpl = 0;
1995                                 num_wrbs = 0;
1996                         }
1997                         if (atomic_read(&txq->used) == 0)
1998                                 pending_txqs--;
1999                 }
2000
2001                 if (pending_txqs == 0 || ++timeo > 200)
2002                         break;
2003
2004                 mdelay(1);
2005         } while (true);
2006
2007         for_all_tx_queues(adapter, txo, i) {
2008                 txq = &txo->q;
2009                 if (atomic_read(&txq->used))
2010                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2011                                 atomic_read(&txq->used));
2012
2013                 /* free posted tx for which compls will never arrive */
2014                 while (atomic_read(&txq->used)) {
2015                         sent_skb = txo->sent_skb_list[txq->tail];
2016                         end_idx = txq->tail;
2017                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2018                                                    &dummy_wrb);
2019                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2020                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2021                         atomic_sub(num_wrbs, &txq->used);
2022                 }
2023         }
2024 }
2025
2026 static void be_evt_queues_destroy(struct be_adapter *adapter)
2027 {
2028         struct be_eq_obj *eqo;
2029         int i;
2030
2031         for_all_evt_queues(adapter, eqo, i) {
2032                 if (eqo->q.created) {
2033                         be_eq_clean(eqo);
2034                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2035                         napi_hash_del(&eqo->napi);
2036                         netif_napi_del(&eqo->napi);
2037                 }
2038                 be_queue_free(adapter, &eqo->q);
2039         }
2040 }
2041
2042 static int be_evt_queues_create(struct be_adapter *adapter)
2043 {
2044         struct be_queue_info *eq;
2045         struct be_eq_obj *eqo;
2046         struct be_aic_obj *aic;
2047         int i, rc;
2048
2049         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2050                                     adapter->cfg_num_qs);
2051
2052         for_all_evt_queues(adapter, eqo, i) {
2053                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2054                                BE_NAPI_WEIGHT);
2055                 napi_hash_add(&eqo->napi);
2056                 aic = &adapter->aic_obj[i];
2057                 eqo->adapter = adapter;
2058                 eqo->tx_budget = BE_TX_BUDGET;
2059                 eqo->idx = i;
2060                 aic->max_eqd = BE_MAX_EQD;
2061                 aic->enable = true;
2062
2063                 eq = &eqo->q;
2064                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2065                                         sizeof(struct be_eq_entry));
2066                 if (rc)
2067                         return rc;
2068
2069                 rc = be_cmd_eq_create(adapter, eqo);
2070                 if (rc)
2071                         return rc;
2072         }
2073         return 0;
2074 }
2075
2076 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2077 {
2078         struct be_queue_info *q;
2079
2080         q = &adapter->mcc_obj.q;
2081         if (q->created)
2082                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2083         be_queue_free(adapter, q);
2084
2085         q = &adapter->mcc_obj.cq;
2086         if (q->created)
2087                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2088         be_queue_free(adapter, q);
2089 }
2090
2091 /* Must be called only after TX qs are created as MCC shares TX EQ */
2092 static int be_mcc_queues_create(struct be_adapter *adapter)
2093 {
2094         struct be_queue_info *q, *cq;
2095
2096         cq = &adapter->mcc_obj.cq;
2097         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2098                         sizeof(struct be_mcc_compl)))
2099                 goto err;
2100
2101         /* Use the default EQ for MCC completions */
2102         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2103                 goto mcc_cq_free;
2104
2105         q = &adapter->mcc_obj.q;
2106         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2107                 goto mcc_cq_destroy;
2108
2109         if (be_cmd_mccq_create(adapter, q, cq))
2110                 goto mcc_q_free;
2111
2112         return 0;
2113
2114 mcc_q_free:
2115         be_queue_free(adapter, q);
2116 mcc_cq_destroy:
2117         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2118 mcc_cq_free:
2119         be_queue_free(adapter, cq);
2120 err:
2121         return -1;
2122 }
2123
2124 static void be_tx_queues_destroy(struct be_adapter *adapter)
2125 {
2126         struct be_queue_info *q;
2127         struct be_tx_obj *txo;
2128         u8 i;
2129
2130         for_all_tx_queues(adapter, txo, i) {
2131                 q = &txo->q;
2132                 if (q->created)
2133                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2134                 be_queue_free(adapter, q);
2135
2136                 q = &txo->cq;
2137                 if (q->created)
2138                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2139                 be_queue_free(adapter, q);
2140         }
2141 }
2142
2143 static int be_tx_qs_create(struct be_adapter *adapter)
2144 {
2145         struct be_queue_info *cq, *eq;
2146         struct be_tx_obj *txo;
2147         int status, i;
2148
2149         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2150
2151         for_all_tx_queues(adapter, txo, i) {
2152                 cq = &txo->cq;
2153                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2154                                         sizeof(struct be_eth_tx_compl));
2155                 if (status)
2156                         return status;
2157
2158                 u64_stats_init(&txo->stats.sync);
2159                 u64_stats_init(&txo->stats.sync_compl);
2160
2161                 /* If num_evt_qs is less than num_tx_qs, then more than
2162                  * one txq share an eq
2163                  */
2164                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2165                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2166                 if (status)
2167                         return status;
2168
2169                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2170                                         sizeof(struct be_eth_wrb));
2171                 if (status)
2172                         return status;
2173
2174                 status = be_cmd_txq_create(adapter, txo);
2175                 if (status)
2176                         return status;
2177         }
2178
2179         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2180                  adapter->num_tx_qs);
2181         return 0;
2182 }
2183
2184 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2185 {
2186         struct be_queue_info *q;
2187         struct be_rx_obj *rxo;
2188         int i;
2189
2190         for_all_rx_queues(adapter, rxo, i) {
2191                 q = &rxo->cq;
2192                 if (q->created)
2193                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2194                 be_queue_free(adapter, q);
2195         }
2196 }
2197
2198 static int be_rx_cqs_create(struct be_adapter *adapter)
2199 {
2200         struct be_queue_info *eq, *cq;
2201         struct be_rx_obj *rxo;
2202         int rc, i;
2203
2204         /* We can create as many RSS rings as there are EQs. */
2205         adapter->num_rx_qs = adapter->num_evt_qs;
2206
2207         /* We'll use RSS only if atleast 2 RSS rings are supported.
2208          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2209          */
2210         if (adapter->num_rx_qs > 1)
2211                 adapter->num_rx_qs++;
2212
2213         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2214         for_all_rx_queues(adapter, rxo, i) {
2215                 rxo->adapter = adapter;
2216                 cq = &rxo->cq;
2217                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2218                                 sizeof(struct be_eth_rx_compl));
2219                 if (rc)
2220                         return rc;
2221
2222                 u64_stats_init(&rxo->stats.sync);
2223                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2224                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2225                 if (rc)
2226                         return rc;
2227         }
2228
2229         dev_info(&adapter->pdev->dev,
2230                  "created %d RSS queue(s) and 1 default RX queue\n",
2231                  adapter->num_rx_qs - 1);
2232         return 0;
2233 }
2234
2235 static irqreturn_t be_intx(int irq, void *dev)
2236 {
2237         struct be_eq_obj *eqo = dev;
2238         struct be_adapter *adapter = eqo->adapter;
2239         int num_evts = 0;
2240
2241         /* IRQ is not expected when NAPI is scheduled as the EQ
2242          * will not be armed.
2243          * But, this can happen on Lancer INTx where it takes
2244          * a while to de-assert INTx or in BE2 where occasionaly
2245          * an interrupt may be raised even when EQ is unarmed.
2246          * If NAPI is already scheduled, then counting & notifying
2247          * events will orphan them.
2248          */
2249         if (napi_schedule_prep(&eqo->napi)) {
2250                 num_evts = events_get(eqo);
2251                 __napi_schedule(&eqo->napi);
2252                 if (num_evts)
2253                         eqo->spurious_intr = 0;
2254         }
2255         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2256
2257         /* Return IRQ_HANDLED only for the the first spurious intr
2258          * after a valid intr to stop the kernel from branding
2259          * this irq as a bad one!
2260          */
2261         if (num_evts || eqo->spurious_intr++ == 0)
2262                 return IRQ_HANDLED;
2263         else
2264                 return IRQ_NONE;
2265 }
2266
2267 static irqreturn_t be_msix(int irq, void *dev)
2268 {
2269         struct be_eq_obj *eqo = dev;
2270
2271         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2272         napi_schedule(&eqo->napi);
2273         return IRQ_HANDLED;
2274 }
2275
2276 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2277 {
2278         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2279 }
2280
2281 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2282                         int budget, int polling)
2283 {
2284         struct be_adapter *adapter = rxo->adapter;
2285         struct be_queue_info *rx_cq = &rxo->cq;
2286         struct be_rx_compl_info *rxcp;
2287         u32 work_done;
2288
2289         for (work_done = 0; work_done < budget; work_done++) {
2290                 rxcp = be_rx_compl_get(rxo);
2291                 if (!rxcp)
2292                         break;
2293
2294                 /* Is it a flush compl that has no data */
2295                 if (unlikely(rxcp->num_rcvd == 0))
2296                         goto loop_continue;
2297
2298                 /* Discard compl with partial DMA Lancer B0 */
2299                 if (unlikely(!rxcp->pkt_size)) {
2300                         be_rx_compl_discard(rxo, rxcp);
2301                         goto loop_continue;
2302                 }
2303
2304                 /* On BE drop pkts that arrive due to imperfect filtering in
2305                  * promiscuous mode on some skews
2306                  */
2307                 if (unlikely(rxcp->port != adapter->port_num &&
2308                                 !lancer_chip(adapter))) {
2309                         be_rx_compl_discard(rxo, rxcp);
2310                         goto loop_continue;
2311                 }
2312
2313                 /* Don't do gro when we're busy_polling */
2314                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2315                         be_rx_compl_process_gro(rxo, napi, rxcp);
2316                 else
2317                         be_rx_compl_process(rxo, napi, rxcp);
2318
2319 loop_continue:
2320                 be_rx_stats_update(rxo, rxcp);
2321         }
2322
2323         if (work_done) {
2324                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2325
2326                 /* When an rx-obj gets into post_starved state, just
2327                  * let be_worker do the posting.
2328                  */
2329                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2330                     !rxo->rx_post_starved)
2331                         be_post_rx_frags(rxo, GFP_ATOMIC);
2332         }
2333
2334         return work_done;
2335 }
2336
2337 static bool be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2338                           int budget, int idx)
2339 {
2340         struct be_eth_tx_compl *txcp;
2341         int num_wrbs = 0, work_done;
2342
2343         for (work_done = 0; work_done < budget; work_done++) {
2344                 txcp = be_tx_compl_get(&txo->cq);
2345                 if (!txcp)
2346                         break;
2347                 num_wrbs += be_tx_compl_process(adapter, txo,
2348                                 AMAP_GET_BITS(struct amap_eth_tx_compl,
2349                                         wrb_index, txcp));
2350         }
2351
2352         if (work_done) {
2353                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2354                 atomic_sub(num_wrbs, &txo->q.used);
2355
2356                 /* As Tx wrbs have been freed up, wake up netdev queue
2357                  * if it was stopped due to lack of tx wrbs.  */
2358                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2359                         atomic_read(&txo->q.used) < txo->q.len / 2) {
2360                         netif_wake_subqueue(adapter->netdev, idx);
2361                 }
2362
2363                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2364                 tx_stats(txo)->tx_compl += work_done;
2365                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2366         }
2367         return (work_done < budget); /* Done */
2368 }
2369
2370 int be_poll(struct napi_struct *napi, int budget)
2371 {
2372         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2373         struct be_adapter *adapter = eqo->adapter;
2374         int max_work = 0, work, i, num_evts;
2375         struct be_rx_obj *rxo;
2376         bool tx_done;
2377
2378         num_evts = events_get(eqo);
2379
2380         /* Process all TXQs serviced by this EQ */
2381         for (i = eqo->idx; i < adapter->num_tx_qs; i += adapter->num_evt_qs) {
2382                 tx_done = be_process_tx(adapter, &adapter->tx_obj[i],
2383                                         eqo->tx_budget, i);
2384                 if (!tx_done)
2385                         max_work = budget;
2386         }
2387
2388         if (be_lock_napi(eqo)) {
2389                 /* This loop will iterate twice for EQ0 in which
2390                  * completions of the last RXQ (default one) are also processed
2391                  * For other EQs the loop iterates only once
2392                  */
2393                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2394                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2395                         max_work = max(work, max_work);
2396                 }
2397                 be_unlock_napi(eqo);
2398         } else {
2399                 max_work = budget;
2400         }
2401
2402         if (is_mcc_eqo(eqo))
2403                 be_process_mcc(adapter);
2404
2405         if (max_work < budget) {
2406                 napi_complete(napi);
2407                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2408         } else {
2409                 /* As we'll continue in polling mode, count and clear events */
2410                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2411         }
2412         return max_work;
2413 }
2414
2415 #ifdef CONFIG_NET_RX_BUSY_POLL
2416 static int be_busy_poll(struct napi_struct *napi)
2417 {
2418         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2419         struct be_adapter *adapter = eqo->adapter;
2420         struct be_rx_obj *rxo;
2421         int i, work = 0;
2422
2423         if (!be_lock_busy_poll(eqo))
2424                 return LL_FLUSH_BUSY;
2425
2426         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2427                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2428                 if (work)
2429                         break;
2430         }
2431
2432         be_unlock_busy_poll(eqo);
2433         return work;
2434 }
2435 #endif
2436
2437 void be_detect_error(struct be_adapter *adapter)
2438 {
2439         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2440         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2441         u32 i;
2442
2443         if (be_hw_error(adapter))
2444                 return;
2445
2446         if (lancer_chip(adapter)) {
2447                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2448                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2449                         sliport_err1 = ioread32(adapter->db +
2450                                         SLIPORT_ERROR1_OFFSET);
2451                         sliport_err2 = ioread32(adapter->db +
2452                                         SLIPORT_ERROR2_OFFSET);
2453                 }
2454         } else {
2455                 pci_read_config_dword(adapter->pdev,
2456                                 PCICFG_UE_STATUS_LOW, &ue_lo);
2457                 pci_read_config_dword(adapter->pdev,
2458                                 PCICFG_UE_STATUS_HIGH, &ue_hi);
2459                 pci_read_config_dword(adapter->pdev,
2460                                 PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2461                 pci_read_config_dword(adapter->pdev,
2462                                 PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2463
2464                 ue_lo = (ue_lo & ~ue_lo_mask);
2465                 ue_hi = (ue_hi & ~ue_hi_mask);
2466         }
2467
2468         /* On certain platforms BE hardware can indicate spurious UEs.
2469          * Allow the h/w to stop working completely in case of a real UE.
2470          * Hence not setting the hw_error for UE detection.
2471          */
2472         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2473                 adapter->hw_error = true;
2474                 /* Do not log error messages if its a FW reset */
2475                 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2476                     sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2477                         dev_info(&adapter->pdev->dev,
2478                                  "Firmware update in progress\n");
2479                         return;
2480                 } else {
2481                         dev_err(&adapter->pdev->dev,
2482                                 "Error detected in the card\n");
2483                 }
2484         }
2485
2486         if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2487                 dev_err(&adapter->pdev->dev,
2488                         "ERR: sliport status 0x%x\n", sliport_status);
2489                 dev_err(&adapter->pdev->dev,
2490                         "ERR: sliport error1 0x%x\n", sliport_err1);
2491                 dev_err(&adapter->pdev->dev,
2492                         "ERR: sliport error2 0x%x\n", sliport_err2);
2493         }
2494
2495         if (ue_lo) {
2496                 for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2497                         if (ue_lo & 1)
2498                                 dev_err(&adapter->pdev->dev,
2499                                 "UE: %s bit set\n", ue_status_low_desc[i]);
2500                 }
2501         }
2502
2503         if (ue_hi) {
2504                 for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2505                         if (ue_hi & 1)
2506                                 dev_err(&adapter->pdev->dev,
2507                                 "UE: %s bit set\n", ue_status_hi_desc[i]);
2508                 }
2509         }
2510
2511 }
2512
2513 static void be_msix_disable(struct be_adapter *adapter)
2514 {
2515         if (msix_enabled(adapter)) {
2516                 pci_disable_msix(adapter->pdev);
2517                 adapter->num_msix_vec = 0;
2518                 adapter->num_msix_roce_vec = 0;
2519         }
2520 }
2521
2522 static int be_msix_enable(struct be_adapter *adapter)
2523 {
2524         int i, status, num_vec;
2525         struct device *dev = &adapter->pdev->dev;
2526
2527         /* If RoCE is supported, program the max number of NIC vectors that
2528          * may be configured via set-channels, along with vectors needed for
2529          * RoCe. Else, just program the number we'll use initially.
2530          */
2531         if (be_roce_supported(adapter))
2532                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2533                                 2 * num_online_cpus());
2534         else
2535                 num_vec = adapter->cfg_num_qs;
2536
2537         for (i = 0; i < num_vec; i++)
2538                 adapter->msix_entries[i].entry = i;
2539
2540         status = pci_enable_msix(adapter->pdev, adapter->msix_entries, num_vec);
2541         if (status == 0) {
2542                 goto done;
2543         } else if (status >= MIN_MSIX_VECTORS) {
2544                 num_vec = status;
2545                 status = pci_enable_msix(adapter->pdev, adapter->msix_entries,
2546                                          num_vec);
2547                 if (!status)
2548                         goto done;
2549         }
2550
2551         dev_warn(dev, "MSIx enable failed\n");
2552
2553         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2554         if (!be_physfn(adapter))
2555                 return status;
2556         return 0;
2557 done:
2558         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2559                 adapter->num_msix_roce_vec = num_vec / 2;
2560                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2561                          adapter->num_msix_roce_vec);
2562         }
2563
2564         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2565
2566         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2567                  adapter->num_msix_vec);
2568         return 0;
2569 }
2570
2571 static inline int be_msix_vec_get(struct be_adapter *adapter,
2572                                 struct be_eq_obj *eqo)
2573 {
2574         return adapter->msix_entries[eqo->msix_idx].vector;
2575 }
2576
2577 static int be_msix_register(struct be_adapter *adapter)
2578 {
2579         struct net_device *netdev = adapter->netdev;
2580         struct be_eq_obj *eqo;
2581         int status, i, vec;
2582
2583         for_all_evt_queues(adapter, eqo, i) {
2584                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2585                 vec = be_msix_vec_get(adapter, eqo);
2586                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2587                 if (status)
2588                         goto err_msix;
2589         }
2590
2591         return 0;
2592 err_msix:
2593         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2594                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2595         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2596                 status);
2597         be_msix_disable(adapter);
2598         return status;
2599 }
2600
2601 static int be_irq_register(struct be_adapter *adapter)
2602 {
2603         struct net_device *netdev = adapter->netdev;
2604         int status;
2605
2606         if (msix_enabled(adapter)) {
2607                 status = be_msix_register(adapter);
2608                 if (status == 0)
2609                         goto done;
2610                 /* INTx is not supported for VF */
2611                 if (!be_physfn(adapter))
2612                         return status;
2613         }
2614
2615         /* INTx: only the first EQ is used */
2616         netdev->irq = adapter->pdev->irq;
2617         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2618                              &adapter->eq_obj[0]);
2619         if (status) {
2620                 dev_err(&adapter->pdev->dev,
2621                         "INTx request IRQ failed - err %d\n", status);
2622                 return status;
2623         }
2624 done:
2625         adapter->isr_registered = true;
2626         return 0;
2627 }
2628
2629 static void be_irq_unregister(struct be_adapter *adapter)
2630 {
2631         struct net_device *netdev = adapter->netdev;
2632         struct be_eq_obj *eqo;
2633         int i;
2634
2635         if (!adapter->isr_registered)
2636                 return;
2637
2638         /* INTx */
2639         if (!msix_enabled(adapter)) {
2640                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2641                 goto done;
2642         }
2643
2644         /* MSIx */
2645         for_all_evt_queues(adapter, eqo, i)
2646                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2647
2648 done:
2649         adapter->isr_registered = false;
2650 }
2651
2652 static void be_rx_qs_destroy(struct be_adapter *adapter)
2653 {
2654         struct be_queue_info *q;
2655         struct be_rx_obj *rxo;
2656         int i;
2657
2658         for_all_rx_queues(adapter, rxo, i) {
2659                 q = &rxo->q;
2660                 if (q->created) {
2661                         be_cmd_rxq_destroy(adapter, q);
2662                         be_rx_cq_clean(rxo);
2663                 }
2664                 be_queue_free(adapter, q);
2665         }
2666 }
2667
2668 static int be_close(struct net_device *netdev)
2669 {
2670         struct be_adapter *adapter = netdev_priv(netdev);
2671         struct be_eq_obj *eqo;
2672         int i;
2673
2674         be_roce_dev_close(adapter);
2675
2676         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2677                 for_all_evt_queues(adapter, eqo, i) {
2678                         napi_disable(&eqo->napi);
2679                         be_disable_busy_poll(eqo);
2680                 }
2681                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2682         }
2683
2684         be_async_mcc_disable(adapter);
2685
2686         /* Wait for all pending tx completions to arrive so that
2687          * all tx skbs are freed.
2688          */
2689         netif_tx_disable(netdev);
2690         be_tx_compl_clean(adapter);
2691
2692         be_rx_qs_destroy(adapter);
2693
2694         for (i = 1; i < (adapter->uc_macs + 1); i++)
2695                 be_cmd_pmac_del(adapter, adapter->if_handle,
2696                                 adapter->pmac_id[i], 0);
2697         adapter->uc_macs = 0;
2698
2699         for_all_evt_queues(adapter, eqo, i) {
2700                 if (msix_enabled(adapter))
2701                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2702                 else
2703                         synchronize_irq(netdev->irq);
2704                 be_eq_clean(eqo);
2705         }
2706
2707         be_irq_unregister(adapter);
2708
2709         return 0;
2710 }
2711
2712 static int be_rx_qs_create(struct be_adapter *adapter)
2713 {
2714         struct be_rx_obj *rxo;
2715         int rc, i, j;
2716         u8 rsstable[128];
2717
2718         for_all_rx_queues(adapter, rxo, i) {
2719                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2720                                     sizeof(struct be_eth_rx_d));
2721                 if (rc)
2722                         return rc;
2723         }
2724
2725         /* The FW would like the default RXQ to be created first */
2726         rxo = default_rxo(adapter);
2727         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2728                                adapter->if_handle, false, &rxo->rss_id);
2729         if (rc)
2730                 return rc;
2731
2732         for_all_rss_queues(adapter, rxo, i) {
2733                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2734                                        rx_frag_size, adapter->if_handle,
2735                                        true, &rxo->rss_id);
2736                 if (rc)
2737                         return rc;
2738         }
2739
2740         if (be_multi_rxq(adapter)) {
2741                 for (j = 0; j < 128; j += adapter->num_rx_qs - 1) {
2742                         for_all_rss_queues(adapter, rxo, i) {
2743                                 if ((j + i) >= 128)
2744                                         break;
2745                                 rsstable[j + i] = rxo->rss_id;
2746                         }
2747                 }
2748                 adapter->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2749                                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2750
2751                 if (!BEx_chip(adapter))
2752                         adapter->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2753                                                 RSS_ENABLE_UDP_IPV6;
2754         } else {
2755                 /* Disable RSS, if only default RX Q is created */
2756                 adapter->rss_flags = RSS_ENABLE_NONE;
2757         }
2758
2759         rc = be_cmd_rss_config(adapter, rsstable, adapter->rss_flags,
2760                                128);
2761         if (rc) {
2762                 adapter->rss_flags = RSS_ENABLE_NONE;
2763                 return rc;
2764         }
2765
2766         /* First time posting */
2767         for_all_rx_queues(adapter, rxo, i)
2768                 be_post_rx_frags(rxo, GFP_KERNEL);
2769         return 0;
2770 }
2771
2772 static int be_open(struct net_device *netdev)
2773 {
2774         struct be_adapter *adapter = netdev_priv(netdev);
2775         struct be_eq_obj *eqo;
2776         struct be_rx_obj *rxo;
2777         struct be_tx_obj *txo;
2778         u8 link_status;
2779         int status, i;
2780
2781         status = be_rx_qs_create(adapter);
2782         if (status)
2783                 goto err;
2784
2785         status = be_irq_register(adapter);
2786         if (status)
2787                 goto err;
2788
2789         for_all_rx_queues(adapter, rxo, i)
2790                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2791
2792         for_all_tx_queues(adapter, txo, i)
2793                 be_cq_notify(adapter, txo->cq.id, true, 0);
2794
2795         be_async_mcc_enable(adapter);
2796
2797         for_all_evt_queues(adapter, eqo, i) {
2798                 napi_enable(&eqo->napi);
2799                 be_enable_busy_poll(eqo);
2800                 be_eq_notify(adapter, eqo->q.id, true, false, 0);
2801         }
2802         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2803
2804         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2805         if (!status)
2806                 be_link_status_update(adapter, link_status);
2807
2808         netif_tx_start_all_queues(netdev);
2809         be_roce_dev_open(adapter);
2810         return 0;
2811 err:
2812         be_close(adapter->netdev);
2813         return -EIO;
2814 }
2815
2816 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2817 {
2818         struct be_dma_mem cmd;
2819         int status = 0;
2820         u8 mac[ETH_ALEN];
2821
2822         memset(mac, 0, ETH_ALEN);
2823
2824         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2825         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2826                                      GFP_KERNEL);
2827         if (cmd.va == NULL)
2828                 return -1;
2829
2830         if (enable) {
2831                 status = pci_write_config_dword(adapter->pdev,
2832                         PCICFG_PM_CONTROL_OFFSET, PCICFG_PM_CONTROL_MASK);
2833                 if (status) {
2834                         dev_err(&adapter->pdev->dev,
2835                                 "Could not enable Wake-on-lan\n");
2836                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2837                                           cmd.dma);
2838                         return status;
2839                 }
2840                 status = be_cmd_enable_magic_wol(adapter,
2841                                 adapter->netdev->dev_addr, &cmd);
2842                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2843                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2844         } else {
2845                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2846                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2847                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2848         }
2849
2850         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
2851         return status;
2852 }
2853
2854 /*
2855  * Generate a seed MAC address from the PF MAC Address using jhash.
2856  * MAC Address for VFs are assigned incrementally starting from the seed.
2857  * These addresses are programmed in the ASIC by the PF and the VF driver
2858  * queries for the MAC address during its probe.
2859  */
2860 static int be_vf_eth_addr_config(struct be_adapter *adapter)
2861 {
2862         u32 vf;
2863         int status = 0;
2864         u8 mac[ETH_ALEN];
2865         struct be_vf_cfg *vf_cfg;
2866
2867         be_vf_eth_addr_generate(adapter, mac);
2868
2869         for_all_vfs(adapter, vf_cfg, vf) {
2870                 if (BEx_chip(adapter))
2871                         status = be_cmd_pmac_add(adapter, mac,
2872                                                  vf_cfg->if_handle,
2873                                                  &vf_cfg->pmac_id, vf + 1);
2874                 else
2875                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
2876                                                 vf + 1);
2877
2878                 if (status)
2879                         dev_err(&adapter->pdev->dev,
2880                         "Mac address assignment failed for VF %d\n", vf);
2881                 else
2882                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
2883
2884                 mac[5] += 1;
2885         }
2886         return status;
2887 }
2888
2889 static int be_vfs_mac_query(struct be_adapter *adapter)
2890 {
2891         int status, vf;
2892         u8 mac[ETH_ALEN];
2893         struct be_vf_cfg *vf_cfg;
2894
2895         for_all_vfs(adapter, vf_cfg, vf) {
2896                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
2897                                                mac, vf_cfg->if_handle,
2898                                                false, vf+1);
2899                 if (status)
2900                         return status;
2901                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
2902         }
2903         return 0;
2904 }
2905
2906 static void be_vf_clear(struct be_adapter *adapter)
2907 {
2908         struct be_vf_cfg *vf_cfg;
2909         u32 vf;
2910
2911         if (pci_vfs_assigned(adapter->pdev)) {
2912                 dev_warn(&adapter->pdev->dev,
2913                          "VFs are assigned to VMs: not disabling VFs\n");
2914                 goto done;
2915         }
2916
2917         pci_disable_sriov(adapter->pdev);
2918
2919         for_all_vfs(adapter, vf_cfg, vf) {
2920                 if (BEx_chip(adapter))
2921                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
2922                                         vf_cfg->pmac_id, vf + 1);
2923                 else
2924                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
2925                                        vf + 1);
2926
2927                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
2928         }
2929 done:
2930         kfree(adapter->vf_cfg);
2931         adapter->num_vfs = 0;
2932 }
2933
2934 static void be_clear_queues(struct be_adapter *adapter)
2935 {
2936         be_mcc_queues_destroy(adapter);
2937         be_rx_cqs_destroy(adapter);
2938         be_tx_queues_destroy(adapter);
2939         be_evt_queues_destroy(adapter);
2940 }
2941
2942 static void be_cancel_worker(struct be_adapter *adapter)
2943 {
2944         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
2945                 cancel_delayed_work_sync(&adapter->work);
2946                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
2947         }
2948 }
2949
2950 static void be_mac_clear(struct be_adapter *adapter)
2951 {
2952         int i;
2953
2954         if (adapter->pmac_id) {
2955                 for (i = 0; i < (adapter->uc_macs + 1); i++)
2956                         be_cmd_pmac_del(adapter, adapter->if_handle,
2957                                         adapter->pmac_id[i], 0);
2958                 adapter->uc_macs = 0;
2959
2960                 kfree(adapter->pmac_id);
2961                 adapter->pmac_id = NULL;
2962         }
2963 }
2964
2965 static int be_clear(struct be_adapter *adapter)
2966 {
2967         be_cancel_worker(adapter);
2968
2969         if (sriov_enabled(adapter))
2970                 be_vf_clear(adapter);
2971
2972         /* delete the primary mac along with the uc-mac list */
2973         be_mac_clear(adapter);
2974
2975         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
2976
2977         be_clear_queues(adapter);
2978
2979         be_msix_disable(adapter);
2980         return 0;
2981 }
2982
2983 static int be_vfs_if_create(struct be_adapter *adapter)
2984 {
2985         struct be_resources res = {0};
2986         struct be_vf_cfg *vf_cfg;
2987         u32 cap_flags, en_flags, vf;
2988         int status = 0;
2989
2990         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
2991                     BE_IF_FLAGS_MULTICAST;
2992
2993         for_all_vfs(adapter, vf_cfg, vf) {
2994                 if (!BE3_chip(adapter)) {
2995                         status = be_cmd_get_profile_config(adapter, &res,
2996                                                            vf + 1);
2997                         if (!status)
2998                                 cap_flags = res.if_cap_flags;
2999                 }
3000
3001                 /* If a FW profile exists, then cap_flags are updated */
3002                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3003                            BE_IF_FLAGS_BROADCAST | BE_IF_FLAGS_MULTICAST);
3004                 status = be_cmd_if_create(adapter, cap_flags, en_flags,
3005                                           &vf_cfg->if_handle, vf + 1);
3006                 if (status)
3007                         goto err;
3008         }
3009 err:
3010         return status;
3011 }
3012
3013 static int be_vf_setup_init(struct be_adapter *adapter)
3014 {
3015         struct be_vf_cfg *vf_cfg;
3016         int vf;
3017
3018         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3019                                   GFP_KERNEL);
3020         if (!adapter->vf_cfg)
3021                 return -ENOMEM;
3022
3023         for_all_vfs(adapter, vf_cfg, vf) {
3024                 vf_cfg->if_handle = -1;
3025                 vf_cfg->pmac_id = -1;
3026         }
3027         return 0;
3028 }
3029
3030 static int be_vf_setup(struct be_adapter *adapter)
3031 {
3032         struct device *dev = &adapter->pdev->dev;
3033         struct be_vf_cfg *vf_cfg;
3034         int status, old_vfs, vf;
3035         u32 privileges;
3036         u16 lnk_speed;
3037
3038         old_vfs = pci_num_vf(adapter->pdev);
3039         if (old_vfs) {
3040                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3041                 if (old_vfs != num_vfs)
3042                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3043                 adapter->num_vfs = old_vfs;
3044         } else {
3045                 if (num_vfs > be_max_vfs(adapter))
3046                         dev_info(dev, "Device supports %d VFs and not %d\n",
3047                                  be_max_vfs(adapter), num_vfs);
3048                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3049                 if (!adapter->num_vfs)
3050                         return 0;
3051         }
3052
3053         status = be_vf_setup_init(adapter);
3054         if (status)
3055                 goto err;
3056
3057         if (old_vfs) {
3058                 for_all_vfs(adapter, vf_cfg, vf) {
3059                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3060                         if (status)
3061                                 goto err;
3062                 }
3063         } else {
3064                 status = be_vfs_if_create(adapter);
3065                 if (status)
3066                         goto err;
3067         }
3068
3069         if (old_vfs) {
3070                 status = be_vfs_mac_query(adapter);
3071                 if (status)
3072                         goto err;
3073         } else {
3074                 status = be_vf_eth_addr_config(adapter);
3075                 if (status)
3076                         goto err;
3077         }
3078
3079         for_all_vfs(adapter, vf_cfg, vf) {
3080                 /* Allow VFs to programs MAC/VLAN filters */
3081                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3082                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3083                         status = be_cmd_set_fn_privileges(adapter,
3084                                                           privileges |
3085                                                           BE_PRIV_FILTMGMT,
3086                                                           vf + 1);
3087                         if (!status)
3088                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3089                                          vf);
3090                 }
3091
3092                 /* BE3 FW, by default, caps VF TX-rate to 100mbps.
3093                  * Allow full available bandwidth
3094                  */
3095                 if (BE3_chip(adapter) && !old_vfs)
3096                         be_cmd_set_qos(adapter, 1000, vf+1);
3097
3098                 status = be_cmd_link_status_query(adapter, &lnk_speed,
3099                                                   NULL, vf + 1);
3100                 if (!status)
3101                         vf_cfg->tx_rate = lnk_speed;
3102
3103                 if (!old_vfs)
3104                         be_cmd_enable_vf(adapter, vf + 1);
3105         }
3106
3107         if (!old_vfs) {
3108                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3109                 if (status) {
3110                         dev_err(dev, "SRIOV enable failed\n");
3111                         adapter->num_vfs = 0;
3112                         goto err;
3113                 }
3114         }
3115         return 0;
3116 err:
3117         dev_err(dev, "VF setup failed\n");
3118         be_vf_clear(adapter);
3119         return status;
3120 }
3121
3122 /* On BE2/BE3 FW does not suggest the supported limits */
3123 static void BEx_get_resources(struct be_adapter *adapter,
3124                               struct be_resources *res)
3125 {
3126         struct pci_dev *pdev = adapter->pdev;
3127         bool use_sriov = false;
3128         int max_vfs;
3129
3130         max_vfs = pci_sriov_get_totalvfs(pdev);
3131
3132         if (BE3_chip(adapter) && sriov_want(adapter)) {
3133                 res->max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3134                 use_sriov = res->max_vfs;
3135         }
3136
3137         if (be_physfn(adapter))
3138                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3139         else
3140                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3141
3142         if (adapter->function_mode & FLEX10_MODE)
3143                 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3144         else if (adapter->function_mode & UMC_ENABLED)
3145                 res->max_vlans = BE_UMC_NUM_VLANS_SUPPORTED;
3146         else
3147                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3148         res->max_mcast_mac = BE_MAX_MC;
3149
3150         /* For BE3 1Gb ports, F/W does not properly support multiple TXQs */
3151         if (BE2_chip(adapter) || use_sriov || be_is_mc(adapter) ||
3152             !be_physfn(adapter) || (adapter->port_num > 1))
3153                 res->max_tx_qs = 1;
3154         else
3155                 res->max_tx_qs = BE3_MAX_TX_QS;
3156
3157         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3158             !use_sriov && be_physfn(adapter))
3159                 res->max_rss_qs = (adapter->be3_native) ?
3160                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3161         res->max_rx_qs = res->max_rss_qs + 1;
3162
3163         if (be_physfn(adapter))
3164                 res->max_evt_qs = (max_vfs > 0) ?
3165                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3166         else
3167                 res->max_evt_qs = 1;
3168
3169         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3170         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3171                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3172 }
3173
3174 static void be_setup_init(struct be_adapter *adapter)
3175 {
3176         adapter->vlan_prio_bmap = 0xff;
3177         adapter->phy.link_speed = -1;
3178         adapter->if_handle = -1;
3179         adapter->be3_native = false;
3180         adapter->promiscuous = false;
3181         if (be_physfn(adapter))
3182                 adapter->cmd_privileges = MAX_PRIVILEGES;
3183         else
3184                 adapter->cmd_privileges = MIN_PRIVILEGES;
3185 }
3186
3187 static int be_get_resources(struct be_adapter *adapter)
3188 {
3189         struct device *dev = &adapter->pdev->dev;
3190         struct be_resources res = {0};
3191         int status;
3192
3193         if (BEx_chip(adapter)) {
3194                 BEx_get_resources(adapter, &res);
3195                 adapter->res = res;
3196         }
3197
3198         /* For Lancer, SH etc read per-function resource limits from FW.
3199          * GET_FUNC_CONFIG returns per function guaranteed limits.
3200          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3201          */
3202         if (!BEx_chip(adapter)) {
3203                 status = be_cmd_get_func_config(adapter, &res);
3204                 if (status)
3205                         return status;
3206
3207                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3208                 if (be_roce_supported(adapter))
3209                         res.max_evt_qs /= 2;
3210                 adapter->res = res;
3211
3212                 if (be_physfn(adapter)) {
3213                         status = be_cmd_get_profile_config(adapter, &res, 0);
3214                         if (status)
3215                                 return status;
3216                         adapter->res.max_vfs = res.max_vfs;
3217                 }
3218
3219                 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3220                          be_max_txqs(adapter), be_max_rxqs(adapter),
3221                          be_max_rss(adapter), be_max_eqs(adapter),
3222                          be_max_vfs(adapter));
3223                 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3224                          be_max_uc(adapter), be_max_mc(adapter),
3225                          be_max_vlans(adapter));
3226         }
3227
3228         return 0;
3229 }
3230
3231 /* Routine to query per function resource limits */
3232 static int be_get_config(struct be_adapter *adapter)
3233 {
3234         u16 profile_id;
3235         int status;
3236
3237         status = be_cmd_query_fw_cfg(adapter, &adapter->port_num,
3238                                      &adapter->function_mode,
3239                                      &adapter->function_caps,
3240                                      &adapter->asic_rev);
3241         if (status)
3242                 return status;
3243
3244          if (be_physfn(adapter)) {
3245                 status = be_cmd_get_active_profile(adapter, &profile_id);
3246                 if (!status)
3247                         dev_info(&adapter->pdev->dev,
3248                                  "Using profile 0x%x\n", profile_id);
3249         }
3250
3251         status = be_get_resources(adapter);
3252         if (status)
3253                 return status;
3254
3255         /* primary mac needs 1 pmac entry */
3256         adapter->pmac_id = kcalloc(be_max_uc(adapter) + 1, sizeof(u32),
3257                                    GFP_KERNEL);
3258         if (!adapter->pmac_id)
3259                 return -ENOMEM;
3260
3261         /* Sanitize cfg_num_qs based on HW and platform limits */
3262         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3263
3264         return 0;
3265 }
3266
3267 static int be_mac_setup(struct be_adapter *adapter)
3268 {
3269         u8 mac[ETH_ALEN];
3270         int status;
3271
3272         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3273                 status = be_cmd_get_perm_mac(adapter, mac);
3274                 if (status)
3275                         return status;
3276
3277                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3278                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3279         } else {
3280                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3281                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3282         }
3283
3284         /* For BE3-R VFs, the PF programs the initial MAC address */
3285         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3286                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3287                                 &adapter->pmac_id[0], 0);
3288         return 0;
3289 }
3290
3291 static void be_schedule_worker(struct be_adapter *adapter)
3292 {
3293         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3294         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3295 }
3296
3297 static int be_setup_queues(struct be_adapter *adapter)
3298 {
3299         struct net_device *netdev = adapter->netdev;
3300         int status;
3301
3302         status = be_evt_queues_create(adapter);
3303         if (status)
3304                 goto err;
3305
3306         status = be_tx_qs_create(adapter);
3307         if (status)
3308                 goto err;
3309
3310         status = be_rx_cqs_create(adapter);
3311         if (status)
3312                 goto err;
3313
3314         status = be_mcc_queues_create(adapter);
3315         if (status)
3316                 goto err;
3317
3318         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3319         if (status)
3320                 goto err;
3321
3322         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3323         if (status)
3324                 goto err;
3325
3326         return 0;
3327 err:
3328         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3329         return status;
3330 }
3331
3332 int be_update_queues(struct be_adapter *adapter)
3333 {
3334         struct net_device *netdev = adapter->netdev;
3335         int status;
3336
3337         if (netif_running(netdev))
3338                 be_close(netdev);
3339
3340         be_cancel_worker(adapter);
3341
3342         /* If any vectors have been shared with RoCE we cannot re-program
3343          * the MSIx table.
3344          */
3345         if (!adapter->num_msix_roce_vec)
3346                 be_msix_disable(adapter);
3347
3348         be_clear_queues(adapter);
3349
3350         if (!msix_enabled(adapter)) {
3351                 status = be_msix_enable(adapter);
3352                 if (status)
3353                         return status;
3354         }
3355
3356         status = be_setup_queues(adapter);
3357         if (status)
3358                 return status;
3359
3360         be_schedule_worker(adapter);
3361
3362         if (netif_running(netdev))
3363                 status = be_open(netdev);
3364
3365         return status;
3366 }
3367
3368 static int be_setup(struct be_adapter *adapter)
3369 {
3370         struct device *dev = &adapter->pdev->dev;
3371         u32 tx_fc, rx_fc, en_flags;
3372         int status;
3373
3374         be_setup_init(adapter);
3375
3376         if (!lancer_chip(adapter))
3377                 be_cmd_req_native_mode(adapter);
3378
3379         status = be_get_config(adapter);
3380         if (status)
3381                 goto err;
3382
3383         status = be_msix_enable(adapter);
3384         if (status)
3385                 goto err;
3386
3387         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3388                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3389         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3390                 en_flags |= BE_IF_FLAGS_RSS;
3391         en_flags = en_flags & be_if_cap_flags(adapter);
3392         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3393                                   &adapter->if_handle, 0);
3394         if (status)
3395                 goto err;
3396
3397         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3398         rtnl_lock();
3399         status = be_setup_queues(adapter);
3400         rtnl_unlock();
3401         if (status)
3402                 goto err;
3403
3404         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3405
3406         status = be_mac_setup(adapter);
3407         if (status)
3408                 goto err;
3409
3410         be_cmd_get_fw_ver(adapter, adapter->fw_ver, adapter->fw_on_flash);
3411
3412         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3413                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work.",
3414                         adapter->fw_ver);
3415                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3416         }
3417
3418         if (adapter->vlans_added)
3419                 be_vid_config(adapter);
3420
3421         be_set_rx_mode(adapter->netdev);
3422
3423         be_cmd_get_acpi_wol_cap(adapter);
3424
3425         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3426
3427         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3428                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3429                                         adapter->rx_fc);
3430
3431         if (sriov_want(adapter)) {
3432                 if (be_max_vfs(adapter))
3433                         be_vf_setup(adapter);
3434                 else
3435                         dev_warn(dev, "device doesn't support SRIOV\n");
3436         }
3437
3438         status = be_cmd_get_phy_info(adapter);
3439         if (!status && be_pause_supported(adapter))
3440                 adapter->phy.fc_autoneg = 1;
3441
3442         be_schedule_worker(adapter);
3443         return 0;
3444 err:
3445         be_clear(adapter);
3446         return status;
3447 }
3448
3449 #ifdef CONFIG_NET_POLL_CONTROLLER
3450 static void be_netpoll(struct net_device *netdev)
3451 {
3452         struct be_adapter *adapter = netdev_priv(netdev);
3453         struct be_eq_obj *eqo;
3454         int i;
3455
3456         for_all_evt_queues(adapter, eqo, i) {
3457                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3458                 napi_schedule(&eqo->napi);
3459         }
3460
3461         return;
3462 }
3463 #endif
3464
3465 #define FW_FILE_HDR_SIGN        "ServerEngines Corp. "
3466 static char flash_cookie[2][16] =      {"*** SE FLAS", "H DIRECTORY *** "};
3467
3468 static bool be_flash_redboot(struct be_adapter *adapter,
3469                         const u8 *p, u32 img_start, int image_size,
3470                         int hdr_size)
3471 {
3472         u32 crc_offset;
3473         u8 flashed_crc[4];
3474         int status;
3475
3476         crc_offset = hdr_size + img_start + image_size - 4;
3477
3478         p += crc_offset;
3479
3480         status = be_cmd_get_flash_crc(adapter, flashed_crc,
3481                         (image_size - 4));
3482         if (status) {
3483                 dev_err(&adapter->pdev->dev,
3484                 "could not get crc from flash, not flashing redboot\n");
3485                 return false;
3486         }
3487
3488         /*update redboot only if crc does not match*/
3489         if (!memcmp(flashed_crc, p, 4))
3490                 return false;
3491         else
3492                 return true;
3493 }
3494
3495 static bool phy_flashing_required(struct be_adapter *adapter)
3496 {
3497         return (adapter->phy.phy_type == TN_8022 &&
3498                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3499 }
3500
3501 static bool is_comp_in_ufi(struct be_adapter *adapter,
3502                            struct flash_section_info *fsec, int type)
3503 {
3504         int i = 0, img_type = 0;
3505         struct flash_section_info_g2 *fsec_g2 = NULL;
3506
3507         if (BE2_chip(adapter))
3508                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3509
3510         for (i = 0; i < MAX_FLASH_COMP; i++) {
3511                 if (fsec_g2)
3512                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3513                 else
3514                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3515
3516                 if (img_type == type)
3517                         return true;
3518         }
3519         return false;
3520
3521 }
3522
3523 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3524                                          int header_size,
3525                                          const struct firmware *fw)
3526 {
3527         struct flash_section_info *fsec = NULL;
3528         const u8 *p = fw->data;
3529
3530         p += header_size;
3531         while (p < (fw->data + fw->size)) {
3532                 fsec = (struct flash_section_info *)p;
3533                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3534                         return fsec;
3535                 p += 32;
3536         }
3537         return NULL;
3538 }
3539
3540 static int be_flash(struct be_adapter *adapter, const u8 *img,
3541                 struct be_dma_mem *flash_cmd, int optype, int img_size)
3542 {
3543         u32 total_bytes = 0, flash_op, num_bytes = 0;
3544         int status = 0;
3545         struct be_cmd_write_flashrom *req = flash_cmd->va;
3546
3547         total_bytes = img_size;
3548         while (total_bytes) {
3549                 num_bytes = min_t(u32, 32*1024, total_bytes);
3550
3551                 total_bytes -= num_bytes;
3552
3553                 if (!total_bytes) {
3554                         if (optype == OPTYPE_PHY_FW)
3555                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3556                         else
3557                                 flash_op = FLASHROM_OPER_FLASH;
3558                 } else {
3559                         if (optype == OPTYPE_PHY_FW)
3560                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3561                         else
3562                                 flash_op = FLASHROM_OPER_SAVE;
3563                 }
3564
3565                 memcpy(req->data_buf, img, num_bytes);
3566                 img += num_bytes;
3567                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3568                                                 flash_op, num_bytes);
3569                 if (status) {
3570                         if (status == ILLEGAL_IOCTL_REQ &&
3571                             optype == OPTYPE_PHY_FW)
3572                                 break;
3573                         dev_err(&adapter->pdev->dev,
3574                                 "cmd to write to flash rom failed.\n");
3575                         return status;
3576                 }
3577         }
3578         return 0;
3579 }
3580
3581 /* For BE2, BE3 and BE3-R */
3582 static int be_flash_BEx(struct be_adapter *adapter,
3583                          const struct firmware *fw,
3584                          struct be_dma_mem *flash_cmd,
3585                          int num_of_images)
3586
3587 {
3588         int status = 0, i, filehdr_size = 0;
3589         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3590         const u8 *p = fw->data;
3591         const struct flash_comp *pflashcomp;
3592         int num_comp, redboot;
3593         struct flash_section_info *fsec = NULL;
3594
3595         struct flash_comp gen3_flash_types[] = {
3596                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3597                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3598                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3599                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3600                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3601                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3602                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3603                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3604                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3605                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3606                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3607                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3608                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3609                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3610                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3611                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3612                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3613                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3614                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3615                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3616         };
3617
3618         struct flash_comp gen2_flash_types[] = {
3619                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3620                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3621                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3622                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3623                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3624                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3625                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3626                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3627                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3628                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3629                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3630                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3631                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3632                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3633                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3634                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3635         };
3636
3637         if (BE3_chip(adapter)) {
3638                 pflashcomp = gen3_flash_types;
3639                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3640                 num_comp = ARRAY_SIZE(gen3_flash_types);
3641         } else {
3642                 pflashcomp = gen2_flash_types;
3643                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3644                 num_comp = ARRAY_SIZE(gen2_flash_types);
3645         }
3646
3647         /* Get flash section info*/
3648         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3649         if (!fsec) {
3650                 dev_err(&adapter->pdev->dev,
3651                         "Invalid Cookie. UFI corrupted ?\n");
3652                 return -1;
3653         }
3654         for (i = 0; i < num_comp; i++) {
3655                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3656                         continue;
3657
3658                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3659                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3660                         continue;
3661
3662                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3663                     !phy_flashing_required(adapter))
3664                                 continue;
3665
3666                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3667                         redboot = be_flash_redboot(adapter, fw->data,
3668                                 pflashcomp[i].offset, pflashcomp[i].size,
3669                                 filehdr_size + img_hdrs_size);
3670                         if (!redboot)
3671                                 continue;
3672                 }
3673
3674                 p = fw->data;
3675                 p += filehdr_size + pflashcomp[i].offset + img_hdrs_size;
3676                 if (p + pflashcomp[i].size > fw->data + fw->size)
3677                         return -1;
3678
3679                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3680                                         pflashcomp[i].size);
3681                 if (status) {
3682                         dev_err(&adapter->pdev->dev,
3683                                 "Flashing section type %d failed.\n",
3684                                 pflashcomp[i].img_type);
3685                         return status;
3686                 }
3687         }
3688         return 0;
3689 }
3690
3691 static int be_flash_skyhawk(struct be_adapter *adapter,
3692                 const struct firmware *fw,
3693                 struct be_dma_mem *flash_cmd, int num_of_images)
3694 {
3695         int status = 0, i, filehdr_size = 0;
3696         int img_offset, img_size, img_optype, redboot;
3697         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
3698         const u8 *p = fw->data;
3699         struct flash_section_info *fsec = NULL;
3700
3701         filehdr_size = sizeof(struct flash_file_hdr_g3);
3702         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3703         if (!fsec) {
3704                 dev_err(&adapter->pdev->dev,
3705                         "Invalid Cookie. UFI corrupted ?\n");
3706                 return -1;
3707         }
3708
3709         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
3710                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
3711                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
3712
3713                 switch (le32_to_cpu(fsec->fsec_entry[i].type)) {
3714                 case IMAGE_FIRMWARE_iSCSI:
3715                         img_optype = OPTYPE_ISCSI_ACTIVE;
3716                         break;
3717                 case IMAGE_BOOT_CODE:
3718                         img_optype = OPTYPE_REDBOOT;
3719                         break;
3720                 case IMAGE_OPTION_ROM_ISCSI:
3721                         img_optype = OPTYPE_BIOS;
3722                         break;
3723                 case IMAGE_OPTION_ROM_PXE:
3724                         img_optype = OPTYPE_PXE_BIOS;
3725                         break;
3726                 case IMAGE_OPTION_ROM_FCoE:
3727                         img_optype = OPTYPE_FCOE_BIOS;
3728                         break;
3729                 case IMAGE_FIRMWARE_BACKUP_iSCSI:
3730                         img_optype = OPTYPE_ISCSI_BACKUP;
3731                         break;
3732                 case IMAGE_NCSI:
3733                         img_optype = OPTYPE_NCSI_FW;
3734                         break;
3735                 default:
3736                         continue;
3737                 }
3738
3739                 if (img_optype == OPTYPE_REDBOOT) {
3740                         redboot = be_flash_redboot(adapter, fw->data,
3741                                         img_offset, img_size,
3742                                         filehdr_size + img_hdrs_size);
3743                         if (!redboot)
3744                                 continue;
3745                 }
3746
3747                 p = fw->data;
3748                 p += filehdr_size + img_offset + img_hdrs_size;
3749                 if (p + img_size > fw->data + fw->size)
3750                         return -1;
3751
3752                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
3753                 if (status) {
3754                         dev_err(&adapter->pdev->dev,
3755                                 "Flashing section type %d failed.\n",
3756                                 fsec->fsec_entry[i].type);
3757                         return status;
3758                 }
3759         }
3760         return 0;
3761 }
3762
3763 static int lancer_fw_download(struct be_adapter *adapter,
3764                                 const struct firmware *fw)
3765 {
3766 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
3767 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
3768         struct be_dma_mem flash_cmd;
3769         const u8 *data_ptr = NULL;
3770         u8 *dest_image_ptr = NULL;
3771         size_t image_size = 0;
3772         u32 chunk_size = 0;
3773         u32 data_written = 0;
3774         u32 offset = 0;
3775         int status = 0;
3776         u8 add_status = 0;
3777         u8 change_status;
3778
3779         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
3780                 dev_err(&adapter->pdev->dev,
3781                         "FW Image not properly aligned. "
3782                         "Length must be 4 byte aligned.\n");
3783                 status = -EINVAL;
3784                 goto lancer_fw_exit;
3785         }
3786
3787         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
3788                                 + LANCER_FW_DOWNLOAD_CHUNK;
3789         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
3790                                           &flash_cmd.dma, GFP_KERNEL);
3791         if (!flash_cmd.va) {
3792                 status = -ENOMEM;
3793                 goto lancer_fw_exit;
3794         }
3795
3796         dest_image_ptr = flash_cmd.va +
3797                                 sizeof(struct lancer_cmd_req_write_object);
3798         image_size = fw->size;
3799         data_ptr = fw->data;
3800
3801         while (image_size) {
3802                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
3803
3804                 /* Copy the image chunk content. */
3805                 memcpy(dest_image_ptr, data_ptr, chunk_size);
3806
3807                 status = lancer_cmd_write_object(adapter, &flash_cmd,
3808                                                  chunk_size, offset,
3809                                                  LANCER_FW_DOWNLOAD_LOCATION,
3810                                                  &data_written, &change_status,
3811                                                  &add_status);
3812                 if (status)
3813                         break;
3814
3815                 offset += data_written;
3816                 data_ptr += data_written;
3817                 image_size -= data_written;
3818         }
3819
3820         if (!status) {
3821                 /* Commit the FW written */
3822                 status = lancer_cmd_write_object(adapter, &flash_cmd,
3823                                                  0, offset,
3824                                                  LANCER_FW_DOWNLOAD_LOCATION,
3825                                                  &data_written, &change_status,
3826                                                  &add_status);
3827         }
3828
3829         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
3830                                 flash_cmd.dma);
3831         if (status) {
3832                 dev_err(&adapter->pdev->dev,
3833                         "Firmware load error. "
3834                         "Status code: 0x%x Additional Status: 0x%x\n",
3835                         status, add_status);
3836                 goto lancer_fw_exit;
3837         }
3838
3839         if (change_status == LANCER_FW_RESET_NEEDED) {
3840                 dev_info(&adapter->pdev->dev,
3841                          "Resetting adapter to activate new FW\n");
3842                 status = lancer_physdev_ctrl(adapter,
3843                                              PHYSDEV_CONTROL_FW_RESET_MASK);
3844                 if (status) {
3845                         dev_err(&adapter->pdev->dev,
3846                                 "Adapter busy for FW reset.\n"
3847                                 "New FW will not be active.\n");
3848                         goto lancer_fw_exit;
3849                 }
3850         } else if (change_status != LANCER_NO_RESET_NEEDED) {
3851                         dev_err(&adapter->pdev->dev,
3852                                 "System reboot required for new FW"
3853                                 " to be active\n");
3854         }
3855
3856         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
3857 lancer_fw_exit:
3858         return status;
3859 }
3860
3861 #define UFI_TYPE2               2
3862 #define UFI_TYPE3               3
3863 #define UFI_TYPE3R              10
3864 #define UFI_TYPE4               4
3865 static int be_get_ufi_type(struct be_adapter *adapter,
3866                            struct flash_file_hdr_g3 *fhdr)
3867 {
3868         if (fhdr == NULL)
3869                 goto be_get_ufi_exit;
3870
3871         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
3872                 return UFI_TYPE4;
3873         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
3874                 if (fhdr->asic_type_rev == 0x10)
3875                         return UFI_TYPE3R;
3876                 else
3877                         return UFI_TYPE3;
3878         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
3879                 return UFI_TYPE2;
3880
3881 be_get_ufi_exit:
3882         dev_err(&adapter->pdev->dev,
3883                 "UFI and Interface are not compatible for flashing\n");
3884         return -1;
3885 }
3886
3887 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
3888 {
3889         struct flash_file_hdr_g3 *fhdr3;
3890         struct image_hdr *img_hdr_ptr = NULL;
3891         struct be_dma_mem flash_cmd;
3892         const u8 *p;
3893         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
3894
3895         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
3896         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
3897                                           &flash_cmd.dma, GFP_KERNEL);
3898         if (!flash_cmd.va) {
3899                 status = -ENOMEM;
3900                 goto be_fw_exit;
3901         }
3902
3903         p = fw->data;
3904         fhdr3 = (struct flash_file_hdr_g3 *)p;
3905
3906         ufi_type = be_get_ufi_type(adapter, fhdr3);
3907
3908         num_imgs = le32_to_cpu(fhdr3->num_imgs);
3909         for (i = 0; i < num_imgs; i++) {
3910                 img_hdr_ptr = (struct image_hdr *)(fw->data +
3911                                 (sizeof(struct flash_file_hdr_g3) +
3912                                  i * sizeof(struct image_hdr)));
3913                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
3914                         switch (ufi_type) {
3915                         case UFI_TYPE4:
3916                                 status = be_flash_skyhawk(adapter, fw,
3917                                                         &flash_cmd, num_imgs);
3918                                 break;
3919                         case UFI_TYPE3R:
3920                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
3921                                                       num_imgs);
3922                                 break;
3923                         case UFI_TYPE3:
3924                                 /* Do not flash this ufi on BE3-R cards */
3925                                 if (adapter->asic_rev < 0x10)
3926                                         status = be_flash_BEx(adapter, fw,
3927                                                               &flash_cmd,
3928                                                               num_imgs);
3929                                 else {
3930                                         status = -1;
3931                                         dev_err(&adapter->pdev->dev,
3932                                                 "Can't load BE3 UFI on BE3R\n");
3933                                 }
3934                         }
3935                 }
3936         }
3937
3938         if (ufi_type == UFI_TYPE2)
3939                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
3940         else if (ufi_type == -1)
3941                 status = -1;
3942
3943         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
3944                           flash_cmd.dma);
3945         if (status) {
3946                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
3947                 goto be_fw_exit;
3948         }
3949
3950         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
3951
3952 be_fw_exit:
3953         return status;
3954 }
3955
3956 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
3957 {
3958         const struct firmware *fw;
3959         int status;
3960
3961         if (!netif_running(adapter->netdev)) {
3962                 dev_err(&adapter->pdev->dev,
3963                         "Firmware load not allowed (interface is down)\n");
3964                 return -1;
3965         }
3966
3967         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
3968         if (status)
3969                 goto fw_exit;
3970
3971         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
3972
3973         if (lancer_chip(adapter))
3974                 status = lancer_fw_download(adapter, fw);
3975         else
3976                 status = be_fw_download(adapter, fw);
3977
3978         if (!status)
3979                 be_cmd_get_fw_ver(adapter, adapter->fw_ver,
3980                                   adapter->fw_on_flash);
3981
3982 fw_exit:
3983         release_firmware(fw);
3984         return status;
3985 }
3986
3987 static int be_ndo_bridge_setlink(struct net_device *dev,
3988                                     struct nlmsghdr *nlh)
3989 {
3990         struct be_adapter *adapter = netdev_priv(dev);
3991         struct nlattr *attr, *br_spec;
3992         int rem;
3993         int status = 0;
3994         u16 mode = 0;
3995
3996         if (!sriov_enabled(adapter))
3997                 return -EOPNOTSUPP;
3998
3999         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4000
4001         nla_for_each_nested(attr, br_spec, rem) {
4002                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4003                         continue;
4004
4005                 mode = nla_get_u16(attr);
4006                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4007                         return -EINVAL;
4008
4009                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4010                                                adapter->if_handle,
4011                                                mode == BRIDGE_MODE_VEPA ?
4012                                                PORT_FWD_TYPE_VEPA :
4013                                                PORT_FWD_TYPE_VEB);
4014                 if (status)
4015                         goto err;
4016
4017                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4018                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4019
4020                 return status;
4021         }
4022 err:
4023         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4024                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4025
4026         return status;
4027 }
4028
4029 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4030                                     struct net_device *dev,
4031                                     u32 filter_mask)
4032 {
4033         struct be_adapter *adapter = netdev_priv(dev);
4034         int status = 0;
4035         u8 hsw_mode;
4036
4037         if (!sriov_enabled(adapter))
4038                 return 0;
4039
4040         /* BE and Lancer chips support VEB mode only */
4041         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4042                 hsw_mode = PORT_FWD_TYPE_VEB;
4043         } else {
4044                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4045                                                adapter->if_handle, &hsw_mode);
4046                 if (status)
4047                         return 0;
4048         }
4049
4050         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4051                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4052                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4053 }
4054
4055 static const struct net_device_ops be_netdev_ops = {
4056         .ndo_open               = be_open,
4057         .ndo_stop               = be_close,
4058         .ndo_start_xmit         = be_xmit,
4059         .ndo_set_rx_mode        = be_set_rx_mode,
4060         .ndo_set_mac_address    = be_mac_addr_set,
4061         .ndo_change_mtu         = be_change_mtu,
4062         .ndo_get_stats64        = be_get_stats64,
4063         .ndo_validate_addr      = eth_validate_addr,
4064         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4065         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4066         .ndo_set_vf_mac         = be_set_vf_mac,
4067         .ndo_set_vf_vlan        = be_set_vf_vlan,
4068         .ndo_set_vf_tx_rate     = be_set_vf_tx_rate,
4069         .ndo_get_vf_config      = be_get_vf_config,
4070 #ifdef CONFIG_NET_POLL_CONTROLLER
4071         .ndo_poll_controller    = be_netpoll,
4072 #endif
4073         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4074         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4075 #ifdef CONFIG_NET_RX_BUSY_POLL
4076         .ndo_busy_poll          = be_busy_poll
4077 #endif
4078 };
4079
4080 static void be_netdev_init(struct net_device *netdev)
4081 {
4082         struct be_adapter *adapter = netdev_priv(netdev);
4083
4084         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4085                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4086                 NETIF_F_HW_VLAN_CTAG_TX;
4087         if (be_multi_rxq(adapter))
4088                 netdev->hw_features |= NETIF_F_RXHASH;
4089
4090         netdev->features |= netdev->hw_features |
4091                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4092
4093         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4094                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4095
4096         netdev->priv_flags |= IFF_UNICAST_FLT;
4097
4098         netdev->flags |= IFF_MULTICAST;
4099
4100         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4101
4102         netdev->netdev_ops = &be_netdev_ops;
4103
4104         SET_ETHTOOL_OPS(netdev, &be_ethtool_ops);
4105 }
4106
4107 static void be_unmap_pci_bars(struct be_adapter *adapter)
4108 {
4109         if (adapter->csr)
4110                 pci_iounmap(adapter->pdev, adapter->csr);
4111         if (adapter->db)
4112                 pci_iounmap(adapter->pdev, adapter->db);
4113 }
4114
4115 static int db_bar(struct be_adapter *adapter)
4116 {
4117         if (lancer_chip(adapter) || !be_physfn(adapter))
4118                 return 0;
4119         else
4120                 return 4;
4121 }
4122
4123 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4124 {
4125         if (skyhawk_chip(adapter)) {
4126                 adapter->roce_db.size = 4096;
4127                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4128                                                               db_bar(adapter));
4129                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4130                                                                db_bar(adapter));
4131         }
4132         return 0;
4133 }
4134
4135 static int be_map_pci_bars(struct be_adapter *adapter)
4136 {
4137         u8 __iomem *addr;
4138
4139         if (BEx_chip(adapter) && be_physfn(adapter)) {
4140                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4141                 if (adapter->csr == NULL)
4142                         return -ENOMEM;
4143         }
4144
4145         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4146         if (addr == NULL)
4147                 goto pci_map_err;
4148         adapter->db = addr;
4149
4150         be_roce_map_pci_bars(adapter);
4151         return 0;
4152
4153 pci_map_err:
4154         be_unmap_pci_bars(adapter);
4155         return -ENOMEM;
4156 }
4157
4158 static void be_ctrl_cleanup(struct be_adapter *adapter)
4159 {
4160         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4161
4162         be_unmap_pci_bars(adapter);
4163
4164         if (mem->va)
4165                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4166                                   mem->dma);
4167
4168         mem = &adapter->rx_filter;
4169         if (mem->va)
4170                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4171                                   mem->dma);
4172 }
4173
4174 static int be_ctrl_init(struct be_adapter *adapter)
4175 {
4176         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4177         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4178         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4179         u32 sli_intf;
4180         int status;
4181
4182         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4183         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4184                                  SLI_INTF_FAMILY_SHIFT;
4185         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4186
4187         status = be_map_pci_bars(adapter);
4188         if (status)
4189                 goto done;
4190
4191         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4192         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4193                                                 mbox_mem_alloc->size,
4194                                                 &mbox_mem_alloc->dma,
4195                                                 GFP_KERNEL);
4196         if (!mbox_mem_alloc->va) {
4197                 status = -ENOMEM;
4198                 goto unmap_pci_bars;
4199         }
4200         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4201         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4202         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4203         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4204
4205         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4206         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4207                                             rx_filter->size, &rx_filter->dma,
4208                                             GFP_KERNEL);
4209         if (rx_filter->va == NULL) {
4210                 status = -ENOMEM;
4211                 goto free_mbox;
4212         }
4213
4214         mutex_init(&adapter->mbox_lock);
4215         spin_lock_init(&adapter->mcc_lock);
4216         spin_lock_init(&adapter->mcc_cq_lock);
4217
4218         init_completion(&adapter->et_cmd_compl);
4219         pci_save_state(adapter->pdev);
4220         return 0;
4221
4222 free_mbox:
4223         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4224                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4225
4226 unmap_pci_bars:
4227         be_unmap_pci_bars(adapter);
4228
4229 done:
4230         return status;
4231 }
4232
4233 static void be_stats_cleanup(struct be_adapter *adapter)
4234 {
4235         struct be_dma_mem *cmd = &adapter->stats_cmd;
4236
4237         if (cmd->va)
4238                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4239                                   cmd->va, cmd->dma);
4240 }
4241
4242 static int be_stats_init(struct be_adapter *adapter)
4243 {
4244         struct be_dma_mem *cmd = &adapter->stats_cmd;
4245
4246         if (lancer_chip(adapter))
4247                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4248         else if (BE2_chip(adapter))
4249                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4250         else if (BE3_chip(adapter))
4251                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4252         else
4253                 /* ALL non-BE ASICs */
4254                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4255
4256         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4257                                       GFP_KERNEL);
4258         if (cmd->va == NULL)
4259                 return -1;
4260         return 0;
4261 }
4262
4263 static void be_remove(struct pci_dev *pdev)
4264 {
4265         struct be_adapter *adapter = pci_get_drvdata(pdev);
4266
4267         if (!adapter)
4268                 return;
4269
4270         be_roce_dev_remove(adapter);
4271         be_intr_set(adapter, false);
4272
4273         cancel_delayed_work_sync(&adapter->func_recovery_work);
4274
4275         unregister_netdev(adapter->netdev);
4276
4277         be_clear(adapter);
4278
4279         /* tell fw we're done with firing cmds */
4280         be_cmd_fw_clean(adapter);
4281
4282         be_stats_cleanup(adapter);
4283
4284         be_ctrl_cleanup(adapter);
4285
4286         pci_disable_pcie_error_reporting(pdev);
4287
4288         pci_release_regions(pdev);
4289         pci_disable_device(pdev);
4290
4291         free_netdev(adapter->netdev);
4292 }
4293
4294 static int be_get_initial_config(struct be_adapter *adapter)
4295 {
4296         int status, level;
4297
4298         status = be_cmd_get_cntl_attributes(adapter);
4299         if (status)
4300                 return status;
4301
4302         /* Must be a power of 2 or else MODULO will BUG_ON */
4303         adapter->be_get_temp_freq = 64;
4304
4305         if (BEx_chip(adapter)) {
4306                 level = be_cmd_get_fw_log_level(adapter);
4307                 adapter->msg_enable =
4308                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4309         }
4310
4311         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4312         return 0;
4313 }
4314
4315 static int lancer_recover_func(struct be_adapter *adapter)
4316 {
4317         struct device *dev = &adapter->pdev->dev;
4318         int status;
4319
4320         status = lancer_test_and_set_rdy_state(adapter);
4321         if (status)
4322                 goto err;
4323
4324         if (netif_running(adapter->netdev))
4325                 be_close(adapter->netdev);
4326
4327         be_clear(adapter);
4328
4329         be_clear_all_error(adapter);
4330
4331         status = be_setup(adapter);
4332         if (status)
4333                 goto err;
4334
4335         if (netif_running(adapter->netdev)) {
4336                 status = be_open(adapter->netdev);
4337                 if (status)
4338                         goto err;
4339         }
4340
4341         dev_err(dev, "Adapter recovery successful\n");
4342         return 0;
4343 err:
4344         if (status == -EAGAIN)
4345                 dev_err(dev, "Waiting for resource provisioning\n");
4346         else
4347                 dev_err(dev, "Adapter recovery failed\n");
4348
4349         return status;
4350 }
4351
4352 static void be_func_recovery_task(struct work_struct *work)
4353 {
4354         struct be_adapter *adapter =
4355                 container_of(work, struct be_adapter,  func_recovery_work.work);
4356         int status = 0;
4357
4358         be_detect_error(adapter);
4359
4360         if (adapter->hw_error && lancer_chip(adapter)) {
4361
4362                 rtnl_lock();
4363                 netif_device_detach(adapter->netdev);
4364                 rtnl_unlock();
4365
4366                 status = lancer_recover_func(adapter);
4367                 if (!status)
4368                         netif_device_attach(adapter->netdev);
4369         }
4370
4371         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4372          * no need to attempt further recovery.
4373          */
4374         if (!status || status == -EAGAIN)
4375                 schedule_delayed_work(&adapter->func_recovery_work,
4376                                       msecs_to_jiffies(1000));
4377 }
4378
4379 static void be_worker(struct work_struct *work)
4380 {
4381         struct be_adapter *adapter =
4382                 container_of(work, struct be_adapter, work.work);
4383         struct be_rx_obj *rxo;
4384         int i;
4385
4386         /* when interrupts are not yet enabled, just reap any pending
4387         * mcc completions */
4388         if (!netif_running(adapter->netdev)) {
4389                 local_bh_disable();
4390                 be_process_mcc(adapter);
4391                 local_bh_enable();
4392                 goto reschedule;
4393         }
4394
4395         if (!adapter->stats_cmd_sent) {
4396                 if (lancer_chip(adapter))
4397                         lancer_cmd_get_pport_stats(adapter,
4398                                                 &adapter->stats_cmd);
4399                 else
4400                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4401         }
4402
4403         if (be_physfn(adapter) &&
4404             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4405                 be_cmd_get_die_temperature(adapter);
4406
4407         for_all_rx_queues(adapter, rxo, i) {
4408                 /* Replenish RX-queues starved due to memory
4409                  * allocation failures.
4410                  */
4411                 if (rxo->rx_post_starved)
4412                         be_post_rx_frags(rxo, GFP_KERNEL);
4413         }
4414
4415         be_eqd_update(adapter);
4416
4417 reschedule:
4418         adapter->work_counter++;
4419         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4420 }
4421
4422 /* If any VFs are already enabled don't FLR the PF */
4423 static bool be_reset_required(struct be_adapter *adapter)
4424 {
4425         return pci_num_vf(adapter->pdev) ? false : true;
4426 }
4427
4428 static char *mc_name(struct be_adapter *adapter)
4429 {
4430         if (adapter->function_mode & FLEX10_MODE)
4431                 return "FLEX10";
4432         else if (adapter->function_mode & VNIC_MODE)
4433                 return "vNIC";
4434         else if (adapter->function_mode & UMC_ENABLED)
4435                 return "UMC";
4436         else
4437                 return "";
4438 }
4439
4440 static inline char *func_name(struct be_adapter *adapter)
4441 {
4442         return be_physfn(adapter) ? "PF" : "VF";
4443 }
4444
4445 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4446 {
4447         int status = 0;
4448         struct be_adapter *adapter;
4449         struct net_device *netdev;
4450         char port_name;
4451
4452         status = pci_enable_device(pdev);
4453         if (status)
4454                 goto do_none;
4455
4456         status = pci_request_regions(pdev, DRV_NAME);
4457         if (status)
4458                 goto disable_dev;
4459         pci_set_master(pdev);
4460
4461         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4462         if (netdev == NULL) {
4463                 status = -ENOMEM;
4464                 goto rel_reg;
4465         }
4466         adapter = netdev_priv(netdev);
4467         adapter->pdev = pdev;
4468         pci_set_drvdata(pdev, adapter);
4469         adapter->netdev = netdev;
4470         SET_NETDEV_DEV(netdev, &pdev->dev);
4471
4472         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4473         if (!status) {
4474                 netdev->features |= NETIF_F_HIGHDMA;
4475         } else {
4476                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4477                 if (status) {
4478                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4479                         goto free_netdev;
4480                 }
4481         }
4482
4483         if (be_physfn(adapter)) {
4484                 status = pci_enable_pcie_error_reporting(pdev);
4485                 if (!status)
4486                         dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4487         }
4488
4489         status = be_ctrl_init(adapter);
4490         if (status)
4491                 goto free_netdev;
4492
4493         /* sync up with fw's ready state */
4494         if (be_physfn(adapter)) {
4495                 status = be_fw_wait_ready(adapter);
4496                 if (status)
4497                         goto ctrl_clean;
4498         }
4499
4500         if (be_reset_required(adapter)) {
4501                 status = be_cmd_reset_function(adapter);
4502                 if (status)
4503                         goto ctrl_clean;
4504
4505                 /* Wait for interrupts to quiesce after an FLR */
4506                 msleep(100);
4507         }
4508
4509         /* Allow interrupts for other ULPs running on NIC function */
4510         be_intr_set(adapter, true);
4511
4512         /* tell fw we're ready to fire cmds */
4513         status = be_cmd_fw_init(adapter);
4514         if (status)
4515                 goto ctrl_clean;
4516
4517         status = be_stats_init(adapter);
4518         if (status)
4519                 goto ctrl_clean;
4520
4521         status = be_get_initial_config(adapter);
4522         if (status)
4523                 goto stats_clean;
4524
4525         INIT_DELAYED_WORK(&adapter->work, be_worker);
4526         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4527         adapter->rx_fc = adapter->tx_fc = true;
4528
4529         status = be_setup(adapter);
4530         if (status)
4531                 goto stats_clean;
4532
4533         be_netdev_init(netdev);
4534         status = register_netdev(netdev);
4535         if (status != 0)
4536                 goto unsetup;
4537
4538         be_roce_dev_add(adapter);
4539
4540         schedule_delayed_work(&adapter->func_recovery_work,
4541                               msecs_to_jiffies(1000));
4542
4543         be_cmd_query_port_name(adapter, &port_name);
4544
4545         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4546                  func_name(adapter), mc_name(adapter), port_name);
4547
4548         return 0;
4549
4550 unsetup:
4551         be_clear(adapter);
4552 stats_clean:
4553         be_stats_cleanup(adapter);
4554 ctrl_clean:
4555         be_ctrl_cleanup(adapter);
4556 free_netdev:
4557         free_netdev(netdev);
4558 rel_reg:
4559         pci_release_regions(pdev);
4560 disable_dev:
4561         pci_disable_device(pdev);
4562 do_none:
4563         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4564         return status;
4565 }
4566
4567 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4568 {
4569         struct be_adapter *adapter = pci_get_drvdata(pdev);
4570         struct net_device *netdev =  adapter->netdev;
4571
4572         if (adapter->wol_en)
4573                 be_setup_wol(adapter, true);
4574
4575         be_intr_set(adapter, false);
4576         cancel_delayed_work_sync(&adapter->func_recovery_work);
4577
4578         netif_device_detach(netdev);
4579         if (netif_running(netdev)) {
4580                 rtnl_lock();
4581                 be_close(netdev);
4582                 rtnl_unlock();
4583         }
4584         be_clear(adapter);
4585
4586         pci_save_state(pdev);
4587         pci_disable_device(pdev);
4588         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4589         return 0;
4590 }
4591
4592 static int be_resume(struct pci_dev *pdev)
4593 {
4594         int status = 0;
4595         struct be_adapter *adapter = pci_get_drvdata(pdev);
4596         struct net_device *netdev =  adapter->netdev;
4597
4598         netif_device_detach(netdev);
4599
4600         status = pci_enable_device(pdev);
4601         if (status)
4602                 return status;
4603
4604         pci_set_power_state(pdev, PCI_D0);
4605         pci_restore_state(pdev);
4606
4607         status = be_fw_wait_ready(adapter);
4608         if (status)
4609                 return status;
4610
4611         be_intr_set(adapter, true);
4612         /* tell fw we're ready to fire cmds */
4613         status = be_cmd_fw_init(adapter);
4614         if (status)
4615                 return status;
4616
4617         be_setup(adapter);
4618         if (netif_running(netdev)) {
4619                 rtnl_lock();
4620                 be_open(netdev);
4621                 rtnl_unlock();
4622         }
4623
4624         schedule_delayed_work(&adapter->func_recovery_work,
4625                               msecs_to_jiffies(1000));
4626         netif_device_attach(netdev);
4627
4628         if (adapter->wol_en)
4629                 be_setup_wol(adapter, false);
4630
4631         return 0;
4632 }
4633
4634 /*
4635  * An FLR will stop BE from DMAing any data.
4636  */
4637 static void be_shutdown(struct pci_dev *pdev)
4638 {
4639         struct be_adapter *adapter = pci_get_drvdata(pdev);
4640
4641         if (!adapter)
4642                 return;
4643
4644         cancel_delayed_work_sync(&adapter->work);
4645         cancel_delayed_work_sync(&adapter->func_recovery_work);
4646
4647         netif_device_detach(adapter->netdev);
4648
4649         be_cmd_reset_function(adapter);
4650
4651         pci_disable_device(pdev);
4652 }
4653
4654 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
4655                                 pci_channel_state_t state)
4656 {
4657         struct be_adapter *adapter = pci_get_drvdata(pdev);
4658         struct net_device *netdev =  adapter->netdev;
4659
4660         dev_err(&adapter->pdev->dev, "EEH error detected\n");
4661
4662         if (!adapter->eeh_error) {
4663                 adapter->eeh_error = true;
4664
4665                 cancel_delayed_work_sync(&adapter->func_recovery_work);
4666
4667                 rtnl_lock();
4668                 netif_device_detach(netdev);
4669                 if (netif_running(netdev))
4670                         be_close(netdev);
4671                 rtnl_unlock();
4672
4673                 be_clear(adapter);
4674         }
4675
4676         if (state == pci_channel_io_perm_failure)
4677                 return PCI_ERS_RESULT_DISCONNECT;
4678
4679         pci_disable_device(pdev);
4680
4681         /* The error could cause the FW to trigger a flash debug dump.
4682          * Resetting the card while flash dump is in progress
4683          * can cause it not to recover; wait for it to finish.
4684          * Wait only for first function as it is needed only once per
4685          * adapter.
4686          */
4687         if (pdev->devfn == 0)
4688                 ssleep(30);
4689
4690         return PCI_ERS_RESULT_NEED_RESET;
4691 }
4692
4693 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
4694 {
4695         struct be_adapter *adapter = pci_get_drvdata(pdev);
4696         int status;
4697
4698         dev_info(&adapter->pdev->dev, "EEH reset\n");
4699
4700         status = pci_enable_device(pdev);
4701         if (status)
4702                 return PCI_ERS_RESULT_DISCONNECT;
4703
4704         pci_set_master(pdev);
4705         pci_set_power_state(pdev, PCI_D0);
4706         pci_restore_state(pdev);
4707
4708         /* Check if card is ok and fw is ready */
4709         dev_info(&adapter->pdev->dev,
4710                  "Waiting for FW to be ready after EEH reset\n");
4711         status = be_fw_wait_ready(adapter);
4712         if (status)
4713                 return PCI_ERS_RESULT_DISCONNECT;
4714
4715         pci_cleanup_aer_uncorrect_error_status(pdev);
4716         be_clear_all_error(adapter);
4717         return PCI_ERS_RESULT_RECOVERED;
4718 }
4719
4720 static void be_eeh_resume(struct pci_dev *pdev)
4721 {
4722         int status = 0;
4723         struct be_adapter *adapter = pci_get_drvdata(pdev);
4724         struct net_device *netdev =  adapter->netdev;
4725
4726         dev_info(&adapter->pdev->dev, "EEH resume\n");
4727
4728         pci_save_state(pdev);
4729
4730         status = be_cmd_reset_function(adapter);
4731         if (status)
4732                 goto err;
4733
4734         /* tell fw we're ready to fire cmds */
4735         status = be_cmd_fw_init(adapter);
4736         if (status)
4737                 goto err;
4738
4739         status = be_setup(adapter);
4740         if (status)
4741                 goto err;
4742
4743         if (netif_running(netdev)) {
4744                 status = be_open(netdev);
4745                 if (status)
4746                         goto err;
4747         }
4748
4749         schedule_delayed_work(&adapter->func_recovery_work,
4750                               msecs_to_jiffies(1000));
4751         netif_device_attach(netdev);
4752         return;
4753 err:
4754         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
4755 }
4756
4757 static const struct pci_error_handlers be_eeh_handlers = {
4758         .error_detected = be_eeh_err_detected,
4759         .slot_reset = be_eeh_reset,
4760         .resume = be_eeh_resume,
4761 };
4762
4763 static struct pci_driver be_driver = {
4764         .name = DRV_NAME,
4765         .id_table = be_dev_ids,
4766         .probe = be_probe,
4767         .remove = be_remove,
4768         .suspend = be_suspend,
4769         .resume = be_resume,
4770         .shutdown = be_shutdown,
4771         .err_handler = &be_eeh_handlers
4772 };
4773
4774 static int __init be_init_module(void)
4775 {
4776         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
4777             rx_frag_size != 2048) {
4778                 printk(KERN_WARNING DRV_NAME
4779                         " : Module param rx_frag_size must be 2048/4096/8192."
4780                         " Using 2048\n");
4781                 rx_frag_size = 2048;
4782         }
4783
4784         return pci_register_driver(&be_driver);
4785 }
4786 module_init(be_init_module);
4787
4788 static void __exit be_exit_module(void)
4789 {
4790         pci_unregister_driver(&be_driver);
4791 }
4792 module_exit(be_exit_module);