drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 static unsigned int num_vfs;
  34 module_param(num_vfs, uint, S_IRUGO);
  35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  36
  37 static ushort rx_frag_size = 2048;
  38 module_param(rx_frag_size, ushort, S_IRUGO);
  39 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  40
  41 static const struct pci_device_id be_dev_ids[] = {
  42         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  44         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  46         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  50         { 0 }
  51 };
  52 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  53 /* UE Status Low CSR */
  54 static const char * const ue_status_low_desc[] = {
  55         "CEV",
  56         "CTX",
  57         "DBUF",
  58         "ERX",
  59         "Host",
  60         "MPU",
  61         "NDMA",
  62         "PTC ",
  63         "RDMA ",
  64         "RXF ",
  65         "RXIPS ",
  66         "RXULP0 ",
  67         "RXULP1 ",
  68         "RXULP2 ",
  69         "TIM ",
  70         "TPOST ",
  71         "TPRE ",
  72         "TXIPS ",
  73         "TXULP0 ",
  74         "TXULP1 ",
  75         "UC ",
  76         "WDMA ",
  77         "TXULP2 ",
  78         "HOST1 ",
  79         "P0_OB_LINK ",
  80         "P1_OB_LINK ",
  81         "HOST_GPIO ",
  82         "MBOX ",
  83         "ERX2 ",
  84         "SPARE ",
  85         "JTAG ",
  86         "MPU_INTPEND "
  87 };
  88
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 126 {
 127         struct be_dma_mem *mem = &q->dma_mem;
 128
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190
 191         val |= qid & DB_RQ_RING_ID_MASK;
 192         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 193
 194         wmb();
 195         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 196 }
 197
 198 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 199                           u16 posted)
 200 {
 201         u32 val = 0;
 202
 203         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 204         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 205
 206         wmb();
 207         iowrite32(val, adapter->db + txo->db_offset);
 208 }
 209
 210 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 211                          bool arm, bool clear_int, u16 num_popped)
 212 {
 213         u32 val = 0;
 214
 215         val |= qid & DB_EQ_RING_ID_MASK;
 216         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 217
 218         if (adapter->eeh_error)
 219                 return;
 220
 221         if (arm)
 222                 val |= 1 << DB_EQ_REARM_SHIFT;
 223         if (clear_int)
 224                 val |= 1 << DB_EQ_CLR_SHIFT;
 225         val |= 1 << DB_EQ_EVNT_SHIFT;
 226         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 227         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 228 }
 229
 230 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 231 {
 232         u32 val = 0;
 233
 234         val |= qid & DB_CQ_RING_ID_MASK;
 235         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 236                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 237
 238         if (adapter->eeh_error)
 239                 return;
 240
 241         if (arm)
 242                 val |= 1 << DB_CQ_REARM_SHIFT;
 243         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 244         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 245 }
 246
 247 static int be_mac_addr_set(struct net_device *netdev, void *p)
 248 {
 249         struct be_adapter *adapter = netdev_priv(netdev);
 250         struct device *dev = &adapter->pdev->dev;
 251         struct sockaddr *addr = p;
 252         int status;
 253         u8 mac[ETH_ALEN];
 254         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 255
 256         if (!is_valid_ether_addr(addr->sa_data))
 257                 return -EADDRNOTAVAIL;
 258
 259         /* Proceed further only if, User provided MAC is different
 260          * from active MAC
 261          */
 262         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 263                 return 0;
 264
 265         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 266          * privilege or if PF did not provision the new MAC address.
 267          * On BE3, this cmd will always fail if the VF doesn't have the
 268          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 269          * the MAC for the VF.
 270          */
 271         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 272                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 273         if (!status) {
 274                 curr_pmac_id = adapter->pmac_id[0];
 275
 276                 /* Delete the old programmed MAC. This call may fail if the
 277                  * old MAC was already deleted by the PF driver.
 278                  */
 279                 if (adapter->pmac_id[0] != old_pmac_id)
 280                         be_cmd_pmac_del(adapter, adapter->if_handle,
 281                                         old_pmac_id, 0);
 282         }
 283
 284         /* Decide if the new MAC is successfully activated only after
 285          * querying the FW
 286          */
 287         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 288                                        adapter->if_handle, true, 0);
 289         if (status)
 290                 goto err;
 291
 292         /* The MAC change did not happen, either due to lack of privilege
 293          * or PF didn't pre-provision.
 294          */
 295         if (!ether_addr_equal(addr->sa_data, mac)) {
 296                 status = -EPERM;
 297                 goto err;
 298         }
 299
 300         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 301         dev_info(dev, "MAC address changed to %pM\n", mac);
 302         return 0;
 303 err:
 304         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 305         return status;
 306 }
 307
 308 /* BE2 supports only v0 cmd */
 309 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 310 {
 311         if (BE2_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else if (BE3_chip(adapter)) {
 316                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         } else {
 320                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 321
 322                 return &cmd->hw_stats;
 323         }
 324 }
 325
 326 /* BE2 supports only v0 cmd */
 327 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 328 {
 329         if (BE2_chip(adapter)) {
 330                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else if (BE3_chip(adapter)) {
 334                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         } else {
 338                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 339
 340                 return &hw_stats->erx;
 341         }
 342 }
 343
 344 static void populate_be_v0_stats(struct be_adapter *adapter)
 345 {
 346         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 347         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 348         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 349         struct be_port_rxf_stats_v0 *port_stats =
 350                                         &rxf_stats->port[adapter->port_num];
 351         struct be_drv_stats *drvs = &adapter->drv_stats;
 352
 353         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 354         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 355         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 356         drvs->rx_control_frames = port_stats->rx_control_frames;
 357         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 358         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 359         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 360         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 361         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 362         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 363         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 364         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 365         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 366         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 367         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 368         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 369         drvs->rx_dropped_header_too_small =
 370                 port_stats->rx_dropped_header_too_small;
 371         drvs->rx_address_filtered =
 372                                         port_stats->rx_address_filtered +
 373                                         port_stats->rx_vlan_filtered;
 374         drvs->rx_alignment_symbol_errors =
 375                 port_stats->rx_alignment_symbol_errors;
 376
 377         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 378         drvs->tx_controlframes = port_stats->tx_controlframes;
 379
 380         if (adapter->port_num)
 381                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 382         else
 383                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 384         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 385         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 386         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 387         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 388         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 389         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 390         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 391 }
 392
 393 static void populate_be_v1_stats(struct be_adapter *adapter)
 394 {
 395         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 396         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 397         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 398         struct be_port_rxf_stats_v1 *port_stats =
 399                                         &rxf_stats->port[adapter->port_num];
 400         struct be_drv_stats *drvs = &adapter->drv_stats;
 401
 402         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 403         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 404         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 405         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 406         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 407         drvs->rx_control_frames = port_stats->rx_control_frames;
 408         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 409         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 410         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 411         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 412         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 413         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 414         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 415         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 416         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 417         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 418         drvs->rx_dropped_header_too_small =
 419                 port_stats->rx_dropped_header_too_small;
 420         drvs->rx_input_fifo_overflow_drop =
 421                 port_stats->rx_input_fifo_overflow_drop;
 422         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 423         drvs->rx_alignment_symbol_errors =
 424                 port_stats->rx_alignment_symbol_errors;
 425         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 426         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 427         drvs->tx_controlframes = port_stats->tx_controlframes;
 428         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 429         drvs->jabber_events = port_stats->jabber_events;
 430         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 431         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 432         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 433         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 434         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 435         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 436         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 437 }
 438
 439 static void populate_be_v2_stats(struct be_adapter *adapter)
 440 {
 441         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 442         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 443         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 444         struct be_port_rxf_stats_v2 *port_stats =
 445                                         &rxf_stats->port[adapter->port_num];
 446         struct be_drv_stats *drvs = &adapter->drv_stats;
 447
 448         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 449         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 450         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 451         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 452         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 453         drvs->rx_control_frames = port_stats->rx_control_frames;
 454         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 455         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 456         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 457         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 458         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 459         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 460         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 461         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 462         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 463         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 464         drvs->rx_dropped_header_too_small =
 465                 port_stats->rx_dropped_header_too_small;
 466         drvs->rx_input_fifo_overflow_drop =
 467                 port_stats->rx_input_fifo_overflow_drop;
 468         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 469         drvs->rx_alignment_symbol_errors =
 470                 port_stats->rx_alignment_symbol_errors;
 471         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 472         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 473         drvs->tx_controlframes = port_stats->tx_controlframes;
 474         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 475         drvs->jabber_events = port_stats->jabber_events;
 476         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 477         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 478         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 479         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 480         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 481         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 482         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 483         if (be_roce_supported(adapter)) {
 484                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 485                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 486                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 487                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 488                 drvs->roce_drops_payload_len =
 489                         port_stats->roce_drops_payload_len;
 490         }
 491 }
 492
 493 static void populate_lancer_stats(struct be_adapter *adapter)
 494 {
 495         struct be_drv_stats *drvs = &adapter->drv_stats;
 496         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 497
 498         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 499         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 500         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 501         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 502         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 503         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 504         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 505         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 506         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 507         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 508         drvs->rx_dropped_tcp_length =
 509                                 pport_stats->rx_dropped_invalid_tcp_length;
 510         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 511         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 512         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 513         drvs->rx_dropped_header_too_small =
 514                                 pport_stats->rx_dropped_header_too_small;
 515         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 516         drvs->rx_address_filtered =
 517                                         pport_stats->rx_address_filtered +
 518                                         pport_stats->rx_vlan_filtered;
 519         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 520         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 521         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 522         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 523         drvs->jabber_events = pport_stats->rx_jabbers;
 524         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 525         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 526         drvs->rx_drops_too_many_frags =
 527                                 pport_stats->rx_drops_too_many_frags_lo;
 528 }
 529
 530 static void accumulate_16bit_val(u32 *acc, u16 val)
 531 {
 532 #define lo(x)                   (x & 0xFFFF)
 533 #define hi(x)                   (x & 0xFFFF0000)
 534         bool wrapped = val < lo(*acc);
 535         u32 newacc = hi(*acc) + val;
 536
 537         if (wrapped)
 538                 newacc += 65536;
 539         ACCESS_ONCE(*acc) = newacc;
 540 }
 541
 542 static void populate_erx_stats(struct be_adapter *adapter,
 543                                struct be_rx_obj *rxo, u32 erx_stat)
 544 {
 545         if (!BEx_chip(adapter))
 546                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 547         else
 548                 /* below erx HW counter can actually wrap around after
 549                  * 65535. Driver accumulates a 32-bit value
 550                  */
 551                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 552                                      (u16)erx_stat);
 553 }
 554
 555 void be_parse_stats(struct be_adapter *adapter)
 556 {
 557         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 558         struct be_rx_obj *rxo;
 559         int i;
 560         u32 erx_stat;
 561
 562         if (lancer_chip(adapter)) {
 563                 populate_lancer_stats(adapter);
 564         } else {
 565                 if (BE2_chip(adapter))
 566                         populate_be_v0_stats(adapter);
 567                 else if (BE3_chip(adapter))
 568                         /* for BE3 */
 569                         populate_be_v1_stats(adapter);
 570                 else
 571                         populate_be_v2_stats(adapter);
 572
 573                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 574                 for_all_rx_queues(adapter, rxo, i) {
 575                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 576                         populate_erx_stats(adapter, rxo, erx_stat);
 577                 }
 578         }
 579 }
 580
 581 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 582                                                 struct rtnl_link_stats64 *stats)
 583 {
 584         struct be_adapter *adapter = netdev_priv(netdev);
 585         struct be_drv_stats *drvs = &adapter->drv_stats;
 586         struct be_rx_obj *rxo;
 587         struct be_tx_obj *txo;
 588         u64 pkts, bytes;
 589         unsigned int start;
 590         int i;
 591
 592         for_all_rx_queues(adapter, rxo, i) {
 593                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 594
 595                 do {
 596                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 597                         pkts = rx_stats(rxo)->rx_pkts;
 598                         bytes = rx_stats(rxo)->rx_bytes;
 599                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 600                 stats->rx_packets += pkts;
 601                 stats->rx_bytes += bytes;
 602                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 603                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 604                                         rx_stats(rxo)->rx_drops_no_frags;
 605         }
 606
 607         for_all_tx_queues(adapter, txo, i) {
 608                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 609
 610                 do {
 611                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 612                         pkts = tx_stats(txo)->tx_pkts;
 613                         bytes = tx_stats(txo)->tx_bytes;
 614                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 615                 stats->tx_packets += pkts;
 616                 stats->tx_bytes += bytes;
 617         }
 618
 619         /* bad pkts received */
 620         stats->rx_errors = drvs->rx_crc_errors +
 621                 drvs->rx_alignment_symbol_errors +
 622                 drvs->rx_in_range_errors +
 623                 drvs->rx_out_range_errors +
 624                 drvs->rx_frame_too_long +
 625                 drvs->rx_dropped_too_small +
 626                 drvs->rx_dropped_too_short +
 627                 drvs->rx_dropped_header_too_small +
 628                 drvs->rx_dropped_tcp_length +
 629                 drvs->rx_dropped_runt;
 630
 631         /* detailed rx errors */
 632         stats->rx_length_errors = drvs->rx_in_range_errors +
 633                 drvs->rx_out_range_errors +
 634                 drvs->rx_frame_too_long;
 635
 636         stats->rx_crc_errors = drvs->rx_crc_errors;
 637
 638         /* frame alignment errors */
 639         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 640
 641         /* receiver fifo overrun */
 642         /* drops_no_pbuf is no per i/f, it's per BE card */
 643         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 644                                 drvs->rx_input_fifo_overflow_drop +
 645                                 drvs->rx_drops_no_pbuf;
 646         return stats;
 647 }
 648
 649 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 650 {
 651         struct net_device *netdev = adapter->netdev;
 652
 653         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 654                 netif_carrier_off(netdev);
 655                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 656         }
 657
 658         if (link_status)
 659                 netif_carrier_on(netdev);
 660         else
 661                 netif_carrier_off(netdev);
 662 }
 663
 664 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 665 {
 666         struct be_tx_stats *stats = tx_stats(txo);
 667
 668         u64_stats_update_begin(&stats->sync);
 669         stats->tx_reqs++;
 670         stats->tx_bytes += skb->len;
 671         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Returns number of WRBs needed for the skb */
 676 static u32 skb_wrb_cnt(struct sk_buff *skb)
 677 {
 678         /* +1 for the header wrb */
 679         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 680 }
 681
 682 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 683 {
 684         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 685         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 686         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 687         wrb->rsvd0 = 0;
 688 }
 689
 690 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 691  * to avoid the swap and shift/mask operations in wrb_fill().
 692  */
 693 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 694 {
 695         wrb->frag_pa_hi = 0;
 696         wrb->frag_pa_lo = 0;
 697         wrb->frag_len = 0;
 698         wrb->rsvd0 = 0;
 699 }
 700
 701 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 702                                      struct sk_buff *skb)
 703 {
 704         u8 vlan_prio;
 705         u16 vlan_tag;
 706
 707         vlan_tag = skb_vlan_tag_get(skb);
 708         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 709         /* If vlan priority provided by OS is NOT in available bmap */
 710         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 711                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 712                                 adapter->recommended_prio;
 713
 714         return vlan_tag;
 715 }
 716
 717 /* Used only for IP tunnel packets */
 718 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 719 {
 720         return (inner_ip_hdr(skb)->version == 4) ?
 721                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 722 }
 723
 724 static u16 skb_ip_proto(struct sk_buff *skb)
 725 {
 726         return (ip_hdr(skb)->version == 4) ?
 727                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 728 }
 729
 730 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 731                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 732                          bool skip_hw_vlan)
 733 {
 734         u16 vlan_tag, proto;
 735
 736         memset(hdr, 0, sizeof(*hdr));
 737
 738         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 739
 740         if (skb_is_gso(skb)) {
 741                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 742                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 743                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 744                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 745         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 746                 if (skb->encapsulation) {
 747                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 748                         proto = skb_inner_ip_proto(skb);
 749                 } else {
 750                         proto = skb_ip_proto(skb);
 751                 }
 752                 if (proto == IPPROTO_TCP)
 753                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 754                 else if (proto == IPPROTO_UDP)
 755                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 756         }
 757
 758         if (skb_vlan_tag_present(skb)) {
 759                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 760                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 761                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 762         }
 763
 764         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 765         SET_TX_WRB_HDR_BITS(len, hdr, len);
 766
 767         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0
 768          * When this hack is not needed, the evt bit is set while ringing DB
 769          */
 770         if (skip_hw_vlan)
 771                 SET_TX_WRB_HDR_BITS(event, hdr, 1);
 772 }
 773
 774 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 775                           bool unmap_single)
 776 {
 777         dma_addr_t dma;
 778         u32 frag_len = le32_to_cpu(wrb->frag_len);
 779
 780
 781         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 782                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 783         if (frag_len) {
 784                 if (unmap_single)
 785                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 786                 else
 787                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 788         }
 789 }
 790
 791 /* Returns the number of WRBs used up by the skb */
 792 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 793                            struct sk_buff *skb, bool skip_hw_vlan)
 794 {
 795         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 796         struct device *dev = &adapter->pdev->dev;
 797         struct be_queue_info *txq = &txo->q;
 798         struct be_eth_hdr_wrb *hdr;
 799         bool map_single = false;
 800         struct be_eth_wrb *wrb;
 801         dma_addr_t busaddr;
 802         u16 head = txq->head;
 803
 804         hdr = queue_head_node(txq);
 805         wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan);
 806         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 807
 808         queue_head_inc(txq);
 809
 810         if (skb->len > skb->data_len) {
 811                 int len = skb_headlen(skb);
 812
 813                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 814                 if (dma_mapping_error(dev, busaddr))
 815                         goto dma_err;
 816                 map_single = true;
 817                 wrb = queue_head_node(txq);
 818                 wrb_fill(wrb, busaddr, len);
 819                 queue_head_inc(txq);
 820                 copied += len;
 821         }
 822
 823         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 824                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 825
 826                 busaddr = skb_frag_dma_map(dev, frag, 0,
 827                                            skb_frag_size(frag), DMA_TO_DEVICE);
 828                 if (dma_mapping_error(dev, busaddr))
 829                         goto dma_err;
 830                 wrb = queue_head_node(txq);
 831                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 832                 queue_head_inc(txq);
 833                 copied += skb_frag_size(frag);
 834         }
 835
 836         BUG_ON(txo->sent_skb_list[head]);
 837         txo->sent_skb_list[head] = skb;
 838         txo->last_req_hdr = head;
 839         atomic_add(wrb_cnt, &txq->used);
 840         txo->last_req_wrb_cnt = wrb_cnt;
 841         txo->pend_wrb_cnt += wrb_cnt;
 842
 843         be_tx_stats_update(txo, skb);
 844         return wrb_cnt;
 845
 846 dma_err:
 847         /* Bring the queue back to the state it was in before this
 848          * routine was invoked.
 849          */
 850         txq->head = head;
 851         /* skip the first wrb (hdr); it's not mapped */
 852         queue_head_inc(txq);
 853         while (copied) {
 854                 wrb = queue_head_node(txq);
 855                 unmap_tx_frag(dev, wrb, map_single);
 856                 map_single = false;
 857                 copied -= le32_to_cpu(wrb->frag_len);
 858                 adapter->drv_stats.dma_map_errors++;
 859                 queue_head_inc(txq);
 860         }
 861         txq->head = head;
 862         return 0;
 863 }
 864
 865 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 866 {
 867         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 868 }
 869
 870 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 871                                              struct sk_buff *skb,
 872                                              bool *skip_hw_vlan)
 873 {
 874         u16 vlan_tag = 0;
 875
 876         skb = skb_share_check(skb, GFP_ATOMIC);
 877         if (unlikely(!skb))
 878                 return skb;
 879
 880         if (skb_vlan_tag_present(skb))
 881                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 882
 883         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 884                 if (!vlan_tag)
 885                         vlan_tag = adapter->pvid;
 886                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 887                  * skip VLAN insertion
 888                  */
 889                 if (skip_hw_vlan)
 890                         *skip_hw_vlan = true;
 891         }
 892
 893         if (vlan_tag) {
 894                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 895                                                 vlan_tag);
 896                 if (unlikely(!skb))
 897                         return skb;
 898                 skb->vlan_tci = 0;
 899         }
 900
 901         /* Insert the outer VLAN, if any */
 902         if (adapter->qnq_vid) {
 903                 vlan_tag = adapter->qnq_vid;
 904                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 905                                                 vlan_tag);
 906                 if (unlikely(!skb))
 907                         return skb;
 908                 if (skip_hw_vlan)
 909                         *skip_hw_vlan = true;
 910         }
 911
 912         return skb;
 913 }
 914
 915 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 916 {
 917         struct ethhdr *eh = (struct ethhdr *)skb->data;
 918         u16 offset = ETH_HLEN;
 919
 920         if (eh->h_proto == htons(ETH_P_IPV6)) {
 921                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 922
 923                 offset += sizeof(struct ipv6hdr);
 924                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 925                     ip6h->nexthdr != NEXTHDR_UDP) {
 926                         struct ipv6_opt_hdr *ehdr =
 927                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 928
 929                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 930                         if (ehdr->hdrlen == 0xff)
 931                                 return true;
 932                 }
 933         }
 934         return false;
 935 }
 936
 937 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 938 {
 939         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 940 }
 941
 942 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 943 {
 944         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 945 }
 946
 947 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 948                                                   struct sk_buff *skb,
 949                                                   bool *skip_hw_vlan)
 950 {
 951         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 952         unsigned int eth_hdr_len;
 953         struct iphdr *ip;
 954
 955         /* For padded packets, BE HW modifies tot_len field in IP header
 956          * incorrecly when VLAN tag is inserted by HW.
 957          * For padded packets, Lancer computes incorrect checksum.
 958          */
 959         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 960                                                 VLAN_ETH_HLEN : ETH_HLEN;
 961         if (skb->len <= 60 &&
 962             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
 963             is_ipv4_pkt(skb)) {
 964                 ip = (struct iphdr *)ip_hdr(skb);
 965                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 966         }
 967
 968         /* If vlan tag is already inlined in the packet, skip HW VLAN
 969          * tagging in pvid-tagging mode
 970          */
 971         if (be_pvid_tagging_enabled(adapter) &&
 972             veh->h_vlan_proto == htons(ETH_P_8021Q))
 973                 *skip_hw_vlan = true;
 974
 975         /* HW has a bug wherein it will calculate CSUM for VLAN
 976          * pkts even though it is disabled.
 977          * Manually insert VLAN in pkt.
 978          */
 979         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 980             skb_vlan_tag_present(skb)) {
 981                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 982                 if (unlikely(!skb))
 983                         goto err;
 984         }
 985
 986         /* HW may lockup when VLAN HW tagging is requested on
 987          * certain ipv6 packets. Drop such pkts if the HW workaround to
 988          * skip HW tagging is not enabled by FW.
 989          */
 990         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 991                      (adapter->pvid || adapter->qnq_vid) &&
 992                      !qnq_async_evt_rcvd(adapter)))
 993                 goto tx_drop;
 994
 995         /* Manual VLAN tag insertion to prevent:
 996          * ASIC lockup when the ASIC inserts VLAN tag into
 997          * certain ipv6 packets. Insert VLAN tags in driver,
 998          * and set event, completion, vlan bits accordingly
 999          * in the Tx WRB.
1000          */
1001         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1002             be_vlan_tag_tx_chk(adapter, skb)) {
1003                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1004                 if (unlikely(!skb))
1005                         goto err;
1006         }
1007
1008         return skb;
1009 tx_drop:
1010         dev_kfree_skb_any(skb);
1011 err:
1012         return NULL;
1013 }
1014
1015 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1016                                            struct sk_buff *skb,
1017                                            bool *skip_hw_vlan)
1018 {
1019         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1020          * less may cause a transmit stall on that port. So the work-around is
1021          * to pad short packets (<= 32 bytes) to a 36-byte length.
1022          */
1023         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1024                 if (skb_put_padto(skb, 36))
1025                         return NULL;
1026         }
1027
1028         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1029                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1030                 if (!skb)
1031                         return NULL;
1032         }
1033
1034         return skb;
1035 }
1036
1037 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1038 {
1039         struct be_queue_info *txq = &txo->q;
1040         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1041
1042         /* Mark the last request eventable if it hasn't been marked already */
1043         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1044                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1045
1046         /* compose a dummy wrb if there are odd set of wrbs to notify */
1047         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1048                 wrb_fill_dummy(queue_head_node(txq));
1049                 queue_head_inc(txq);
1050                 atomic_inc(&txq->used);
1051                 txo->pend_wrb_cnt++;
1052                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1053                                            TX_HDR_WRB_NUM_SHIFT);
1054                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1055                                           TX_HDR_WRB_NUM_SHIFT);
1056         }
1057         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1058         txo->pend_wrb_cnt = 0;
1059 }
1060
1061 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1062 {
1063         bool skip_hw_vlan = false, flush = !skb->xmit_more;
1064         struct be_adapter *adapter = netdev_priv(netdev);
1065         u16 q_idx = skb_get_queue_mapping(skb);
1066         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1067         struct be_queue_info *txq = &txo->q;
1068         u16 wrb_cnt;
1069
1070         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1071         if (unlikely(!skb))
1072                 goto drop;
1073
1074         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan);
1075         if (unlikely(!wrb_cnt)) {
1076                 dev_kfree_skb_any(skb);
1077                 goto drop;
1078         }
1079
1080         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1081                 netif_stop_subqueue(netdev, q_idx);
1082                 tx_stats(txo)->tx_stops++;
1083         }
1084
1085         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1086                 be_xmit_flush(adapter, txo);
1087
1088         return NETDEV_TX_OK;
1089 drop:
1090         tx_stats(txo)->tx_drv_drops++;
1091         /* Flush the already enqueued tx requests */
1092         if (flush && txo->pend_wrb_cnt)
1093                 be_xmit_flush(adapter, txo);
1094
1095         return NETDEV_TX_OK;
1096 }
1097
1098 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1099 {
1100         struct be_adapter *adapter = netdev_priv(netdev);
1101         struct device *dev = &adapter->pdev->dev;
1102
1103         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1104                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1105                          BE_MIN_MTU, BE_MAX_MTU);
1106                 return -EINVAL;
1107         }
1108
1109         dev_info(dev, "MTU changed from %d to %d bytes\n",
1110                  netdev->mtu, new_mtu);
1111         netdev->mtu = new_mtu;
1112         return 0;
1113 }
1114
1115 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1116 {
1117         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1118                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1119 }
1120
1121 static int be_set_vlan_promisc(struct be_adapter *adapter)
1122 {
1123         struct device *dev = &adapter->pdev->dev;
1124         int status;
1125
1126         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1127                 return 0;
1128
1129         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1130         if (!status) {
1131                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1132                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1133         } else {
1134                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1135         }
1136         return status;
1137 }
1138
1139 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1140 {
1141         struct device *dev = &adapter->pdev->dev;
1142         int status;
1143
1144         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1145         if (!status) {
1146                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1147                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1148         }
1149         return status;
1150 }
1151
1152 /*
1153  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1154  * If the user configures more, place BE in vlan promiscuous mode.
1155  */
1156 static int be_vid_config(struct be_adapter *adapter)
1157 {
1158         struct device *dev = &adapter->pdev->dev;
1159         u16 vids[BE_NUM_VLANS_SUPPORTED];
1160         u16 num = 0, i = 0;
1161         int status = 0;
1162
1163         /* No need to further configure vids if in promiscuous mode */
1164         if (be_in_all_promisc(adapter))
1165                 return 0;
1166
1167         if (adapter->vlans_added > be_max_vlans(adapter))
1168                 return be_set_vlan_promisc(adapter);
1169
1170         /* Construct VLAN Table to give to HW */
1171         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1172                 vids[num++] = cpu_to_le16(i);
1173
1174         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1175         if (status) {
1176                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1177                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1178                 if (addl_status(status) ==
1179                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1180                         return be_set_vlan_promisc(adapter);
1181         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1182                 status = be_clear_vlan_promisc(adapter);
1183         }
1184         return status;
1185 }
1186
1187 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1188 {
1189         struct be_adapter *adapter = netdev_priv(netdev);
1190         int status = 0;
1191
1192         /* Packets with VID 0 are always received by Lancer by default */
1193         if (lancer_chip(adapter) && vid == 0)
1194                 return status;
1195
1196         if (test_bit(vid, adapter->vids))
1197                 return status;
1198
1199         set_bit(vid, adapter->vids);
1200         adapter->vlans_added++;
1201
1202         status = be_vid_config(adapter);
1203         if (status) {
1204                 adapter->vlans_added--;
1205                 clear_bit(vid, adapter->vids);
1206         }
1207
1208         return status;
1209 }
1210
1211 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1212 {
1213         struct be_adapter *adapter = netdev_priv(netdev);
1214
1215         /* Packets with VID 0 are always received by Lancer by default */
1216         if (lancer_chip(adapter) && vid == 0)
1217                 return 0;
1218
1219         clear_bit(vid, adapter->vids);
1220         adapter->vlans_added--;
1221
1222         return be_vid_config(adapter);
1223 }
1224
1225 static void be_clear_all_promisc(struct be_adapter *adapter)
1226 {
1227         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1228         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1229 }
1230
1231 static void be_set_all_promisc(struct be_adapter *adapter)
1232 {
1233         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1234         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1235 }
1236
1237 static void be_set_mc_promisc(struct be_adapter *adapter)
1238 {
1239         int status;
1240
1241         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1242                 return;
1243
1244         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1245         if (!status)
1246                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1247 }
1248
1249 static void be_set_mc_list(struct be_adapter *adapter)
1250 {
1251         int status;
1252
1253         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1254         if (!status)
1255                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1256         else
1257                 be_set_mc_promisc(adapter);
1258 }
1259
1260 static void be_set_uc_list(struct be_adapter *adapter)
1261 {
1262         struct netdev_hw_addr *ha;
1263         int i = 1; /* First slot is claimed by the Primary MAC */
1264
1265         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1266                 be_cmd_pmac_del(adapter, adapter->if_handle,
1267                                 adapter->pmac_id[i], 0);
1268
1269         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1270                 be_set_all_promisc(adapter);
1271                 return;
1272         }
1273
1274         netdev_for_each_uc_addr(ha, adapter->netdev) {
1275                 adapter->uc_macs++; /* First slot is for Primary MAC */
1276                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1277                                 &adapter->pmac_id[adapter->uc_macs], 0);
1278         }
1279 }
1280
1281 static void be_clear_uc_list(struct be_adapter *adapter)
1282 {
1283         int i;
1284
1285         for (i = 1; i < (adapter->uc_macs + 1); i++)
1286                 be_cmd_pmac_del(adapter, adapter->if_handle,
1287                                 adapter->pmac_id[i], 0);
1288         adapter->uc_macs = 0;
1289 }
1290
1291 static void be_set_rx_mode(struct net_device *netdev)
1292 {
1293         struct be_adapter *adapter = netdev_priv(netdev);
1294
1295         if (netdev->flags & IFF_PROMISC) {
1296                 be_set_all_promisc(adapter);
1297                 return;
1298         }
1299
1300         /* Interface was previously in promiscuous mode; disable it */
1301         if (be_in_all_promisc(adapter)) {
1302                 be_clear_all_promisc(adapter);
1303                 if (adapter->vlans_added)
1304                         be_vid_config(adapter);
1305         }
1306
1307         /* Enable multicast promisc if num configured exceeds what we support */
1308         if (netdev->flags & IFF_ALLMULTI ||
1309             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1310                 be_set_mc_promisc(adapter);
1311                 return;
1312         }
1313
1314         if (netdev_uc_count(netdev) != adapter->uc_macs)
1315                 be_set_uc_list(adapter);
1316
1317         be_set_mc_list(adapter);
1318 }
1319
1320 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1321 {
1322         struct be_adapter *adapter = netdev_priv(netdev);
1323         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1324         int status;
1325
1326         if (!sriov_enabled(adapter))
1327                 return -EPERM;
1328
1329         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1330                 return -EINVAL;
1331
1332         /* Proceed further only if user provided MAC is different
1333          * from active MAC
1334          */
1335         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1336                 return 0;
1337
1338         if (BEx_chip(adapter)) {
1339                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1340                                 vf + 1);
1341
1342                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1343                                          &vf_cfg->pmac_id, vf + 1);
1344         } else {
1345                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1346                                         vf + 1);
1347         }
1348
1349         if (status) {
1350                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1351                         mac, vf, status);
1352                 return be_cmd_status(status);
1353         }
1354
1355         ether_addr_copy(vf_cfg->mac_addr, mac);
1356
1357         return 0;
1358 }
1359
1360 static int be_get_vf_config(struct net_device *netdev, int vf,
1361                             struct ifla_vf_info *vi)
1362 {
1363         struct be_adapter *adapter = netdev_priv(netdev);
1364         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1365
1366         if (!sriov_enabled(adapter))
1367                 return -EPERM;
1368
1369         if (vf >= adapter->num_vfs)
1370                 return -EINVAL;
1371
1372         vi->vf = vf;
1373         vi->max_tx_rate = vf_cfg->tx_rate;
1374         vi->min_tx_rate = 0;
1375         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1376         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1377         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1378         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1379
1380         return 0;
1381 }
1382
1383 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1384 {
1385         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1386         u16 vids[BE_NUM_VLANS_SUPPORTED];
1387         int vf_if_id = vf_cfg->if_handle;
1388         int status;
1389
1390         /* Enable Transparent VLAN Tagging */
1391         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0);
1392         if (status)
1393                 return status;
1394
1395         /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1396         vids[0] = 0;
1397         status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1398         if (!status)
1399                 dev_info(&adapter->pdev->dev,
1400                          "Cleared guest VLANs on VF%d", vf);
1401
1402         /* After TVT is enabled, disallow VFs to program VLAN filters */
1403         if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1404                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1405                                                   ~BE_PRIV_FILTMGMT, vf + 1);
1406                 if (!status)
1407                         vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1408         }
1409         return 0;
1410 }
1411
1412 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1413 {
1414         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1415         struct device *dev = &adapter->pdev->dev;
1416         int status;
1417
1418         /* Reset Transparent VLAN Tagging. */
1419         status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1420                                        vf_cfg->if_handle, 0);
1421         if (status)
1422                 return status;
1423
1424         /* Allow VFs to program VLAN filtering */
1425         if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1426                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1427                                                   BE_PRIV_FILTMGMT, vf + 1);
1428                 if (!status) {
1429                         vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1430                         dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1431                 }
1432         }
1433
1434         dev_info(dev,
1435                  "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1436         return 0;
1437 }
1438
1439 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1440 {
1441         struct be_adapter *adapter = netdev_priv(netdev);
1442         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1443         int status;
1444
1445         if (!sriov_enabled(adapter))
1446                 return -EPERM;
1447
1448         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1449                 return -EINVAL;
1450
1451         if (vlan || qos) {
1452                 vlan |= qos << VLAN_PRIO_SHIFT;
1453                 status = be_set_vf_tvt(adapter, vf, vlan);
1454         } else {
1455                 status = be_clear_vf_tvt(adapter, vf);
1456         }
1457
1458         if (status) {
1459                 dev_err(&adapter->pdev->dev,
1460                         "VLAN %d config on VF %d failed : %#x\n", vlan, vf,
1461                         status);
1462                 return be_cmd_status(status);
1463         }
1464
1465         vf_cfg->vlan_tag = vlan;
1466         return 0;
1467 }
1468
1469 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1470                              int min_tx_rate, int max_tx_rate)
1471 {
1472         struct be_adapter *adapter = netdev_priv(netdev);
1473         struct device *dev = &adapter->pdev->dev;
1474         int percent_rate, status = 0;
1475         u16 link_speed = 0;
1476         u8 link_status;
1477
1478         if (!sriov_enabled(adapter))
1479                 return -EPERM;
1480
1481         if (vf >= adapter->num_vfs)
1482                 return -EINVAL;
1483
1484         if (min_tx_rate)
1485                 return -EINVAL;
1486
1487         if (!max_tx_rate)
1488                 goto config_qos;
1489
1490         status = be_cmd_link_status_query(adapter, &link_speed,
1491                                           &link_status, 0);
1492         if (status)
1493                 goto err;
1494
1495         if (!link_status) {
1496                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1497                 status = -ENETDOWN;
1498                 goto err;
1499         }
1500
1501         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1502                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1503                         link_speed);
1504                 status = -EINVAL;
1505                 goto err;
1506         }
1507
1508         /* On Skyhawk the QOS setting must be done only as a % value */
1509         percent_rate = link_speed / 100;
1510         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1511                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1512                         percent_rate);
1513                 status = -EINVAL;
1514                 goto err;
1515         }
1516
1517 config_qos:
1518         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1519         if (status)
1520                 goto err;
1521
1522         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1523         return 0;
1524
1525 err:
1526         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1527                 max_tx_rate, vf);
1528         return be_cmd_status(status);
1529 }
1530
1531 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1532                                 int link_state)
1533 {
1534         struct be_adapter *adapter = netdev_priv(netdev);
1535         int status;
1536
1537         if (!sriov_enabled(adapter))
1538                 return -EPERM;
1539
1540         if (vf >= adapter->num_vfs)
1541                 return -EINVAL;
1542
1543         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1544         if (status) {
1545                 dev_err(&adapter->pdev->dev,
1546                         "Link state change on VF %d failed: %#x\n", vf, status);
1547                 return be_cmd_status(status);
1548         }
1549
1550         adapter->vf_cfg[vf].plink_tracking = link_state;
1551
1552         return 0;
1553 }
1554
1555 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1556                           ulong now)
1557 {
1558         aic->rx_pkts_prev = rx_pkts;
1559         aic->tx_reqs_prev = tx_pkts;
1560         aic->jiffies = now;
1561 }
1562
1563 static void be_eqd_update(struct be_adapter *adapter)
1564 {
1565         struct be_set_eqd set_eqd[MAX_EVT_QS];
1566         int eqd, i, num = 0, start;
1567         struct be_aic_obj *aic;
1568         struct be_eq_obj *eqo;
1569         struct be_rx_obj *rxo;
1570         struct be_tx_obj *txo;
1571         u64 rx_pkts, tx_pkts;
1572         ulong now;
1573         u32 pps, delta;
1574
1575         for_all_evt_queues(adapter, eqo, i) {
1576                 aic = &adapter->aic_obj[eqo->idx];
1577                 if (!aic->enable) {
1578                         if (aic->jiffies)
1579                                 aic->jiffies = 0;
1580                         eqd = aic->et_eqd;
1581                         goto modify_eqd;
1582                 }
1583
1584                 rxo = &adapter->rx_obj[eqo->idx];
1585                 do {
1586                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1587                         rx_pkts = rxo->stats.rx_pkts;
1588                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1589
1590                 txo = &adapter->tx_obj[eqo->idx];
1591                 do {
1592                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1593                         tx_pkts = txo->stats.tx_reqs;
1594                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1595
1596                 /* Skip, if wrapped around or first calculation */
1597                 now = jiffies;
1598                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1599                     rx_pkts < aic->rx_pkts_prev ||
1600                     tx_pkts < aic->tx_reqs_prev) {
1601                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1602                         continue;
1603                 }
1604
1605                 delta = jiffies_to_msecs(now - aic->jiffies);
1606                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1607                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1608                 eqd = (pps / 15000) << 2;
1609
1610                 if (eqd < 8)
1611                         eqd = 0;
1612                 eqd = min_t(u32, eqd, aic->max_eqd);
1613                 eqd = max_t(u32, eqd, aic->min_eqd);
1614
1615                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1616 modify_eqd:
1617                 if (eqd != aic->prev_eqd) {
1618                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1619                         set_eqd[num].eq_id = eqo->q.id;
1620                         aic->prev_eqd = eqd;
1621                         num++;
1622                 }
1623         }
1624
1625         if (num)
1626                 be_cmd_modify_eqd(adapter, set_eqd, num);
1627 }
1628
1629 static void be_rx_stats_update(struct be_rx_obj *rxo,
1630                                struct be_rx_compl_info *rxcp)
1631 {
1632         struct be_rx_stats *stats = rx_stats(rxo);
1633
1634         u64_stats_update_begin(&stats->sync);
1635         stats->rx_compl++;
1636         stats->rx_bytes += rxcp->pkt_size;
1637         stats->rx_pkts++;
1638         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1639                 stats->rx_mcast_pkts++;
1640         if (rxcp->err)
1641                 stats->rx_compl_err++;
1642         u64_stats_update_end(&stats->sync);
1643 }
1644
1645 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1646 {
1647         /* L4 checksum is not reliable for non TCP/UDP packets.
1648          * Also ignore ipcksm for ipv6 pkts
1649          */
1650         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1651                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1652 }
1653
1654 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1655 {
1656         struct be_adapter *adapter = rxo->adapter;
1657         struct be_rx_page_info *rx_page_info;
1658         struct be_queue_info *rxq = &rxo->q;
1659         u16 frag_idx = rxq->tail;
1660
1661         rx_page_info = &rxo->page_info_tbl[frag_idx];
1662         BUG_ON(!rx_page_info->page);
1663
1664         if (rx_page_info->last_frag) {
1665                 dma_unmap_page(&adapter->pdev->dev,
1666                                dma_unmap_addr(rx_page_info, bus),
1667                                adapter->big_page_size, DMA_FROM_DEVICE);
1668                 rx_page_info->last_frag = false;
1669         } else {
1670                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1671                                         dma_unmap_addr(rx_page_info, bus),
1672                                         rx_frag_size, DMA_FROM_DEVICE);
1673         }
1674
1675         queue_tail_inc(rxq);
1676         atomic_dec(&rxq->used);
1677         return rx_page_info;
1678 }
1679
1680 /* Throwaway the data in the Rx completion */
1681 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1682                                 struct be_rx_compl_info *rxcp)
1683 {
1684         struct be_rx_page_info *page_info;
1685         u16 i, num_rcvd = rxcp->num_rcvd;
1686
1687         for (i = 0; i < num_rcvd; i++) {
1688                 page_info = get_rx_page_info(rxo);
1689                 put_page(page_info->page);
1690                 memset(page_info, 0, sizeof(*page_info));
1691         }
1692 }
1693
1694 /*
1695  * skb_fill_rx_data forms a complete skb for an ether frame
1696  * indicated by rxcp.
1697  */
1698 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1699                              struct be_rx_compl_info *rxcp)
1700 {
1701         struct be_rx_page_info *page_info;
1702         u16 i, j;
1703         u16 hdr_len, curr_frag_len, remaining;
1704         u8 *start;
1705
1706         page_info = get_rx_page_info(rxo);
1707         start = page_address(page_info->page) + page_info->page_offset;
1708         prefetch(start);
1709
1710         /* Copy data in the first descriptor of this completion */
1711         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1712
1713         skb->len = curr_frag_len;
1714         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1715                 memcpy(skb->data, start, curr_frag_len);
1716                 /* Complete packet has now been moved to data */
1717                 put_page(page_info->page);
1718                 skb->data_len = 0;
1719                 skb->tail += curr_frag_len;
1720         } else {
1721                 hdr_len = ETH_HLEN;
1722                 memcpy(skb->data, start, hdr_len);
1723                 skb_shinfo(skb)->nr_frags = 1;
1724                 skb_frag_set_page(skb, 0, page_info->page);
1725                 skb_shinfo(skb)->frags[0].page_offset =
1726                                         page_info->page_offset + hdr_len;
1727                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1728                                   curr_frag_len - hdr_len);
1729                 skb->data_len = curr_frag_len - hdr_len;
1730                 skb->truesize += rx_frag_size;
1731                 skb->tail += hdr_len;
1732         }
1733         page_info->page = NULL;
1734
1735         if (rxcp->pkt_size <= rx_frag_size) {
1736                 BUG_ON(rxcp->num_rcvd != 1);
1737                 return;
1738         }
1739
1740         /* More frags present for this completion */
1741         remaining = rxcp->pkt_size - curr_frag_len;
1742         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1743                 page_info = get_rx_page_info(rxo);
1744                 curr_frag_len = min(remaining, rx_frag_size);
1745
1746                 /* Coalesce all frags from the same physical page in one slot */
1747                 if (page_info->page_offset == 0) {
1748                         /* Fresh page */
1749                         j++;
1750                         skb_frag_set_page(skb, j, page_info->page);
1751                         skb_shinfo(skb)->frags[j].page_offset =
1752                                                         page_info->page_offset;
1753                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1754                         skb_shinfo(skb)->nr_frags++;
1755                 } else {
1756                         put_page(page_info->page);
1757                 }
1758
1759                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1760                 skb->len += curr_frag_len;
1761                 skb->data_len += curr_frag_len;
1762                 skb->truesize += rx_frag_size;
1763                 remaining -= curr_frag_len;
1764                 page_info->page = NULL;
1765         }
1766         BUG_ON(j > MAX_SKB_FRAGS);
1767 }
1768
1769 /* Process the RX completion indicated by rxcp when GRO is disabled */
1770 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1771                                 struct be_rx_compl_info *rxcp)
1772 {
1773         struct be_adapter *adapter = rxo->adapter;
1774         struct net_device *netdev = adapter->netdev;
1775         struct sk_buff *skb;
1776
1777         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1778         if (unlikely(!skb)) {
1779                 rx_stats(rxo)->rx_drops_no_skbs++;
1780                 be_rx_compl_discard(rxo, rxcp);
1781                 return;
1782         }
1783
1784         skb_fill_rx_data(rxo, skb, rxcp);
1785
1786         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1787                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1788         else
1789                 skb_checksum_none_assert(skb);
1790
1791         skb->protocol = eth_type_trans(skb, netdev);
1792         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1793         if (netdev->features & NETIF_F_RXHASH)
1794                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1795
1796         skb->csum_level = rxcp->tunneled;
1797         skb_mark_napi_id(skb, napi);
1798
1799         if (rxcp->vlanf)
1800                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1801
1802         netif_receive_skb(skb);
1803 }
1804
1805 /* Process the RX completion indicated by rxcp when GRO is enabled */
1806 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1807                                     struct napi_struct *napi,
1808                                     struct be_rx_compl_info *rxcp)
1809 {
1810         struct be_adapter *adapter = rxo->adapter;
1811         struct be_rx_page_info *page_info;
1812         struct sk_buff *skb = NULL;
1813         u16 remaining, curr_frag_len;
1814         u16 i, j;
1815
1816         skb = napi_get_frags(napi);
1817         if (!skb) {
1818                 be_rx_compl_discard(rxo, rxcp);
1819                 return;
1820         }
1821
1822         remaining = rxcp->pkt_size;
1823         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1824                 page_info = get_rx_page_info(rxo);
1825
1826                 curr_frag_len = min(remaining, rx_frag_size);
1827
1828                 /* Coalesce all frags from the same physical page in one slot */
1829                 if (i == 0 || page_info->page_offset == 0) {
1830                         /* First frag or Fresh page */
1831                         j++;
1832                         skb_frag_set_page(skb, j, page_info->page);
1833                         skb_shinfo(skb)->frags[j].page_offset =
1834                                                         page_info->page_offset;
1835                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1836                 } else {
1837                         put_page(page_info->page);
1838                 }
1839                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1840                 skb->truesize += rx_frag_size;
1841                 remaining -= curr_frag_len;
1842                 memset(page_info, 0, sizeof(*page_info));
1843         }
1844         BUG_ON(j > MAX_SKB_FRAGS);
1845
1846         skb_shinfo(skb)->nr_frags = j + 1;
1847         skb->len = rxcp->pkt_size;
1848         skb->data_len = rxcp->pkt_size;
1849         skb->ip_summed = CHECKSUM_UNNECESSARY;
1850         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1851         if (adapter->netdev->features & NETIF_F_RXHASH)
1852                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1853
1854         skb->csum_level = rxcp->tunneled;
1855         skb_mark_napi_id(skb, napi);
1856
1857         if (rxcp->vlanf)
1858                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1859
1860         napi_gro_frags(napi);
1861 }
1862
1863 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1864                                  struct be_rx_compl_info *rxcp)
1865 {
1866         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1867         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1868         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1869         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1870         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1871         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1872         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1873         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1874         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1875         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1876         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1877         if (rxcp->vlanf) {
1878                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1879                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1880         }
1881         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1882         rxcp->tunneled =
1883                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1884 }
1885
1886 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1887                                  struct be_rx_compl_info *rxcp)
1888 {
1889         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1890         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1891         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1892         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1893         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1894         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1895         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1896         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1897         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1898         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1899         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1900         if (rxcp->vlanf) {
1901                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1902                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1903         }
1904         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1905         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1906 }
1907
1908 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1909 {
1910         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1911         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1912         struct be_adapter *adapter = rxo->adapter;
1913
1914         /* For checking the valid bit it is Ok to use either definition as the
1915          * valid bit is at the same position in both v0 and v1 Rx compl */
1916         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1917                 return NULL;
1918
1919         rmb();
1920         be_dws_le_to_cpu(compl, sizeof(*compl));
1921
1922         if (adapter->be3_native)
1923                 be_parse_rx_compl_v1(compl, rxcp);
1924         else
1925                 be_parse_rx_compl_v0(compl, rxcp);
1926
1927         if (rxcp->ip_frag)
1928                 rxcp->l4_csum = 0;
1929
1930         if (rxcp->vlanf) {
1931                 /* In QNQ modes, if qnq bit is not set, then the packet was
1932                  * tagged only with the transparent outer vlan-tag and must
1933                  * not be treated as a vlan packet by host
1934                  */
1935                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1936                         rxcp->vlanf = 0;
1937
1938                 if (!lancer_chip(adapter))
1939                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1940
1941                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1942                     !test_bit(rxcp->vlan_tag, adapter->vids))
1943                         rxcp->vlanf = 0;
1944         }
1945
1946         /* As the compl has been parsed, reset it; we wont touch it again */
1947         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1948
1949         queue_tail_inc(&rxo->cq);
1950         return rxcp;
1951 }
1952
1953 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1954 {
1955         u32 order = get_order(size);
1956
1957         if (order > 0)
1958                 gfp |= __GFP_COMP;
1959         return  alloc_pages(gfp, order);
1960 }
1961
1962 /*
1963  * Allocate a page, split it to fragments of size rx_frag_size and post as
1964  * receive buffers to BE
1965  */
1966 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1967 {
1968         struct be_adapter *adapter = rxo->adapter;
1969         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1970         struct be_queue_info *rxq = &rxo->q;
1971         struct page *pagep = NULL;
1972         struct device *dev = &adapter->pdev->dev;
1973         struct be_eth_rx_d *rxd;
1974         u64 page_dmaaddr = 0, frag_dmaaddr;
1975         u32 posted, page_offset = 0, notify = 0;
1976
1977         page_info = &rxo->page_info_tbl[rxq->head];
1978         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1979                 if (!pagep) {
1980                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1981                         if (unlikely(!pagep)) {
1982                                 rx_stats(rxo)->rx_post_fail++;
1983                                 break;
1984                         }
1985                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1986                                                     adapter->big_page_size,
1987                                                     DMA_FROM_DEVICE);
1988                         if (dma_mapping_error(dev, page_dmaaddr)) {
1989                                 put_page(pagep);
1990                                 pagep = NULL;
1991                                 adapter->drv_stats.dma_map_errors++;
1992                                 break;
1993                         }
1994                         page_offset = 0;
1995                 } else {
1996                         get_page(pagep);
1997                         page_offset += rx_frag_size;
1998                 }
1999                 page_info->page_offset = page_offset;
2000                 page_info->page = pagep;
2001
2002                 rxd = queue_head_node(rxq);
2003                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2004                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2005                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2006
2007                 /* Any space left in the current big page for another frag? */
2008                 if ((page_offset + rx_frag_size + rx_frag_size) >
2009                                         adapter->big_page_size) {
2010                         pagep = NULL;
2011                         page_info->last_frag = true;
2012                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2013                 } else {
2014                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2015                 }
2016
2017                 prev_page_info = page_info;
2018                 queue_head_inc(rxq);
2019                 page_info = &rxo->page_info_tbl[rxq->head];
2020         }
2021
2022         /* Mark the last frag of a page when we break out of the above loop
2023          * with no more slots available in the RXQ
2024          */
2025         if (pagep) {
2026                 prev_page_info->last_frag = true;
2027                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2028         }
2029
2030         if (posted) {
2031                 atomic_add(posted, &rxq->used);
2032                 if (rxo->rx_post_starved)
2033                         rxo->rx_post_starved = false;
2034                 do {
2035                         notify = min(256u, posted);
2036                         be_rxq_notify(adapter, rxq->id, notify);
2037                         posted -= notify;
2038                 } while (posted);
2039         } else if (atomic_read(&rxq->used) == 0) {
2040                 /* Let be_worker replenish when memory is available */
2041                 rxo->rx_post_starved = true;
2042         }
2043 }
2044
2045 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
2046 {
2047         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
2048
2049         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2050                 return NULL;
2051
2052         rmb();
2053         be_dws_le_to_cpu(txcp, sizeof(*txcp));
2054
2055         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2056
2057         queue_tail_inc(tx_cq);
2058         return txcp;
2059 }
2060
2061 static u16 be_tx_compl_process(struct be_adapter *adapter,
2062                                struct be_tx_obj *txo, u16 last_index)
2063 {
2064         struct sk_buff **sent_skbs = txo->sent_skb_list;
2065         struct be_queue_info *txq = &txo->q;
2066         u16 frag_index, num_wrbs = 0;
2067         struct sk_buff *skb = NULL;
2068         bool unmap_skb_hdr = false;
2069         struct be_eth_wrb *wrb;
2070
2071         do {
2072                 if (sent_skbs[txq->tail]) {
2073                         /* Free skb from prev req */
2074                         if (skb)
2075                                 dev_consume_skb_any(skb);
2076                         skb = sent_skbs[txq->tail];
2077                         sent_skbs[txq->tail] = NULL;
2078                         queue_tail_inc(txq);  /* skip hdr wrb */
2079                         num_wrbs++;
2080                         unmap_skb_hdr = true;
2081                 }
2082                 wrb = queue_tail_node(txq);
2083                 frag_index = txq->tail;
2084                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2085                               (unmap_skb_hdr && skb_headlen(skb)));
2086                 unmap_skb_hdr = false;
2087                 queue_tail_inc(txq);
2088                 num_wrbs++;
2089         } while (frag_index != last_index);
2090         dev_consume_skb_any(skb);
2091
2092         return num_wrbs;
2093 }
2094
2095 /* Return the number of events in the event queue */
2096 static inline int events_get(struct be_eq_obj *eqo)
2097 {
2098         struct be_eq_entry *eqe;
2099         int num = 0;
2100
2101         do {
2102                 eqe = queue_tail_node(&eqo->q);
2103                 if (eqe->evt == 0)
2104                         break;
2105
2106                 rmb();
2107                 eqe->evt = 0;
2108                 num++;
2109                 queue_tail_inc(&eqo->q);
2110         } while (true);
2111
2112         return num;
2113 }
2114
2115 /* Leaves the EQ is disarmed state */
2116 static void be_eq_clean(struct be_eq_obj *eqo)
2117 {
2118         int num = events_get(eqo);
2119
2120         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2121 }
2122
2123 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2124 {
2125         struct be_rx_page_info *page_info;
2126         struct be_queue_info *rxq = &rxo->q;
2127         struct be_queue_info *rx_cq = &rxo->cq;
2128         struct be_rx_compl_info *rxcp;
2129         struct be_adapter *adapter = rxo->adapter;
2130         int flush_wait = 0;
2131
2132         /* Consume pending rx completions.
2133          * Wait for the flush completion (identified by zero num_rcvd)
2134          * to arrive. Notify CQ even when there are no more CQ entries
2135          * for HW to flush partially coalesced CQ entries.
2136          * In Lancer, there is no need to wait for flush compl.
2137          */
2138         for (;;) {
2139                 rxcp = be_rx_compl_get(rxo);
2140                 if (!rxcp) {
2141                         if (lancer_chip(adapter))
2142                                 break;
2143
2144                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2145                                 dev_warn(&adapter->pdev->dev,
2146                                          "did not receive flush compl\n");
2147                                 break;
2148                         }
2149                         be_cq_notify(adapter, rx_cq->id, true, 0);
2150                         mdelay(1);
2151                 } else {
2152                         be_rx_compl_discard(rxo, rxcp);
2153                         be_cq_notify(adapter, rx_cq->id, false, 1);
2154                         if (rxcp->num_rcvd == 0)
2155                                 break;
2156                 }
2157         }
2158
2159         /* After cleanup, leave the CQ in unarmed state */
2160         be_cq_notify(adapter, rx_cq->id, false, 0);
2161
2162         /* Then free posted rx buffers that were not used */
2163         while (atomic_read(&rxq->used) > 0) {
2164                 page_info = get_rx_page_info(rxo);
2165                 put_page(page_info->page);
2166                 memset(page_info, 0, sizeof(*page_info));
2167         }
2168         BUG_ON(atomic_read(&rxq->used));
2169         rxq->tail = 0;
2170         rxq->head = 0;
2171 }
2172
2173 static void be_tx_compl_clean(struct be_adapter *adapter)
2174 {
2175         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2176         struct device *dev = &adapter->pdev->dev;
2177         struct be_tx_obj *txo;
2178         struct be_queue_info *txq;
2179         struct be_eth_tx_compl *txcp;
2180         int i, pending_txqs;
2181
2182         /* Stop polling for compls when HW has been silent for 10ms */
2183         do {
2184                 pending_txqs = adapter->num_tx_qs;
2185
2186                 for_all_tx_queues(adapter, txo, i) {
2187                         cmpl = 0;
2188                         num_wrbs = 0;
2189                         txq = &txo->q;
2190                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2191                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2192                                 num_wrbs += be_tx_compl_process(adapter, txo,
2193                                                                 end_idx);
2194                                 cmpl++;
2195                         }
2196                         if (cmpl) {
2197                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2198                                 atomic_sub(num_wrbs, &txq->used);
2199                                 timeo = 0;
2200                         }
2201                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2202                                 pending_txqs--;
2203                 }
2204
2205                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2206                         break;
2207
2208                 mdelay(1);
2209         } while (true);
2210
2211         /* Free enqueued TX that was never notified to HW */
2212         for_all_tx_queues(adapter, txo, i) {
2213                 txq = &txo->q;
2214
2215                 if (atomic_read(&txq->used)) {
2216                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2217                                  i, atomic_read(&txq->used));
2218                         notified_idx = txq->tail;
2219                         end_idx = txq->tail;
2220                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2221                                   txq->len);
2222                         /* Use the tx-compl process logic to handle requests
2223                          * that were not sent to the HW.
2224                          */
2225                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2226                         atomic_sub(num_wrbs, &txq->used);
2227                         BUG_ON(atomic_read(&txq->used));
2228                         txo->pend_wrb_cnt = 0;
2229                         /* Since hw was never notified of these requests,
2230                          * reset TXQ indices
2231                          */
2232                         txq->head = notified_idx;
2233                         txq->tail = notified_idx;
2234                 }
2235         }
2236 }
2237
2238 static void be_evt_queues_destroy(struct be_adapter *adapter)
2239 {
2240         struct be_eq_obj *eqo;
2241         int i;
2242
2243         for_all_evt_queues(adapter, eqo, i) {
2244                 if (eqo->q.created) {
2245                         be_eq_clean(eqo);
2246                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2247                         napi_hash_del(&eqo->napi);
2248                         netif_napi_del(&eqo->napi);
2249                 }
2250                 be_queue_free(adapter, &eqo->q);
2251         }
2252 }
2253
2254 static int be_evt_queues_create(struct be_adapter *adapter)
2255 {
2256         struct be_queue_info *eq;
2257         struct be_eq_obj *eqo;
2258         struct be_aic_obj *aic;
2259         int i, rc;
2260
2261         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2262                                     adapter->cfg_num_qs);
2263
2264         for_all_evt_queues(adapter, eqo, i) {
2265                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2266                                BE_NAPI_WEIGHT);
2267                 napi_hash_add(&eqo->napi);
2268                 aic = &adapter->aic_obj[i];
2269                 eqo->adapter = adapter;
2270                 eqo->idx = i;
2271                 aic->max_eqd = BE_MAX_EQD;
2272                 aic->enable = true;
2273
2274                 eq = &eqo->q;
2275                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2276                                     sizeof(struct be_eq_entry));
2277                 if (rc)
2278                         return rc;
2279
2280                 rc = be_cmd_eq_create(adapter, eqo);
2281                 if (rc)
2282                         return rc;
2283         }
2284         return 0;
2285 }
2286
2287 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2288 {
2289         struct be_queue_info *q;
2290
2291         q = &adapter->mcc_obj.q;
2292         if (q->created)
2293                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2294         be_queue_free(adapter, q);
2295
2296         q = &adapter->mcc_obj.cq;
2297         if (q->created)
2298                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2299         be_queue_free(adapter, q);
2300 }
2301
2302 /* Must be called only after TX qs are created as MCC shares TX EQ */
2303 static int be_mcc_queues_create(struct be_adapter *adapter)
2304 {
2305         struct be_queue_info *q, *cq;
2306
2307         cq = &adapter->mcc_obj.cq;
2308         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2309                            sizeof(struct be_mcc_compl)))
2310                 goto err;
2311
2312         /* Use the default EQ for MCC completions */
2313         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2314                 goto mcc_cq_free;
2315
2316         q = &adapter->mcc_obj.q;
2317         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2318                 goto mcc_cq_destroy;
2319
2320         if (be_cmd_mccq_create(adapter, q, cq))
2321                 goto mcc_q_free;
2322
2323         return 0;
2324
2325 mcc_q_free:
2326         be_queue_free(adapter, q);
2327 mcc_cq_destroy:
2328         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2329 mcc_cq_free:
2330         be_queue_free(adapter, cq);
2331 err:
2332         return -1;
2333 }
2334
2335 static void be_tx_queues_destroy(struct be_adapter *adapter)
2336 {
2337         struct be_queue_info *q;
2338         struct be_tx_obj *txo;
2339         u8 i;
2340
2341         for_all_tx_queues(adapter, txo, i) {
2342                 q = &txo->q;
2343                 if (q->created)
2344                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2345                 be_queue_free(adapter, q);
2346
2347                 q = &txo->cq;
2348                 if (q->created)
2349                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2350                 be_queue_free(adapter, q);
2351         }
2352 }
2353
2354 static int be_tx_qs_create(struct be_adapter *adapter)
2355 {
2356         struct be_queue_info *cq, *eq;
2357         struct be_tx_obj *txo;
2358         int status, i;
2359
2360         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2361
2362         for_all_tx_queues(adapter, txo, i) {
2363                 cq = &txo->cq;
2364                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2365                                         sizeof(struct be_eth_tx_compl));
2366                 if (status)
2367                         return status;
2368
2369                 u64_stats_init(&txo->stats.sync);
2370                 u64_stats_init(&txo->stats.sync_compl);
2371
2372                 /* If num_evt_qs is less than num_tx_qs, then more than
2373                  * one txq share an eq
2374                  */
2375                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2376                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2377                 if (status)
2378                         return status;
2379
2380                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2381                                         sizeof(struct be_eth_wrb));
2382                 if (status)
2383                         return status;
2384
2385                 status = be_cmd_txq_create(adapter, txo);
2386                 if (status)
2387                         return status;
2388         }
2389
2390         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2391                  adapter->num_tx_qs);
2392         return 0;
2393 }
2394
2395 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2396 {
2397         struct be_queue_info *q;
2398         struct be_rx_obj *rxo;
2399         int i;
2400
2401         for_all_rx_queues(adapter, rxo, i) {
2402                 q = &rxo->cq;
2403                 if (q->created)
2404                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2405                 be_queue_free(adapter, q);
2406         }
2407 }
2408
2409 static int be_rx_cqs_create(struct be_adapter *adapter)
2410 {
2411         struct be_queue_info *eq, *cq;
2412         struct be_rx_obj *rxo;
2413         int rc, i;
2414
2415         /* We can create as many RSS rings as there are EQs. */
2416         adapter->num_rx_qs = adapter->num_evt_qs;
2417
2418         /* We'll use RSS only if atleast 2 RSS rings are supported.
2419          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2420          */
2421         if (adapter->num_rx_qs > 1)
2422                 adapter->num_rx_qs++;
2423
2424         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2425         for_all_rx_queues(adapter, rxo, i) {
2426                 rxo->adapter = adapter;
2427                 cq = &rxo->cq;
2428                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2429                                     sizeof(struct be_eth_rx_compl));
2430                 if (rc)
2431                         return rc;
2432
2433                 u64_stats_init(&rxo->stats.sync);
2434                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2435                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2436                 if (rc)
2437                         return rc;
2438         }
2439
2440         dev_info(&adapter->pdev->dev,
2441                  "created %d RSS queue(s) and 1 default RX queue\n",
2442                  adapter->num_rx_qs - 1);
2443         return 0;
2444 }
2445
2446 static irqreturn_t be_intx(int irq, void *dev)
2447 {
2448         struct be_eq_obj *eqo = dev;
2449         struct be_adapter *adapter = eqo->adapter;
2450         int num_evts = 0;
2451
2452         /* IRQ is not expected when NAPI is scheduled as the EQ
2453          * will not be armed.
2454          * But, this can happen on Lancer INTx where it takes
2455          * a while to de-assert INTx or in BE2 where occasionaly
2456          * an interrupt may be raised even when EQ is unarmed.
2457          * If NAPI is already scheduled, then counting & notifying
2458          * events will orphan them.
2459          */
2460         if (napi_schedule_prep(&eqo->napi)) {
2461                 num_evts = events_get(eqo);
2462                 __napi_schedule(&eqo->napi);
2463                 if (num_evts)
2464                         eqo->spurious_intr = 0;
2465         }
2466         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2467
2468         /* Return IRQ_HANDLED only for the the first spurious intr
2469          * after a valid intr to stop the kernel from branding
2470          * this irq as a bad one!
2471          */
2472         if (num_evts || eqo->spurious_intr++ == 0)
2473                 return IRQ_HANDLED;
2474         else
2475                 return IRQ_NONE;
2476 }
2477
2478 static irqreturn_t be_msix(int irq, void *dev)
2479 {
2480         struct be_eq_obj *eqo = dev;
2481
2482         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2483         napi_schedule(&eqo->napi);
2484         return IRQ_HANDLED;
2485 }
2486
2487 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2488 {
2489         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2490 }
2491
2492 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2493                          int budget, int polling)
2494 {
2495         struct be_adapter *adapter = rxo->adapter;
2496         struct be_queue_info *rx_cq = &rxo->cq;
2497         struct be_rx_compl_info *rxcp;
2498         u32 work_done;
2499         u32 frags_consumed = 0;
2500
2501         for (work_done = 0; work_done < budget; work_done++) {
2502                 rxcp = be_rx_compl_get(rxo);
2503                 if (!rxcp)
2504                         break;
2505
2506                 /* Is it a flush compl that has no data */
2507                 if (unlikely(rxcp->num_rcvd == 0))
2508                         goto loop_continue;
2509
2510                 /* Discard compl with partial DMA Lancer B0 */
2511                 if (unlikely(!rxcp->pkt_size)) {
2512                         be_rx_compl_discard(rxo, rxcp);
2513                         goto loop_continue;
2514                 }
2515
2516                 /* On BE drop pkts that arrive due to imperfect filtering in
2517                  * promiscuous mode on some skews
2518                  */
2519                 if (unlikely(rxcp->port != adapter->port_num &&
2520                              !lancer_chip(adapter))) {
2521                         be_rx_compl_discard(rxo, rxcp);
2522                         goto loop_continue;
2523                 }
2524
2525                 /* Don't do gro when we're busy_polling */
2526                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2527                         be_rx_compl_process_gro(rxo, napi, rxcp);
2528                 else
2529                         be_rx_compl_process(rxo, napi, rxcp);
2530
2531 loop_continue:
2532                 frags_consumed += rxcp->num_rcvd;
2533                 be_rx_stats_update(rxo, rxcp);
2534         }
2535
2536         if (work_done) {
2537                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2538
2539                 /* When an rx-obj gets into post_starved state, just
2540                  * let be_worker do the posting.
2541                  */
2542                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2543                     !rxo->rx_post_starved)
2544                         be_post_rx_frags(rxo, GFP_ATOMIC,
2545                                          max_t(u32, MAX_RX_POST,
2546                                                frags_consumed));
2547         }
2548
2549         return work_done;
2550 }
2551
2552 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2553 {
2554         switch (status) {
2555         case BE_TX_COMP_HDR_PARSE_ERR:
2556                 tx_stats(txo)->tx_hdr_parse_err++;
2557                 break;
2558         case BE_TX_COMP_NDMA_ERR:
2559                 tx_stats(txo)->tx_dma_err++;
2560                 break;
2561         case BE_TX_COMP_ACL_ERR:
2562                 tx_stats(txo)->tx_spoof_check_err++;
2563                 break;
2564         }
2565 }
2566
2567 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2568 {
2569         switch (status) {
2570         case LANCER_TX_COMP_LSO_ERR:
2571                 tx_stats(txo)->tx_tso_err++;
2572                 break;
2573         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2574         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2575                 tx_stats(txo)->tx_spoof_check_err++;
2576                 break;
2577         case LANCER_TX_COMP_QINQ_ERR:
2578                 tx_stats(txo)->tx_qinq_err++;
2579                 break;
2580         case LANCER_TX_COMP_PARITY_ERR:
2581                 tx_stats(txo)->tx_internal_parity_err++;
2582                 break;
2583         case LANCER_TX_COMP_DMA_ERR:
2584                 tx_stats(txo)->tx_dma_err++;
2585                 break;
2586         }
2587 }
2588
2589 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2590                           int idx)
2591 {
2592         struct be_eth_tx_compl *txcp;
2593         int num_wrbs = 0, work_done = 0;
2594         u32 compl_status;
2595         u16 last_idx;
2596
2597         while ((txcp = be_tx_compl_get(&txo->cq))) {
2598                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2599                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2600                 work_done++;
2601
2602                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2603                 if (compl_status) {
2604                         if (lancer_chip(adapter))
2605                                 lancer_update_tx_err(txo, compl_status);
2606                         else
2607                                 be_update_tx_err(txo, compl_status);
2608                 }
2609         }
2610
2611         if (work_done) {
2612                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2613                 atomic_sub(num_wrbs, &txo->q.used);
2614
2615                 /* As Tx wrbs have been freed up, wake up netdev queue
2616                  * if it was stopped due to lack of tx wrbs.  */
2617                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2618                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2619                         netif_wake_subqueue(adapter->netdev, idx);
2620                 }
2621
2622                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2623                 tx_stats(txo)->tx_compl += work_done;
2624                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2625         }
2626 }
2627
2628 #ifdef CONFIG_NET_RX_BUSY_POLL
2629 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2630 {
2631         bool status = true;
2632
2633         spin_lock(&eqo->lock); /* BH is already disabled */
2634         if (eqo->state & BE_EQ_LOCKED) {
2635                 WARN_ON(eqo->state & BE_EQ_NAPI);
2636                 eqo->state |= BE_EQ_NAPI_YIELD;
2637                 status = false;
2638         } else {
2639                 eqo->state = BE_EQ_NAPI;
2640         }
2641         spin_unlock(&eqo->lock);
2642         return status;
2643 }
2644
2645 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2646 {
2647         spin_lock(&eqo->lock); /* BH is already disabled */
2648
2649         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2650         eqo->state = BE_EQ_IDLE;
2651
2652         spin_unlock(&eqo->lock);
2653 }
2654
2655 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2656 {
2657         bool status = true;
2658
2659         spin_lock_bh(&eqo->lock);
2660         if (eqo->state & BE_EQ_LOCKED) {
2661                 eqo->state |= BE_EQ_POLL_YIELD;
2662                 status = false;
2663         } else {
2664                 eqo->state |= BE_EQ_POLL;
2665         }
2666         spin_unlock_bh(&eqo->lock);
2667         return status;
2668 }
2669
2670 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2671 {
2672         spin_lock_bh(&eqo->lock);
2673
2674         WARN_ON(eqo->state & (BE_EQ_NAPI));
2675         eqo->state = BE_EQ_IDLE;
2676
2677         spin_unlock_bh(&eqo->lock);
2678 }
2679
2680 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2681 {
2682         spin_lock_init(&eqo->lock);
2683         eqo->state = BE_EQ_IDLE;
2684 }
2685
2686 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2687 {
2688         local_bh_disable();
2689
2690         /* It's enough to just acquire napi lock on the eqo to stop
2691          * be_busy_poll() from processing any queueus.
2692          */
2693         while (!be_lock_napi(eqo))
2694                 mdelay(1);
2695
2696         local_bh_enable();
2697 }
2698
2699 #else /* CONFIG_NET_RX_BUSY_POLL */
2700
2701 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2702 {
2703         return true;
2704 }
2705
2706 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2707 {
2708 }
2709
2710 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2711 {
2712         return false;
2713 }
2714
2715 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2716 {
2717 }
2718
2719 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2720 {
2721 }
2722
2723 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2724 {
2725 }
2726 #endif /* CONFIG_NET_RX_BUSY_POLL */
2727
2728 int be_poll(struct napi_struct *napi, int budget)
2729 {
2730         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2731         struct be_adapter *adapter = eqo->adapter;
2732         int max_work = 0, work, i, num_evts;
2733         struct be_rx_obj *rxo;
2734         struct be_tx_obj *txo;
2735
2736         num_evts = events_get(eqo);
2737
2738         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2739                 be_process_tx(adapter, txo, i);
2740
2741         if (be_lock_napi(eqo)) {
2742                 /* This loop will iterate twice for EQ0 in which
2743                  * completions of the last RXQ (default one) are also processed
2744                  * For other EQs the loop iterates only once
2745                  */
2746                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2747                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2748                         max_work = max(work, max_work);
2749                 }
2750                 be_unlock_napi(eqo);
2751         } else {
2752                 max_work = budget;
2753         }
2754
2755         if (is_mcc_eqo(eqo))
2756                 be_process_mcc(adapter);
2757
2758         if (max_work < budget) {
2759                 napi_complete(napi);
2760                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2761         } else {
2762                 /* As we'll continue in polling mode, count and clear events */
2763                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2764         }
2765         return max_work;
2766 }
2767
2768 #ifdef CONFIG_NET_RX_BUSY_POLL
2769 static int be_busy_poll(struct napi_struct *napi)
2770 {
2771         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2772         struct be_adapter *adapter = eqo->adapter;
2773         struct be_rx_obj *rxo;
2774         int i, work = 0;
2775
2776         if (!be_lock_busy_poll(eqo))
2777                 return LL_FLUSH_BUSY;
2778
2779         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2780                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2781                 if (work)
2782                         break;
2783         }
2784
2785         be_unlock_busy_poll(eqo);
2786         return work;
2787 }
2788 #endif
2789
2790 void be_detect_error(struct be_adapter *adapter)
2791 {
2792         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2793         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2794         u32 i;
2795         bool error_detected = false;
2796         struct device *dev = &adapter->pdev->dev;
2797         struct net_device *netdev = adapter->netdev;
2798
2799         if (be_hw_error(adapter))
2800                 return;
2801
2802         if (lancer_chip(adapter)) {
2803                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2804                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2805                         sliport_err1 = ioread32(adapter->db +
2806                                                 SLIPORT_ERROR1_OFFSET);
2807                         sliport_err2 = ioread32(adapter->db +
2808                                                 SLIPORT_ERROR2_OFFSET);
2809                         adapter->hw_error = true;
2810                         /* Do not log error messages if its a FW reset */
2811                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2812                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2813                                 dev_info(dev, "Firmware update in progress\n");
2814                         } else {
2815                                 error_detected = true;
2816                                 dev_err(dev, "Error detected in the card\n");
2817                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2818                                         sliport_status);
2819                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2820                                         sliport_err1);
2821                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2822                                         sliport_err2);
2823                         }
2824                 }
2825         } else {
2826                 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW);
2827                 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH);
2828                 ue_lo_mask = ioread32(adapter->pcicfg +
2829                                       PCICFG_UE_STATUS_LOW_MASK);
2830                 ue_hi_mask = ioread32(adapter->pcicfg +
2831                                       PCICFG_UE_STATUS_HI_MASK);
2832
2833                 ue_lo = (ue_lo & ~ue_lo_mask);
2834                 ue_hi = (ue_hi & ~ue_hi_mask);
2835
2836                 /* On certain platforms BE hardware can indicate spurious UEs.
2837                  * Allow HW to stop working completely in case of a real UE.
2838                  * Hence not setting the hw_error for UE detection.
2839                  */
2840
2841                 if (ue_lo || ue_hi) {
2842                         error_detected = true;
2843                         dev_err(dev,
2844                                 "Unrecoverable Error detected in the adapter");
2845                         dev_err(dev, "Please reboot server to recover");
2846                         if (skyhawk_chip(adapter))
2847                                 adapter->hw_error = true;
2848                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2849                                 if (ue_lo & 1)
2850                                         dev_err(dev, "UE: %s bit set\n",
2851                                                 ue_status_low_desc[i]);
2852                         }
2853                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2854                                 if (ue_hi & 1)
2855                                         dev_err(dev, "UE: %s bit set\n",
2856                                                 ue_status_hi_desc[i]);
2857                         }
2858                 }
2859         }
2860         if (error_detected)
2861                 netif_carrier_off(netdev);
2862 }
2863
2864 static void be_msix_disable(struct be_adapter *adapter)
2865 {
2866         if (msix_enabled(adapter)) {
2867                 pci_disable_msix(adapter->pdev);
2868                 adapter->num_msix_vec = 0;
2869                 adapter->num_msix_roce_vec = 0;
2870         }
2871 }
2872
2873 static int be_msix_enable(struct be_adapter *adapter)
2874 {
2875         int i, num_vec;
2876         struct device *dev = &adapter->pdev->dev;
2877
2878         /* If RoCE is supported, program the max number of NIC vectors that
2879          * may be configured via set-channels, along with vectors needed for
2880          * RoCe. Else, just program the number we'll use initially.
2881          */
2882         if (be_roce_supported(adapter))
2883                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2884                                 2 * num_online_cpus());
2885         else
2886                 num_vec = adapter->cfg_num_qs;
2887
2888         for (i = 0; i < num_vec; i++)
2889                 adapter->msix_entries[i].entry = i;
2890
2891         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2892                                         MIN_MSIX_VECTORS, num_vec);
2893         if (num_vec < 0)
2894                 goto fail;
2895
2896         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2897                 adapter->num_msix_roce_vec = num_vec / 2;
2898                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2899                          adapter->num_msix_roce_vec);
2900         }
2901
2902         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2903
2904         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2905                  adapter->num_msix_vec);
2906         return 0;
2907
2908 fail:
2909         dev_warn(dev, "MSIx enable failed\n");
2910
2911         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2912         if (!be_physfn(adapter))
2913                 return num_vec;
2914         return 0;
2915 }
2916
2917 static inline int be_msix_vec_get(struct be_adapter *adapter,
2918                                   struct be_eq_obj *eqo)
2919 {
2920         return adapter->msix_entries[eqo->msix_idx].vector;
2921 }
2922
2923 static int be_msix_register(struct be_adapter *adapter)
2924 {
2925         struct net_device *netdev = adapter->netdev;
2926         struct be_eq_obj *eqo;
2927         int status, i, vec;
2928
2929         for_all_evt_queues(adapter, eqo, i) {
2930                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2931                 vec = be_msix_vec_get(adapter, eqo);
2932                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2933                 if (status)
2934                         goto err_msix;
2935         }
2936
2937         return 0;
2938 err_msix:
2939         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2940                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2941         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2942                  status);
2943         be_msix_disable(adapter);
2944         return status;
2945 }
2946
2947 static int be_irq_register(struct be_adapter *adapter)
2948 {
2949         struct net_device *netdev = adapter->netdev;
2950         int status;
2951
2952         if (msix_enabled(adapter)) {
2953                 status = be_msix_register(adapter);
2954                 if (status == 0)
2955                         goto done;
2956                 /* INTx is not supported for VF */
2957                 if (!be_physfn(adapter))
2958                         return status;
2959         }
2960
2961         /* INTx: only the first EQ is used */
2962         netdev->irq = adapter->pdev->irq;
2963         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2964                              &adapter->eq_obj[0]);
2965         if (status) {
2966                 dev_err(&adapter->pdev->dev,
2967                         "INTx request IRQ failed - err %d\n", status);
2968                 return status;
2969         }
2970 done:
2971         adapter->isr_registered = true;
2972         return 0;
2973 }
2974
2975 static void be_irq_unregister(struct be_adapter *adapter)
2976 {
2977         struct net_device *netdev = adapter->netdev;
2978         struct be_eq_obj *eqo;
2979         int i;
2980
2981         if (!adapter->isr_registered)
2982                 return;
2983
2984         /* INTx */
2985         if (!msix_enabled(adapter)) {
2986                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2987                 goto done;
2988         }
2989
2990         /* MSIx */
2991         for_all_evt_queues(adapter, eqo, i)
2992                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2993
2994 done:
2995         adapter->isr_registered = false;
2996 }
2997
2998 static void be_rx_qs_destroy(struct be_adapter *adapter)
2999 {
3000         struct be_queue_info *q;
3001         struct be_rx_obj *rxo;
3002         int i;
3003
3004         for_all_rx_queues(adapter, rxo, i) {
3005                 q = &rxo->q;
3006                 if (q->created) {
3007                         be_cmd_rxq_destroy(adapter, q);
3008                         be_rx_cq_clean(rxo);
3009                 }
3010                 be_queue_free(adapter, q);
3011         }
3012 }
3013
3014 static int be_close(struct net_device *netdev)
3015 {
3016         struct be_adapter *adapter = netdev_priv(netdev);
3017         struct be_eq_obj *eqo;
3018         int i;
3019
3020         /* This protection is needed as be_close() may be called even when the
3021          * adapter is in cleared state (after eeh perm failure)
3022          */
3023         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3024                 return 0;
3025
3026         be_roce_dev_close(adapter);
3027
3028         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3029                 for_all_evt_queues(adapter, eqo, i) {
3030                         napi_disable(&eqo->napi);
3031                         be_disable_busy_poll(eqo);
3032                 }
3033                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3034         }
3035
3036         be_async_mcc_disable(adapter);
3037
3038         /* Wait for all pending tx completions to arrive so that
3039          * all tx skbs are freed.
3040          */
3041         netif_tx_disable(netdev);
3042         be_tx_compl_clean(adapter);
3043
3044         be_rx_qs_destroy(adapter);
3045         be_clear_uc_list(adapter);
3046
3047         for_all_evt_queues(adapter, eqo, i) {
3048                 if (msix_enabled(adapter))
3049                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3050                 else
3051                         synchronize_irq(netdev->irq);
3052                 be_eq_clean(eqo);
3053         }
3054
3055         be_irq_unregister(adapter);
3056
3057         return 0;
3058 }
3059
3060 static int be_rx_qs_create(struct be_adapter *adapter)
3061 {
3062         struct rss_info *rss = &adapter->rss_info;
3063         u8 rss_key[RSS_HASH_KEY_LEN];
3064         struct be_rx_obj *rxo;
3065         int rc, i, j;
3066
3067         for_all_rx_queues(adapter, rxo, i) {
3068                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3069                                     sizeof(struct be_eth_rx_d));
3070                 if (rc)
3071                         return rc;
3072         }
3073
3074         /* The FW would like the default RXQ to be created first */
3075         rxo = default_rxo(adapter);
3076         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
3077                                adapter->if_handle, false, &rxo->rss_id);
3078         if (rc)
3079                 return rc;
3080
3081         for_all_rss_queues(adapter, rxo, i) {
3082                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3083                                        rx_frag_size, adapter->if_handle,
3084                                        true, &rxo->rss_id);
3085                 if (rc)
3086                         return rc;
3087         }
3088
3089         if (be_multi_rxq(adapter)) {
3090                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
3091                         j += adapter->num_rx_qs - 1) {
3092                         for_all_rss_queues(adapter, rxo, i) {
3093                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3094                                         break;
3095                                 rss->rsstable[j + i] = rxo->rss_id;
3096                                 rss->rss_queue[j + i] = i;
3097                         }
3098                 }
3099                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3100                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3101
3102                 if (!BEx_chip(adapter))
3103                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3104                                 RSS_ENABLE_UDP_IPV6;
3105         } else {
3106                 /* Disable RSS, if only default RX Q is created */
3107                 rss->rss_flags = RSS_ENABLE_NONE;
3108         }
3109
3110         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3111         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3112                                128, rss_key);
3113         if (rc) {
3114                 rss->rss_flags = RSS_ENABLE_NONE;
3115                 return rc;
3116         }
3117
3118         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3119
3120         /* First time posting */
3121         for_all_rx_queues(adapter, rxo, i)
3122                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
3123         return 0;
3124 }
3125
3126 static int be_open(struct net_device *netdev)
3127 {
3128         struct be_adapter *adapter = netdev_priv(netdev);
3129         struct be_eq_obj *eqo;
3130         struct be_rx_obj *rxo;
3131         struct be_tx_obj *txo;
3132         u8 link_status;
3133         int status, i;
3134
3135         status = be_rx_qs_create(adapter);
3136         if (status)
3137                 goto err;
3138
3139         status = be_irq_register(adapter);
3140         if (status)
3141                 goto err;
3142
3143         for_all_rx_queues(adapter, rxo, i)
3144                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3145
3146         for_all_tx_queues(adapter, txo, i)
3147                 be_cq_notify(adapter, txo->cq.id, true, 0);
3148
3149         be_async_mcc_enable(adapter);
3150
3151         for_all_evt_queues(adapter, eqo, i) {
3152                 napi_enable(&eqo->napi);
3153                 be_enable_busy_poll(eqo);
3154                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
3155         }
3156         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3157
3158         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3159         if (!status)
3160                 be_link_status_update(adapter, link_status);
3161
3162         netif_tx_start_all_queues(netdev);
3163         be_roce_dev_open(adapter);
3164
3165 #ifdef CONFIG_BE2NET_VXLAN
3166         if (skyhawk_chip(adapter))
3167                 vxlan_get_rx_port(netdev);
3168 #endif
3169
3170         return 0;
3171 err:
3172         be_close(adapter->netdev);
3173         return -EIO;
3174 }
3175
3176 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3177 {
3178         struct be_dma_mem cmd;
3179         int status = 0;
3180         u8 mac[ETH_ALEN];
3181
3182         memset(mac, 0, ETH_ALEN);
3183
3184         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3185         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3186                                      GFP_KERNEL);
3187         if (!cmd.va)
3188                 return -ENOMEM;
3189
3190         if (enable) {
3191                 status = pci_write_config_dword(adapter->pdev,
3192                                                 PCICFG_PM_CONTROL_OFFSET,
3193                                                 PCICFG_PM_CONTROL_MASK);
3194                 if (status) {
3195                         dev_err(&adapter->pdev->dev,
3196                                 "Could not enable Wake-on-lan\n");
3197                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3198                                           cmd.dma);
3199                         return status;
3200                 }
3201                 status = be_cmd_enable_magic_wol(adapter,
3202                                                  adapter->netdev->dev_addr,
3203                                                  &cmd);
3204                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3205                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3206         } else {
3207                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3208                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3209                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3210         }
3211
3212         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3213         return status;
3214 }
3215
3216 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3217 {
3218         u32 addr;
3219
3220         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3221
3222         mac[5] = (u8)(addr & 0xFF);
3223         mac[4] = (u8)((addr >> 8) & 0xFF);
3224         mac[3] = (u8)((addr >> 16) & 0xFF);
3225         /* Use the OUI from the current MAC address */
3226         memcpy(mac, adapter->netdev->dev_addr, 3);
3227 }
3228
3229 /*
3230  * Generate a seed MAC address from the PF MAC Address using jhash.
3231  * MAC Address for VFs are assigned incrementally starting from the seed.
3232  * These addresses are programmed in the ASIC by the PF and the VF driver
3233  * queries for the MAC address during its probe.
3234  */
3235 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3236 {
3237         u32 vf;
3238         int status = 0;
3239         u8 mac[ETH_ALEN];
3240         struct be_vf_cfg *vf_cfg;
3241
3242         be_vf_eth_addr_generate(adapter, mac);
3243
3244         for_all_vfs(adapter, vf_cfg, vf) {
3245                 if (BEx_chip(adapter))
3246                         status = be_cmd_pmac_add(adapter, mac,
3247                                                  vf_cfg->if_handle,
3248                                                  &vf_cfg->pmac_id, vf + 1);
3249                 else
3250                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3251                                                 vf + 1);
3252
3253                 if (status)
3254                         dev_err(&adapter->pdev->dev,
3255                                 "Mac address assignment failed for VF %d\n",
3256                                 vf);
3257                 else
3258                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3259
3260                 mac[5] += 1;
3261         }
3262         return status;
3263 }
3264
3265 static int be_vfs_mac_query(struct be_adapter *adapter)
3266 {
3267         int status, vf;
3268         u8 mac[ETH_ALEN];
3269         struct be_vf_cfg *vf_cfg;
3270
3271         for_all_vfs(adapter, vf_cfg, vf) {
3272                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3273                                                mac, vf_cfg->if_handle,
3274                                                false, vf+1);
3275                 if (status)
3276                         return status;
3277                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3278         }
3279         return 0;
3280 }
3281
3282 static void be_vf_clear(struct be_adapter *adapter)
3283 {
3284         struct be_vf_cfg *vf_cfg;
3285         u32 vf;
3286
3287         if (pci_vfs_assigned(adapter->pdev)) {
3288                 dev_warn(&adapter->pdev->dev,
3289                          "VFs are assigned to VMs: not disabling VFs\n");
3290                 goto done;
3291         }
3292
3293         pci_disable_sriov(adapter->pdev);
3294
3295         for_all_vfs(adapter, vf_cfg, vf) {
3296                 if (BEx_chip(adapter))
3297                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3298                                         vf_cfg->pmac_id, vf + 1);
3299                 else
3300                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3301                                        vf + 1);
3302
3303                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3304         }
3305 done:
3306         kfree(adapter->vf_cfg);
3307         adapter->num_vfs = 0;
3308         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3309 }
3310
3311 static void be_clear_queues(struct be_adapter *adapter)
3312 {
3313         be_mcc_queues_destroy(adapter);
3314         be_rx_cqs_destroy(adapter);
3315         be_tx_queues_destroy(adapter);
3316         be_evt_queues_destroy(adapter);
3317 }
3318
3319 static void be_cancel_worker(struct be_adapter *adapter)
3320 {
3321         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3322                 cancel_delayed_work_sync(&adapter->work);
3323                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3324         }
3325 }
3326
3327 static void be_mac_clear(struct be_adapter *adapter)
3328 {
3329         if (adapter->pmac_id) {
3330                 be_cmd_pmac_del(adapter, adapter->if_handle,
3331                                 adapter->pmac_id[0], 0);
3332                 kfree(adapter->pmac_id);
3333                 adapter->pmac_id = NULL;
3334         }
3335 }
3336
3337 #ifdef CONFIG_BE2NET_VXLAN
3338 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3339 {
3340         struct net_device *netdev = adapter->netdev;
3341
3342         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3343                 be_cmd_manage_iface(adapter, adapter->if_handle,
3344                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3345
3346         if (adapter->vxlan_port)
3347                 be_cmd_set_vxlan_port(adapter, 0);
3348
3349         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3350         adapter->vxlan_port = 0;
3351
3352         netdev->hw_enc_features = 0;
3353         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3354         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3355 }
3356 #endif
3357
3358 static int be_clear(struct be_adapter *adapter)
3359 {
3360         be_cancel_worker(adapter);
3361
3362         if (sriov_enabled(adapter))
3363                 be_vf_clear(adapter);
3364
3365         /* Re-configure FW to distribute resources evenly across max-supported
3366          * number of VFs, only when VFs are not already enabled.
3367          */
3368         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3369                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3370                                         pci_sriov_get_totalvfs(adapter->pdev));
3371
3372 #ifdef CONFIG_BE2NET_VXLAN
3373         be_disable_vxlan_offloads(adapter);
3374 #endif
3375         /* delete the primary mac along with the uc-mac list */
3376         be_mac_clear(adapter);
3377
3378         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3379
3380         be_clear_queues(adapter);
3381
3382         be_msix_disable(adapter);
3383         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3384         return 0;
3385 }
3386
3387 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3388                         u32 cap_flags, u32 vf)
3389 {
3390         u32 en_flags;
3391
3392         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3393                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3394                    BE_IF_FLAGS_RSS;
3395
3396         en_flags &= cap_flags;
3397
3398         return be_cmd_if_create(adapter, cap_flags, en_flags, if_handle, vf);
3399 }
3400
3401 static int be_vfs_if_create(struct be_adapter *adapter)
3402 {
3403         struct be_resources res = {0};
3404         struct be_vf_cfg *vf_cfg;
3405         u32 cap_flags, vf;
3406         int status;
3407
3408         /* If a FW profile exists, then cap_flags are updated */
3409         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3410                     BE_IF_FLAGS_MULTICAST;
3411
3412         for_all_vfs(adapter, vf_cfg, vf) {
3413                 if (!BE3_chip(adapter)) {
3414                         status = be_cmd_get_profile_config(adapter, &res,
3415                                                            vf + 1);
3416                         if (!status) {
3417                                 cap_flags = res.if_cap_flags;
3418                                 /* Prevent VFs from enabling VLAN promiscuous
3419                                  * mode
3420                                  */
3421                                 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
3422                         }
3423                 }
3424
3425                 status = be_if_create(adapter, &vf_cfg->if_handle,
3426                                       cap_flags, vf + 1);
3427                 if (status)
3428                         return status;
3429         }
3430
3431         return 0;
3432 }
3433
3434 static int be_vf_setup_init(struct be_adapter *adapter)
3435 {
3436         struct be_vf_cfg *vf_cfg;
3437         int vf;
3438
3439         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3440                                   GFP_KERNEL);
3441         if (!adapter->vf_cfg)
3442                 return -ENOMEM;
3443
3444         for_all_vfs(adapter, vf_cfg, vf) {
3445                 vf_cfg->if_handle = -1;
3446                 vf_cfg->pmac_id = -1;
3447         }
3448         return 0;
3449 }
3450
3451 static int be_vf_setup(struct be_adapter *adapter)
3452 {
3453         struct device *dev = &adapter->pdev->dev;
3454         struct be_vf_cfg *vf_cfg;
3455         int status, old_vfs, vf;
3456
3457         old_vfs = pci_num_vf(adapter->pdev);
3458
3459         status = be_vf_setup_init(adapter);
3460         if (status)
3461                 goto err;
3462
3463         if (old_vfs) {
3464                 for_all_vfs(adapter, vf_cfg, vf) {
3465                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3466                         if (status)
3467                                 goto err;
3468                 }
3469
3470                 status = be_vfs_mac_query(adapter);
3471                 if (status)
3472                         goto err;
3473         } else {
3474                 status = be_vfs_if_create(adapter);
3475                 if (status)
3476                         goto err;
3477
3478                 status = be_vf_eth_addr_config(adapter);
3479                 if (status)
3480                         goto err;
3481         }
3482
3483         for_all_vfs(adapter, vf_cfg, vf) {
3484                 /* Allow VFs to programs MAC/VLAN filters */
3485                 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
3486                                                   vf + 1);
3487                 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
3488                         status = be_cmd_set_fn_privileges(adapter,
3489                                                           vf_cfg->privileges |
3490                                                           BE_PRIV_FILTMGMT,
3491                                                           vf + 1);
3492                         if (!status) {
3493                                 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
3494                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3495                                          vf);
3496                         }
3497                 }
3498
3499                 /* Allow full available bandwidth */
3500                 if (!old_vfs)
3501                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3502
3503                 if (!old_vfs) {
3504                         be_cmd_enable_vf(adapter, vf + 1);
3505                         be_cmd_set_logical_link_config(adapter,
3506                                                        IFLA_VF_LINK_STATE_AUTO,
3507                                                        vf+1);
3508                 }
3509         }
3510
3511         if (!old_vfs) {
3512                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3513                 if (status) {
3514                         dev_err(dev, "SRIOV enable failed\n");
3515                         adapter->num_vfs = 0;
3516                         goto err;
3517                 }
3518         }
3519
3520         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3521         return 0;
3522 err:
3523         dev_err(dev, "VF setup failed\n");
3524         be_vf_clear(adapter);
3525         return status;
3526 }
3527
3528 /* Converting function_mode bits on BE3 to SH mc_type enums */
3529
3530 static u8 be_convert_mc_type(u32 function_mode)
3531 {
3532         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3533                 return vNIC1;
3534         else if (function_mode & QNQ_MODE)
3535                 return FLEX10;
3536         else if (function_mode & VNIC_MODE)
3537                 return vNIC2;
3538         else if (function_mode & UMC_ENABLED)
3539                 return UMC;
3540         else
3541                 return MC_NONE;
3542 }
3543
3544 /* On BE2/BE3 FW does not suggest the supported limits */
3545 static void BEx_get_resources(struct be_adapter *adapter,
3546                               struct be_resources *res)
3547 {
3548         bool use_sriov = adapter->num_vfs ? 1 : 0;
3549
3550         if (be_physfn(adapter))
3551                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3552         else
3553                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3554
3555         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3556
3557         if (be_is_mc(adapter)) {
3558                 /* Assuming that there are 4 channels per port,
3559                  * when multi-channel is enabled
3560                  */
3561                 if (be_is_qnq_mode(adapter))
3562                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3563                 else
3564                         /* In a non-qnq multichannel mode, the pvid
3565                          * takes up one vlan entry
3566                          */
3567                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3568         } else {
3569                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3570         }
3571
3572         res->max_mcast_mac = BE_MAX_MC;
3573
3574         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3575          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3576          *    *only* if it is RSS-capable.
3577          */
3578         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3579             !be_physfn(adapter) || (be_is_mc(adapter) &&
3580             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3581                 res->max_tx_qs = 1;
3582         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3583                 struct be_resources super_nic_res = {0};
3584
3585                 /* On a SuperNIC profile, the driver needs to use the
3586                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3587                  */
3588                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3589                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3590                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3591         } else {
3592                 res->max_tx_qs = BE3_MAX_TX_QS;
3593         }
3594
3595         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3596             !use_sriov && be_physfn(adapter))
3597                 res->max_rss_qs = (adapter->be3_native) ?
3598                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3599         res->max_rx_qs = res->max_rss_qs + 1;
3600
3601         if (be_physfn(adapter))
3602                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3603                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3604         else
3605                 res->max_evt_qs = 1;
3606
3607         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3608         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3609                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3610 }
3611
3612 static void be_setup_init(struct be_adapter *adapter)
3613 {
3614         adapter->vlan_prio_bmap = 0xff;
3615         adapter->phy.link_speed = -1;
3616         adapter->if_handle = -1;
3617         adapter->be3_native = false;
3618         adapter->if_flags = 0;
3619         if (be_physfn(adapter))
3620                 adapter->cmd_privileges = MAX_PRIVILEGES;
3621         else
3622                 adapter->cmd_privileges = MIN_PRIVILEGES;
3623 }
3624
3625 static int be_get_sriov_config(struct be_adapter *adapter)
3626 {
3627         struct device *dev = &adapter->pdev->dev;
3628         struct be_resources res = {0};
3629         int max_vfs, old_vfs;
3630
3631         /* Some old versions of BE3 FW don't report max_vfs value */
3632         be_cmd_get_profile_config(adapter, &res, 0);
3633
3634         if (BE3_chip(adapter) && !res.max_vfs) {
3635                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3636                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3637         }
3638
3639         adapter->pool_res = res;
3640
3641         if (!be_max_vfs(adapter)) {
3642                 if (num_vfs)
3643                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3644                 adapter->num_vfs = 0;
3645                 return 0;
3646         }
3647
3648         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3649
3650         /* validate num_vfs module param */
3651         old_vfs = pci_num_vf(adapter->pdev);
3652         if (old_vfs) {
3653                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3654                 if (old_vfs != num_vfs)
3655                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3656                 adapter->num_vfs = old_vfs;
3657         } else {
3658                 if (num_vfs > be_max_vfs(adapter)) {
3659                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3660                                  num_vfs);
3661                         dev_info(dev, "Limiting to %d VFs\n",
3662                                  be_max_vfs(adapter));
3663                 }
3664                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3665         }
3666
3667         return 0;
3668 }
3669
3670 static int be_get_resources(struct be_adapter *adapter)
3671 {
3672         struct device *dev = &adapter->pdev->dev;
3673         struct be_resources res = {0};
3674         int status;
3675
3676         if (BEx_chip(adapter)) {
3677                 BEx_get_resources(adapter, &res);
3678                 adapter->res = res;
3679         }
3680
3681         /* For Lancer, SH etc read per-function resource limits from FW.
3682          * GET_FUNC_CONFIG returns per function guaranteed limits.
3683          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3684          */
3685         if (!BEx_chip(adapter)) {
3686                 status = be_cmd_get_func_config(adapter, &res);
3687                 if (status)
3688                         return status;
3689
3690                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3691                 if (be_roce_supported(adapter))
3692                         res.max_evt_qs /= 2;
3693                 adapter->res = res;
3694         }
3695
3696         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3697                  be_max_txqs(adapter), be_max_rxqs(adapter),
3698                  be_max_rss(adapter), be_max_eqs(adapter),
3699                  be_max_vfs(adapter));
3700         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3701                  be_max_uc(adapter), be_max_mc(adapter),
3702                  be_max_vlans(adapter));
3703
3704         return 0;
3705 }
3706
3707 static void be_sriov_config(struct be_adapter *adapter)
3708 {
3709         struct device *dev = &adapter->pdev->dev;
3710         int status;
3711
3712         status = be_get_sriov_config(adapter);
3713         if (status) {
3714                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3715                 dev_err(dev, "SR-IOV cannot be enabled\n");
3716                 return;
3717         }
3718
3719         /* When the HW is in SRIOV capable configuration, the PF-pool
3720          * resources are equally distributed across the max-number of
3721          * VFs. The user may request only a subset of the max-vfs to be
3722          * enabled. Based on num_vfs, redistribute the resources across
3723          * num_vfs so that each VF will have access to more number of
3724          * resources. This facility is not available in BE3 FW.
3725          * Also, this is done by FW in Lancer chip.
3726          */
3727         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3728                 status = be_cmd_set_sriov_config(adapter,
3729                                                  adapter->pool_res,
3730                                                  adapter->num_vfs);
3731                 if (status)
3732                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3733         }
3734 }
3735
3736 static int be_get_config(struct be_adapter *adapter)
3737 {
3738         u16 profile_id;
3739         int status;
3740
3741         status = be_cmd_query_fw_cfg(adapter);
3742         if (status)
3743                 return status;
3744
3745         be_cmd_query_port_name(adapter);
3746
3747         if (be_physfn(adapter)) {
3748                 status = be_cmd_get_active_profile(adapter, &profile_id);
3749                 if (!status)
3750                         dev_info(&adapter->pdev->dev,
3751                                  "Using profile 0x%x\n", profile_id);
3752         }
3753
3754         if (!BE2_chip(adapter) && be_physfn(adapter))
3755                 be_sriov_config(adapter);
3756
3757         status = be_get_resources(adapter);
3758         if (status)
3759                 return status;
3760
3761         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3762                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3763         if (!adapter->pmac_id)
3764                 return -ENOMEM;
3765
3766         /* Sanitize cfg_num_qs based on HW and platform limits */
3767         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3768
3769         return 0;
3770 }
3771
3772 static int be_mac_setup(struct be_adapter *adapter)
3773 {
3774         u8 mac[ETH_ALEN];
3775         int status;
3776
3777         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3778                 status = be_cmd_get_perm_mac(adapter, mac);
3779                 if (status)
3780                         return status;
3781
3782                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3783                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3784         } else {
3785                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3786                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3787         }
3788
3789         /* For BE3-R VFs, the PF programs the initial MAC address */
3790         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3791                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3792                                 &adapter->pmac_id[0], 0);
3793         return 0;
3794 }
3795
3796 static void be_schedule_worker(struct be_adapter *adapter)
3797 {
3798         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3799         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3800 }
3801
3802 static int be_setup_queues(struct be_adapter *adapter)
3803 {
3804         struct net_device *netdev = adapter->netdev;
3805         int status;
3806
3807         status = be_evt_queues_create(adapter);
3808         if (status)
3809                 goto err;
3810
3811         status = be_tx_qs_create(adapter);
3812         if (status)
3813                 goto err;
3814
3815         status = be_rx_cqs_create(adapter);
3816         if (status)
3817                 goto err;
3818
3819         status = be_mcc_queues_create(adapter);
3820         if (status)
3821                 goto err;
3822
3823         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3824         if (status)
3825                 goto err;
3826
3827         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3828         if (status)
3829                 goto err;
3830
3831         return 0;
3832 err:
3833         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3834         return status;
3835 }
3836
3837 int be_update_queues(struct be_adapter *adapter)
3838 {
3839         struct net_device *netdev = adapter->netdev;
3840         int status;
3841
3842         if (netif_running(netdev))
3843                 be_close(netdev);
3844
3845         be_cancel_worker(adapter);
3846
3847         /* If any vectors have been shared with RoCE we cannot re-program
3848          * the MSIx table.
3849          */
3850         if (!adapter->num_msix_roce_vec)
3851                 be_msix_disable(adapter);
3852
3853         be_clear_queues(adapter);
3854
3855         if (!msix_enabled(adapter)) {
3856                 status = be_msix_enable(adapter);
3857                 if (status)
3858                         return status;
3859         }
3860
3861         status = be_setup_queues(adapter);
3862         if (status)
3863                 return status;
3864
3865         be_schedule_worker(adapter);
3866
3867         if (netif_running(netdev))
3868                 status = be_open(netdev);
3869
3870         return status;
3871 }
3872
3873 static inline int fw_major_num(const char *fw_ver)
3874 {
3875         int fw_major = 0, i;
3876
3877         i = sscanf(fw_ver, "%d.", &fw_major);
3878         if (i != 1)
3879                 return 0;
3880
3881         return fw_major;
3882 }
3883
3884 static int be_setup(struct be_adapter *adapter)
3885 {
3886         struct device *dev = &adapter->pdev->dev;
3887         int status;
3888
3889         be_setup_init(adapter);
3890
3891         if (!lancer_chip(adapter))
3892                 be_cmd_req_native_mode(adapter);
3893
3894         status = be_get_config(adapter);
3895         if (status)
3896                 goto err;
3897
3898         status = be_msix_enable(adapter);
3899         if (status)
3900                 goto err;
3901
3902         status = be_if_create(adapter, &adapter->if_handle,
3903                               be_if_cap_flags(adapter), 0);
3904         if (status)
3905                 goto err;
3906
3907         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3908         rtnl_lock();
3909         status = be_setup_queues(adapter);
3910         rtnl_unlock();
3911         if (status)
3912                 goto err;
3913
3914         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3915
3916         status = be_mac_setup(adapter);
3917         if (status)
3918                 goto err;
3919
3920         be_cmd_get_fw_ver(adapter);
3921         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3922
3923         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3924                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3925                         adapter->fw_ver);
3926                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3927         }
3928
3929         if (adapter->vlans_added)
3930                 be_vid_config(adapter);
3931
3932         be_set_rx_mode(adapter->netdev);
3933
3934         be_cmd_get_acpi_wol_cap(adapter);
3935
3936         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3937                                          adapter->rx_fc);
3938         if (status)
3939                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3940                                         &adapter->rx_fc);
3941
3942         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3943                  adapter->tx_fc, adapter->rx_fc);
3944
3945         if (be_physfn(adapter))
3946                 be_cmd_set_logical_link_config(adapter,
3947                                                IFLA_VF_LINK_STATE_AUTO, 0);
3948
3949         if (adapter->num_vfs)
3950                 be_vf_setup(adapter);
3951
3952         status = be_cmd_get_phy_info(adapter);
3953         if (!status && be_pause_supported(adapter))
3954                 adapter->phy.fc_autoneg = 1;
3955
3956         be_schedule_worker(adapter);
3957         adapter->flags |= BE_FLAGS_SETUP_DONE;
3958         return 0;
3959 err:
3960         be_clear(adapter);
3961         return status;
3962 }
3963
3964 #ifdef CONFIG_NET_POLL_CONTROLLER
3965 static void be_netpoll(struct net_device *netdev)
3966 {
3967         struct be_adapter *adapter = netdev_priv(netdev);
3968         struct be_eq_obj *eqo;
3969         int i;
3970
3971         for_all_evt_queues(adapter, eqo, i) {
3972                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3973                 napi_schedule(&eqo->napi);
3974         }
3975 }
3976 #endif
3977
3978 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3979
3980 static bool phy_flashing_required(struct be_adapter *adapter)
3981 {
3982         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
3983                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3984 }
3985
3986 static bool is_comp_in_ufi(struct be_adapter *adapter,
3987                            struct flash_section_info *fsec, int type)
3988 {
3989         int i = 0, img_type = 0;
3990         struct flash_section_info_g2 *fsec_g2 = NULL;
3991
3992         if (BE2_chip(adapter))
3993                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3994
3995         for (i = 0; i < MAX_FLASH_COMP; i++) {
3996                 if (fsec_g2)
3997                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3998                 else
3999                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
4000
4001                 if (img_type == type)
4002                         return true;
4003         }
4004         return false;
4005
4006 }
4007
4008 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
4009                                                 int header_size,
4010                                                 const struct firmware *fw)
4011 {
4012         struct flash_section_info *fsec = NULL;
4013         const u8 *p = fw->data;
4014
4015         p += header_size;
4016         while (p < (fw->data + fw->size)) {
4017                 fsec = (struct flash_section_info *)p;
4018                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
4019                         return fsec;
4020                 p += 32;
4021         }
4022         return NULL;
4023 }
4024
4025 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
4026                               u32 img_offset, u32 img_size, int hdr_size,
4027                               u16 img_optype, bool *crc_match)
4028 {
4029         u32 crc_offset;
4030         int status;
4031         u8 crc[4];
4032
4033         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
4034                                       img_size - 4);
4035         if (status)
4036                 return status;
4037
4038         crc_offset = hdr_size + img_offset + img_size - 4;
4039
4040         /* Skip flashing, if crc of flashed region matches */
4041         if (!memcmp(crc, p + crc_offset, 4))
4042                 *crc_match = true;
4043         else
4044                 *crc_match = false;
4045
4046         return status;
4047 }
4048
4049 static int be_flash(struct be_adapter *adapter, const u8 *img,
4050                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4051                     u32 img_offset)
4052 {
4053         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4054         struct be_cmd_write_flashrom *req = flash_cmd->va;
4055         int status;
4056
4057         while (total_bytes) {
4058                 num_bytes = min_t(u32, 32*1024, total_bytes);
4059
4060                 total_bytes -= num_bytes;
4061
4062                 if (!total_bytes) {
4063                         if (optype == OPTYPE_PHY_FW)
4064                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4065                         else
4066                                 flash_op = FLASHROM_OPER_FLASH;
4067                 } else {
4068                         if (optype == OPTYPE_PHY_FW)
4069                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4070                         else
4071                                 flash_op = FLASHROM_OPER_SAVE;
4072                 }
4073
4074                 memcpy(req->data_buf, img, num_bytes);
4075                 img += num_bytes;
4076                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4077                                                flash_op, img_offset +
4078                                                bytes_sent, num_bytes);
4079                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4080                     optype == OPTYPE_PHY_FW)
4081                         break;
4082                 else if (status)
4083                         return status;
4084
4085                 bytes_sent += num_bytes;
4086         }
4087         return 0;
4088 }
4089
4090 /* For BE2, BE3 and BE3-R */
4091 static int be_flash_BEx(struct be_adapter *adapter,
4092                         const struct firmware *fw,
4093                         struct be_dma_mem *flash_cmd, int num_of_images)
4094 {
4095         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4096         struct device *dev = &adapter->pdev->dev;
4097         struct flash_section_info *fsec = NULL;
4098         int status, i, filehdr_size, num_comp;
4099         const struct flash_comp *pflashcomp;
4100         bool crc_match;
4101         const u8 *p;
4102
4103         struct flash_comp gen3_flash_types[] = {
4104                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4105                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4106                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4107                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4108                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4109                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4110                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4111                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4112                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4113                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4114                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4115                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4116                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4117                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4118                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4119                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4120                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4121                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4122                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4123                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4124         };
4125
4126         struct flash_comp gen2_flash_types[] = {
4127                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4128                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4129                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4130                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4131                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4132                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4133                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4134                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4135                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4136                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4137                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4138                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4139                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4140                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4141                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4142                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4143         };
4144
4145         if (BE3_chip(adapter)) {
4146                 pflashcomp = gen3_flash_types;
4147                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4148                 num_comp = ARRAY_SIZE(gen3_flash_types);
4149         } else {
4150                 pflashcomp = gen2_flash_types;
4151                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4152                 num_comp = ARRAY_SIZE(gen2_flash_types);
4153                 img_hdrs_size = 0;
4154         }
4155
4156         /* Get flash section info*/
4157         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4158         if (!fsec) {
4159                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4160                 return -1;
4161         }
4162         for (i = 0; i < num_comp; i++) {
4163                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4164                         continue;
4165
4166                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4167                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4168                         continue;
4169
4170                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4171                     !phy_flashing_required(adapter))
4172                                 continue;
4173
4174                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4175                         status = be_check_flash_crc(adapter, fw->data,
4176                                                     pflashcomp[i].offset,
4177                                                     pflashcomp[i].size,
4178                                                     filehdr_size +
4179                                                     img_hdrs_size,
4180                                                     OPTYPE_REDBOOT, &crc_match);
4181                         if (status) {
4182                                 dev_err(dev,
4183                                         "Could not get CRC for 0x%x region\n",
4184                                         pflashcomp[i].optype);
4185                                 continue;
4186                         }
4187
4188                         if (crc_match)
4189                                 continue;
4190                 }
4191
4192                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4193                         img_hdrs_size;
4194                 if (p + pflashcomp[i].size > fw->data + fw->size)
4195                         return -1;
4196
4197                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4198                                   pflashcomp[i].size, 0);
4199                 if (status) {
4200                         dev_err(dev, "Flashing section type 0x%x failed\n",
4201                                 pflashcomp[i].img_type);
4202                         return status;
4203                 }
4204         }
4205         return 0;
4206 }
4207
4208 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4209 {
4210         u32 img_type = le32_to_cpu(fsec_entry.type);
4211         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4212
4213         if (img_optype != 0xFFFF)
4214                 return img_optype;
4215
4216         switch (img_type) {
4217         case IMAGE_FIRMWARE_iSCSI:
4218                 img_optype = OPTYPE_ISCSI_ACTIVE;
4219                 break;
4220         case IMAGE_BOOT_CODE:
4221                 img_optype = OPTYPE_REDBOOT;
4222                 break;
4223         case IMAGE_OPTION_ROM_ISCSI:
4224                 img_optype = OPTYPE_BIOS;
4225                 break;
4226         case IMAGE_OPTION_ROM_PXE:
4227                 img_optype = OPTYPE_PXE_BIOS;
4228                 break;
4229         case IMAGE_OPTION_ROM_FCoE:
4230                 img_optype = OPTYPE_FCOE_BIOS;
4231                 break;
4232         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4233                 img_optype = OPTYPE_ISCSI_BACKUP;
4234                 break;
4235         case IMAGE_NCSI:
4236                 img_optype = OPTYPE_NCSI_FW;
4237                 break;
4238         case IMAGE_FLASHISM_JUMPVECTOR:
4239                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4240                 break;
4241         case IMAGE_FIRMWARE_PHY:
4242                 img_optype = OPTYPE_SH_PHY_FW;
4243                 break;
4244         case IMAGE_REDBOOT_DIR:
4245                 img_optype = OPTYPE_REDBOOT_DIR;
4246                 break;
4247         case IMAGE_REDBOOT_CONFIG:
4248                 img_optype = OPTYPE_REDBOOT_CONFIG;
4249                 break;
4250         case IMAGE_UFI_DIR:
4251                 img_optype = OPTYPE_UFI_DIR;
4252                 break;
4253         default:
4254                 break;
4255         }
4256
4257         return img_optype;
4258 }
4259
4260 static int be_flash_skyhawk(struct be_adapter *adapter,
4261                             const struct firmware *fw,
4262                             struct be_dma_mem *flash_cmd, int num_of_images)
4263 {
4264         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4265         bool crc_match, old_fw_img, flash_offset_support = true;
4266         struct device *dev = &adapter->pdev->dev;
4267         struct flash_section_info *fsec = NULL;
4268         u32 img_offset, img_size, img_type;
4269         u16 img_optype, flash_optype;
4270         int status, i, filehdr_size;
4271         const u8 *p;
4272
4273         filehdr_size = sizeof(struct flash_file_hdr_g3);
4274         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4275         if (!fsec) {
4276                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4277                 return -EINVAL;
4278         }
4279
4280 retry_flash:
4281         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4282                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4283                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4284                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4285                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4286                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4287
4288                 if (img_optype == 0xFFFF)
4289                         continue;
4290
4291                 if (flash_offset_support)
4292                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4293                 else
4294                         flash_optype = img_optype;
4295
4296                 /* Don't bother verifying CRC if an old FW image is being
4297                  * flashed
4298                  */
4299                 if (old_fw_img)
4300                         goto flash;
4301
4302                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4303                                             img_size, filehdr_size +
4304                                             img_hdrs_size, flash_optype,
4305                                             &crc_match);
4306                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4307                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4308                         /* The current FW image on the card does not support
4309                          * OFFSET based flashing. Retry using older mechanism
4310                          * of OPTYPE based flashing
4311                          */
4312                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4313                                 flash_offset_support = false;
4314                                 goto retry_flash;
4315                         }
4316
4317                         /* The current FW image on the card does not recognize
4318                          * the new FLASH op_type. The FW download is partially
4319                          * complete. Reboot the server now to enable FW image
4320                          * to recognize the new FLASH op_type. To complete the
4321                          * remaining process, download the same FW again after
4322                          * the reboot.
4323                          */
4324                         dev_err(dev, "Flash incomplete. Reset the server\n");
4325                         dev_err(dev, "Download FW image again after reset\n");
4326                         return -EAGAIN;
4327                 } else if (status) {
4328                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4329                                 img_optype);
4330                         return -EFAULT;
4331                 }
4332
4333                 if (crc_match)
4334                         continue;
4335
4336 flash:
4337                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4338                 if (p + img_size > fw->data + fw->size)
4339                         return -1;
4340
4341                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4342                                   img_offset);
4343
4344                 /* The current FW image on the card does not support OFFSET
4345                  * based flashing. Retry using older mechanism of OPTYPE based
4346                  * flashing
4347                  */
4348                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4349                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4350                         flash_offset_support = false;
4351                         goto retry_flash;
4352                 }
4353
4354                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4355                  * UFI_DIR region
4356                  */
4357                 if (old_fw_img &&
4358                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4359                      (img_optype == OPTYPE_UFI_DIR &&
4360                       base_status(status) == MCC_STATUS_FAILED))) {
4361                         continue;
4362                 } else if (status) {
4363                         dev_err(dev, "Flashing section type 0x%x failed\n",
4364                                 img_type);
4365                         return -EFAULT;
4366                 }
4367         }
4368         return 0;
4369 }
4370
4371 static int lancer_fw_download(struct be_adapter *adapter,
4372                               const struct firmware *fw)
4373 {
4374 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4375 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4376         struct device *dev = &adapter->pdev->dev;
4377         struct be_dma_mem flash_cmd;
4378         const u8 *data_ptr = NULL;
4379         u8 *dest_image_ptr = NULL;
4380         size_t image_size = 0;
4381         u32 chunk_size = 0;
4382         u32 data_written = 0;
4383         u32 offset = 0;
4384         int status = 0;
4385         u8 add_status = 0;
4386         u8 change_status;
4387
4388         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4389                 dev_err(dev, "FW image size should be multiple of 4\n");
4390                 return -EINVAL;
4391         }
4392
4393         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4394                                 + LANCER_FW_DOWNLOAD_CHUNK;
4395         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4396                                           &flash_cmd.dma, GFP_KERNEL);
4397         if (!flash_cmd.va)
4398                 return -ENOMEM;
4399
4400         dest_image_ptr = flash_cmd.va +
4401                                 sizeof(struct lancer_cmd_req_write_object);
4402         image_size = fw->size;
4403         data_ptr = fw->data;
4404
4405         while (image_size) {
4406                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4407
4408                 /* Copy the image chunk content. */
4409                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4410
4411                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4412                                                  chunk_size, offset,
4413                                                  LANCER_FW_DOWNLOAD_LOCATION,
4414                                                  &data_written, &change_status,
4415                                                  &add_status);
4416                 if (status)
4417                         break;
4418
4419                 offset += data_written;
4420                 data_ptr += data_written;
4421                 image_size -= data_written;
4422         }
4423
4424         if (!status) {
4425                 /* Commit the FW written */
4426                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4427                                                  0, offset,
4428                                                  LANCER_FW_DOWNLOAD_LOCATION,
4429                                                  &data_written, &change_status,
4430                                                  &add_status);
4431         }
4432
4433         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4434         if (status) {
4435                 dev_err(dev, "Firmware load error\n");
4436                 return be_cmd_status(status);
4437         }
4438
4439         dev_info(dev, "Firmware flashed successfully\n");
4440
4441         if (change_status == LANCER_FW_RESET_NEEDED) {
4442                 dev_info(dev, "Resetting adapter to activate new FW\n");
4443                 status = lancer_physdev_ctrl(adapter,
4444                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4445                 if (status) {
4446                         dev_err(dev, "Adapter busy, could not reset FW\n");
4447                         dev_err(dev, "Reboot server to activate new FW\n");
4448                 }
4449         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4450                 dev_info(dev, "Reboot server to activate new FW\n");
4451         }
4452
4453         return 0;
4454 }
4455
4456 #define BE2_UFI         2
4457 #define BE3_UFI         3
4458 #define BE3R_UFI        10
4459 #define SH_UFI          4
4460 #define SH_P2_UFI       11
4461
4462 static int be_get_ufi_type(struct be_adapter *adapter,
4463                            struct flash_file_hdr_g3 *fhdr)
4464 {
4465         if (!fhdr) {
4466                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4467                 return -1;
4468         }
4469
4470         /* First letter of the build version is used to identify
4471          * which chip this image file is meant for.
4472          */
4473         switch (fhdr->build[0]) {
4474         case BLD_STR_UFI_TYPE_SH:
4475                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4476                                                                 SH_UFI;
4477         case BLD_STR_UFI_TYPE_BE3:
4478                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4479                                                                 BE3_UFI;
4480         case BLD_STR_UFI_TYPE_BE2:
4481                 return BE2_UFI;
4482         default:
4483                 return -1;
4484         }
4485 }
4486
4487 /* Check if the flash image file is compatible with the adapter that
4488  * is being flashed.
4489  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4490  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4491  */
4492 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4493                                        struct flash_file_hdr_g3 *fhdr)
4494 {
4495         int ufi_type = be_get_ufi_type(adapter, fhdr);
4496
4497         switch (ufi_type) {
4498         case SH_P2_UFI:
4499                 return skyhawk_chip(adapter);
4500         case SH_UFI:
4501                 return (skyhawk_chip(adapter) &&
4502                         adapter->asic_rev < ASIC_REV_P2);
4503         case BE3R_UFI:
4504                 return BE3_chip(adapter);
4505         case BE3_UFI:
4506                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4507         case BE2_UFI:
4508                 return BE2_chip(adapter);
4509         default:
4510                 return false;
4511         }
4512 }
4513
4514 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4515 {
4516         struct device *dev = &adapter->pdev->dev;
4517         struct flash_file_hdr_g3 *fhdr3;
4518         struct image_hdr *img_hdr_ptr;
4519         int status = 0, i, num_imgs;
4520         struct be_dma_mem flash_cmd;
4521
4522         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4523         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4524                 dev_err(dev, "Flash image is not compatible with adapter\n");
4525                 return -EINVAL;
4526         }
4527
4528         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4529         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4530                                           GFP_KERNEL);
4531         if (!flash_cmd.va)
4532                 return -ENOMEM;
4533
4534         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4535         for (i = 0; i < num_imgs; i++) {
4536                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4537                                 (sizeof(struct flash_file_hdr_g3) +
4538                                  i * sizeof(struct image_hdr)));
4539                 if (!BE2_chip(adapter) &&
4540                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4541                         continue;
4542
4543                 if (skyhawk_chip(adapter))
4544                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4545                                                   num_imgs);
4546                 else
4547                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4548                                               num_imgs);
4549         }
4550
4551         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4552         if (!status)
4553                 dev_info(dev, "Firmware flashed successfully\n");
4554
4555         return status;
4556 }
4557
4558 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4559 {
4560         const struct firmware *fw;
4561         int status;
4562
4563         if (!netif_running(adapter->netdev)) {
4564                 dev_err(&adapter->pdev->dev,
4565                         "Firmware load not allowed (interface is down)\n");
4566                 return -ENETDOWN;
4567         }
4568
4569         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4570         if (status)
4571                 goto fw_exit;
4572
4573         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4574
4575         if (lancer_chip(adapter))
4576                 status = lancer_fw_download(adapter, fw);
4577         else
4578                 status = be_fw_download(adapter, fw);
4579
4580         if (!status)
4581                 be_cmd_get_fw_ver(adapter);
4582
4583 fw_exit:
4584         release_firmware(fw);
4585         return status;
4586 }
4587
4588 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4589                                  u16 flags)
4590 {
4591         struct be_adapter *adapter = netdev_priv(dev);
4592         struct nlattr *attr, *br_spec;
4593         int rem;
4594         int status = 0;
4595         u16 mode = 0;
4596
4597         if (!sriov_enabled(adapter))
4598                 return -EOPNOTSUPP;
4599
4600         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4601         if (!br_spec)
4602                 return -EINVAL;
4603
4604         nla_for_each_nested(attr, br_spec, rem) {
4605                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4606                         continue;
4607
4608                 if (nla_len(attr) < sizeof(mode))
4609                         return -EINVAL;
4610
4611                 mode = nla_get_u16(attr);
4612                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4613                         return -EINVAL;
4614
4615                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4616                                                adapter->if_handle,
4617                                                mode == BRIDGE_MODE_VEPA ?
4618                                                PORT_FWD_TYPE_VEPA :
4619                                                PORT_FWD_TYPE_VEB);
4620                 if (status)
4621                         goto err;
4622
4623                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4624                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4625
4626                 return status;
4627         }
4628 err:
4629         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4630                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4631
4632         return status;
4633 }
4634
4635 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4636                                  struct net_device *dev, u32 filter_mask)
4637 {
4638         struct be_adapter *adapter = netdev_priv(dev);
4639         int status = 0;
4640         u8 hsw_mode;
4641
4642         if (!sriov_enabled(adapter))
4643                 return 0;
4644
4645         /* BE and Lancer chips support VEB mode only */
4646         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4647                 hsw_mode = PORT_FWD_TYPE_VEB;
4648         } else {
4649                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4650                                                adapter->if_handle, &hsw_mode);
4651                 if (status)
4652                         return 0;
4653         }
4654
4655         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4656                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4657                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4658                                        0, 0);
4659 }
4660
4661 #ifdef CONFIG_BE2NET_VXLAN
4662 /* VxLAN offload Notes:
4663  *
4664  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4665  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4666  * is expected to work across all types of IP tunnels once exported. Skyhawk
4667  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4668  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4669  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4670  * those other tunnels are unexported on the fly through ndo_features_check().
4671  *
4672  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4673  * adds more than one port, disable offloads and don't re-enable them again
4674  * until after all the tunnels are removed.
4675  */
4676 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4677                               __be16 port)
4678 {
4679         struct be_adapter *adapter = netdev_priv(netdev);
4680         struct device *dev = &adapter->pdev->dev;
4681         int status;
4682
4683         if (lancer_chip(adapter) || BEx_chip(adapter))
4684                 return;
4685
4686         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4687                 dev_info(dev,
4688                          "Only one UDP port supported for VxLAN offloads\n");
4689                 dev_info(dev, "Disabling VxLAN offloads\n");
4690                 adapter->vxlan_port_count++;
4691                 goto err;
4692         }
4693
4694         if (adapter->vxlan_port_count++ >= 1)
4695                 return;
4696
4697         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4698                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4699         if (status) {
4700                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4701                 goto err;
4702         }
4703
4704         status = be_cmd_set_vxlan_port(adapter, port);
4705         if (status) {
4706                 dev_warn(dev, "Failed to add VxLAN port\n");
4707                 goto err;
4708         }
4709         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4710         adapter->vxlan_port = port;
4711
4712         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4713                                    NETIF_F_TSO | NETIF_F_TSO6 |
4714                                    NETIF_F_GSO_UDP_TUNNEL;
4715         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4716         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4717
4718         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4719                  be16_to_cpu(port));
4720         return;
4721 err:
4722         be_disable_vxlan_offloads(adapter);
4723 }
4724
4725 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4726                               __be16 port)
4727 {
4728         struct be_adapter *adapter = netdev_priv(netdev);
4729
4730         if (lancer_chip(adapter) || BEx_chip(adapter))
4731                 return;
4732
4733         if (adapter->vxlan_port != port)
4734                 goto done;
4735
4736         be_disable_vxlan_offloads(adapter);
4737
4738         dev_info(&adapter->pdev->dev,
4739                  "Disabled VxLAN offloads for UDP port %d\n",
4740                  be16_to_cpu(port));
4741 done:
4742         adapter->vxlan_port_count--;
4743 }
4744
4745 static netdev_features_t be_features_check(struct sk_buff *skb,
4746                                            struct net_device *dev,
4747                                            netdev_features_t features)
4748 {
4749         struct be_adapter *adapter = netdev_priv(dev);
4750         u8 l4_hdr = 0;
4751
4752         /* The code below restricts offload features for some tunneled packets.
4753          * Offload features for normal (non tunnel) packets are unchanged.
4754          */
4755         if (!skb->encapsulation ||
4756             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4757                 return features;
4758
4759         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4760          * should disable tunnel offload features if it's not a VxLAN packet,
4761          * as tunnel offloads have been enabled only for VxLAN. This is done to
4762          * allow other tunneled traffic like GRE work fine while VxLAN
4763          * offloads are configured in Skyhawk-R.
4764          */
4765         switch (vlan_get_protocol(skb)) {
4766         case htons(ETH_P_IP):
4767                 l4_hdr = ip_hdr(skb)->protocol;
4768                 break;
4769         case htons(ETH_P_IPV6):
4770                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4771                 break;
4772         default:
4773                 return features;
4774         }
4775
4776         if (l4_hdr != IPPROTO_UDP ||
4777             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4778             skb->inner_protocol != htons(ETH_P_TEB) ||
4779             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4780             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4781                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4782
4783         return features;
4784 }
4785 #endif
4786
4787 static const struct net_device_ops be_netdev_ops = {
4788         .ndo_open               = be_open,
4789         .ndo_stop               = be_close,
4790         .ndo_start_xmit         = be_xmit,
4791         .ndo_set_rx_mode        = be_set_rx_mode,
4792         .ndo_set_mac_address    = be_mac_addr_set,
4793         .ndo_change_mtu         = be_change_mtu,
4794         .ndo_get_stats64        = be_get_stats64,
4795         .ndo_validate_addr      = eth_validate_addr,
4796         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4797         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4798         .ndo_set_vf_mac         = be_set_vf_mac,
4799         .ndo_set_vf_vlan        = be_set_vf_vlan,
4800         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4801         .ndo_get_vf_config      = be_get_vf_config,
4802         .ndo_set_vf_link_state  = be_set_vf_link_state,
4803 #ifdef CONFIG_NET_POLL_CONTROLLER
4804         .ndo_poll_controller    = be_netpoll,
4805 #endif
4806         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4807         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4808 #ifdef CONFIG_NET_RX_BUSY_POLL
4809         .ndo_busy_poll          = be_busy_poll,
4810 #endif
4811 #ifdef CONFIG_BE2NET_VXLAN
4812         .ndo_add_vxlan_port     = be_add_vxlan_port,
4813         .ndo_del_vxlan_port     = be_del_vxlan_port,
4814         .ndo_features_check     = be_features_check,
4815 #endif
4816 };
4817
4818 static void be_netdev_init(struct net_device *netdev)
4819 {
4820         struct be_adapter *adapter = netdev_priv(netdev);
4821
4822         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4823                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4824                 NETIF_F_HW_VLAN_CTAG_TX;
4825         if (be_multi_rxq(adapter))
4826                 netdev->hw_features |= NETIF_F_RXHASH;
4827
4828         netdev->features |= netdev->hw_features |
4829                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4830
4831         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4832                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4833
4834         netdev->priv_flags |= IFF_UNICAST_FLT;
4835
4836         netdev->flags |= IFF_MULTICAST;
4837
4838         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4839
4840         netdev->netdev_ops = &be_netdev_ops;
4841
4842         netdev->ethtool_ops = &be_ethtool_ops;
4843 }
4844
4845 static void be_unmap_pci_bars(struct be_adapter *adapter)
4846 {
4847         if (adapter->csr)
4848                 pci_iounmap(adapter->pdev, adapter->csr);
4849         if (adapter->db)
4850                 pci_iounmap(adapter->pdev, adapter->db);
4851 }
4852
4853 static int db_bar(struct be_adapter *adapter)
4854 {
4855         if (lancer_chip(adapter) || !be_physfn(adapter))
4856                 return 0;
4857         else
4858                 return 4;
4859 }
4860
4861 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4862 {
4863         if (skyhawk_chip(adapter)) {
4864                 adapter->roce_db.size = 4096;
4865                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4866                                                               db_bar(adapter));
4867                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4868                                                                db_bar(adapter));
4869         }
4870         return 0;
4871 }
4872
4873 static int be_map_pci_bars(struct be_adapter *adapter)
4874 {
4875         struct pci_dev *pdev = adapter->pdev;
4876         u8 __iomem *addr;
4877
4878         if (BEx_chip(adapter) && be_physfn(adapter)) {
4879                 adapter->csr = pci_iomap(pdev, 2, 0);
4880                 if (!adapter->csr)
4881                         return -ENOMEM;
4882         }
4883
4884         addr = pci_iomap(pdev, db_bar(adapter), 0);
4885         if (!addr)
4886                 goto pci_map_err;
4887         adapter->db = addr;
4888
4889         if (skyhawk_chip(adapter) || BEx_chip(adapter)) {
4890                 if (be_physfn(adapter)) {
4891                         /* PCICFG is the 2nd BAR in BE2 */
4892                         addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0);
4893                         if (!addr)
4894                                 goto pci_map_err;
4895                         adapter->pcicfg = addr;
4896                 } else {
4897                         adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
4898                 }
4899         }
4900
4901         be_roce_map_pci_bars(adapter);
4902         return 0;
4903
4904 pci_map_err:
4905         dev_err(&pdev->dev, "Error in mapping PCI BARs\n");
4906         be_unmap_pci_bars(adapter);
4907         return -ENOMEM;
4908 }
4909
4910 static void be_ctrl_cleanup(struct be_adapter *adapter)
4911 {
4912         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4913
4914         be_unmap_pci_bars(adapter);
4915
4916         if (mem->va)
4917                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4918                                   mem->dma);
4919
4920         mem = &adapter->rx_filter;
4921         if (mem->va)
4922                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4923                                   mem->dma);
4924 }
4925
4926 static int be_ctrl_init(struct be_adapter *adapter)
4927 {
4928         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4929         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4930         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4931         u32 sli_intf;
4932         int status;
4933
4934         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4935         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4936                                  SLI_INTF_FAMILY_SHIFT;
4937         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4938
4939         status = be_map_pci_bars(adapter);
4940         if (status)
4941                 goto done;
4942
4943         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4944         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4945                                                 mbox_mem_alloc->size,
4946                                                 &mbox_mem_alloc->dma,
4947                                                 GFP_KERNEL);
4948         if (!mbox_mem_alloc->va) {
4949                 status = -ENOMEM;
4950                 goto unmap_pci_bars;
4951         }
4952         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4953         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4954         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4955         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4956
4957         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4958         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4959                                             rx_filter->size, &rx_filter->dma,
4960                                             GFP_KERNEL);
4961         if (!rx_filter->va) {
4962                 status = -ENOMEM;
4963                 goto free_mbox;
4964         }
4965
4966         mutex_init(&adapter->mbox_lock);
4967         spin_lock_init(&adapter->mcc_lock);
4968         spin_lock_init(&adapter->mcc_cq_lock);
4969
4970         init_completion(&adapter->et_cmd_compl);
4971         pci_save_state(adapter->pdev);
4972         return 0;
4973
4974 free_mbox:
4975         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4976                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4977
4978 unmap_pci_bars:
4979         be_unmap_pci_bars(adapter);
4980
4981 done:
4982         return status;
4983 }
4984
4985 static void be_stats_cleanup(struct be_adapter *adapter)
4986 {
4987         struct be_dma_mem *cmd = &adapter->stats_cmd;
4988
4989         if (cmd->va)
4990                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4991                                   cmd->va, cmd->dma);
4992 }
4993
4994 static int be_stats_init(struct be_adapter *adapter)
4995 {
4996         struct be_dma_mem *cmd = &adapter->stats_cmd;
4997
4998         if (lancer_chip(adapter))
4999                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5000         else if (BE2_chip(adapter))
5001                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5002         else if (BE3_chip(adapter))
5003                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5004         else
5005                 /* ALL non-BE ASICs */
5006                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5007
5008         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
5009                                       GFP_KERNEL);
5010         if (!cmd->va)
5011                 return -ENOMEM;
5012         return 0;
5013 }
5014
5015 static void be_remove(struct pci_dev *pdev)
5016 {
5017         struct be_adapter *adapter = pci_get_drvdata(pdev);
5018
5019         if (!adapter)
5020                 return;
5021
5022         be_roce_dev_remove(adapter);
5023         be_intr_set(adapter, false);
5024
5025         cancel_delayed_work_sync(&adapter->func_recovery_work);
5026
5027         unregister_netdev(adapter->netdev);
5028
5029         be_clear(adapter);
5030
5031         /* tell fw we're done with firing cmds */
5032         be_cmd_fw_clean(adapter);
5033
5034         be_stats_cleanup(adapter);
5035
5036         be_ctrl_cleanup(adapter);
5037
5038         pci_disable_pcie_error_reporting(pdev);
5039
5040         pci_release_regions(pdev);
5041         pci_disable_device(pdev);
5042
5043         free_netdev(adapter->netdev);
5044 }
5045
5046 static int be_get_initial_config(struct be_adapter *adapter)
5047 {
5048         int status, level;
5049
5050         status = be_cmd_get_cntl_attributes(adapter);
5051         if (status)
5052                 return status;
5053
5054         /* Must be a power of 2 or else MODULO will BUG_ON */
5055         adapter->be_get_temp_freq = 64;
5056
5057         if (BEx_chip(adapter)) {
5058                 level = be_cmd_get_fw_log_level(adapter);
5059                 adapter->msg_enable =
5060                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
5061         }
5062
5063         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
5064         return 0;
5065 }
5066
5067 static int lancer_recover_func(struct be_adapter *adapter)
5068 {
5069         struct device *dev = &adapter->pdev->dev;
5070         int status;
5071
5072         status = lancer_test_and_set_rdy_state(adapter);
5073         if (status)
5074                 goto err;
5075
5076         if (netif_running(adapter->netdev))
5077                 be_close(adapter->netdev);
5078
5079         be_clear(adapter);
5080
5081         be_clear_all_error(adapter);
5082
5083         status = be_setup(adapter);
5084         if (status)
5085                 goto err;
5086
5087         if (netif_running(adapter->netdev)) {
5088                 status = be_open(adapter->netdev);
5089                 if (status)
5090                         goto err;
5091         }
5092
5093         dev_err(dev, "Adapter recovery successful\n");
5094         return 0;
5095 err:
5096         if (status == -EAGAIN)
5097                 dev_err(dev, "Waiting for resource provisioning\n");
5098         else
5099                 dev_err(dev, "Adapter recovery failed\n");
5100
5101         return status;
5102 }
5103
5104 static void be_func_recovery_task(struct work_struct *work)
5105 {
5106         struct be_adapter *adapter =
5107                 container_of(work, struct be_adapter,  func_recovery_work.work);
5108         int status = 0;
5109
5110         be_detect_error(adapter);
5111
5112         if (adapter->hw_error && lancer_chip(adapter)) {
5113                 rtnl_lock();
5114                 netif_device_detach(adapter->netdev);
5115                 rtnl_unlock();
5116
5117                 status = lancer_recover_func(adapter);
5118                 if (!status)
5119                         netif_device_attach(adapter->netdev);
5120         }
5121
5122         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
5123          * no need to attempt further recovery.
5124          */
5125         if (!status || status == -EAGAIN)
5126                 schedule_delayed_work(&adapter->func_recovery_work,
5127                                       msecs_to_jiffies(1000));
5128 }
5129
5130 static void be_log_sfp_info(struct be_adapter *adapter)
5131 {
5132         int status;
5133
5134         status = be_cmd_query_sfp_info(adapter);
5135         if (!status) {
5136                 dev_err(&adapter->pdev->dev,
5137                         "Unqualified SFP+ detected on %c from %s part no: %s",
5138                         adapter->port_name, adapter->phy.vendor_name,
5139                         adapter->phy.vendor_pn);
5140         }
5141         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5142 }
5143
5144 static void be_worker(struct work_struct *work)
5145 {
5146         struct be_adapter *adapter =
5147                 container_of(work, struct be_adapter, work.work);
5148         struct be_rx_obj *rxo;
5149         int i;
5150
5151         /* when interrupts are not yet enabled, just reap any pending
5152         * mcc completions */
5153         if (!netif_running(adapter->netdev)) {
5154                 local_bh_disable();
5155                 be_process_mcc(adapter);
5156                 local_bh_enable();
5157                 goto reschedule;
5158         }
5159
5160         if (!adapter->stats_cmd_sent) {
5161                 if (lancer_chip(adapter))
5162                         lancer_cmd_get_pport_stats(adapter,
5163                                                    &adapter->stats_cmd);
5164                 else
5165                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5166         }
5167
5168         if (be_physfn(adapter) &&
5169             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5170                 be_cmd_get_die_temperature(adapter);
5171
5172         for_all_rx_queues(adapter, rxo, i) {
5173                 /* Replenish RX-queues starved due to memory
5174                  * allocation failures.
5175                  */
5176                 if (rxo->rx_post_starved)
5177                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5178         }
5179
5180         be_eqd_update(adapter);
5181
5182         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5183                 be_log_sfp_info(adapter);
5184
5185 reschedule:
5186         adapter->work_counter++;
5187         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5188 }
5189
5190 /* If any VFs are already enabled don't FLR the PF */
5191 static bool be_reset_required(struct be_adapter *adapter)
5192 {
5193         return pci_num_vf(adapter->pdev) ? false : true;
5194 }
5195
5196 static char *mc_name(struct be_adapter *adapter)
5197 {
5198         char *str = ""; /* default */
5199
5200         switch (adapter->mc_type) {
5201         case UMC:
5202                 str = "UMC";
5203                 break;
5204         case FLEX10:
5205                 str = "FLEX10";
5206                 break;
5207         case vNIC1:
5208                 str = "vNIC-1";
5209                 break;
5210         case nPAR:
5211                 str = "nPAR";
5212                 break;
5213         case UFP:
5214                 str = "UFP";
5215                 break;
5216         case vNIC2:
5217                 str = "vNIC-2";
5218                 break;
5219         default:
5220                 str = "";
5221         }
5222
5223         return str;
5224 }
5225
5226 static inline char *func_name(struct be_adapter *adapter)
5227 {
5228         return be_physfn(adapter) ? "PF" : "VF";
5229 }
5230
5231 static inline char *nic_name(struct pci_dev *pdev)
5232 {
5233         switch (pdev->device) {
5234         case OC_DEVICE_ID1:
5235                 return OC_NAME;
5236         case OC_DEVICE_ID2:
5237                 return OC_NAME_BE;
5238         case OC_DEVICE_ID3:
5239         case OC_DEVICE_ID4:
5240                 return OC_NAME_LANCER;
5241         case BE_DEVICE_ID2:
5242                 return BE3_NAME;
5243         case OC_DEVICE_ID5:
5244         case OC_DEVICE_ID6:
5245                 return OC_NAME_SH;
5246         default:
5247                 return BE_NAME;
5248         }
5249 }
5250
5251 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5252 {
5253         struct be_adapter *adapter;
5254         struct net_device *netdev;
5255         int status = 0;
5256
5257         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5258
5259         status = pci_enable_device(pdev);
5260         if (status)
5261                 goto do_none;
5262
5263         status = pci_request_regions(pdev, DRV_NAME);
5264         if (status)
5265                 goto disable_dev;
5266         pci_set_master(pdev);
5267
5268         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5269         if (!netdev) {
5270                 status = -ENOMEM;
5271                 goto rel_reg;
5272         }
5273         adapter = netdev_priv(netdev);
5274         adapter->pdev = pdev;
5275         pci_set_drvdata(pdev, adapter);
5276         adapter->netdev = netdev;
5277         SET_NETDEV_DEV(netdev, &pdev->dev);
5278
5279         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5280         if (!status) {
5281                 netdev->features |= NETIF_F_HIGHDMA;
5282         } else {
5283                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5284                 if (status) {
5285                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5286                         goto free_netdev;
5287                 }
5288         }
5289
5290         status = pci_enable_pcie_error_reporting(pdev);
5291         if (!status)
5292                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5293
5294         status = be_ctrl_init(adapter);
5295         if (status)
5296                 goto free_netdev;
5297
5298         /* sync up with fw's ready state */
5299         if (be_physfn(adapter)) {
5300                 status = be_fw_wait_ready(adapter);
5301                 if (status)
5302                         goto ctrl_clean;
5303         }
5304
5305         if (be_reset_required(adapter)) {
5306                 status = be_cmd_reset_function(adapter);
5307                 if (status)
5308                         goto ctrl_clean;
5309
5310                 /* Wait for interrupts to quiesce after an FLR */
5311                 msleep(100);
5312         }
5313
5314         /* Allow interrupts for other ULPs running on NIC function */
5315         be_intr_set(adapter, true);
5316
5317         /* tell fw we're ready to fire cmds */
5318         status = be_cmd_fw_init(adapter);
5319         if (status)
5320                 goto ctrl_clean;
5321
5322         status = be_stats_init(adapter);
5323         if (status)
5324                 goto ctrl_clean;
5325
5326         status = be_get_initial_config(adapter);
5327         if (status)
5328                 goto stats_clean;
5329
5330         INIT_DELAYED_WORK(&adapter->work, be_worker);
5331         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5332         adapter->rx_fc = true;
5333         adapter->tx_fc = true;
5334
5335         status = be_setup(adapter);
5336         if (status)
5337                 goto stats_clean;
5338
5339         be_netdev_init(netdev);
5340         status = register_netdev(netdev);
5341         if (status != 0)
5342                 goto unsetup;
5343
5344         be_roce_dev_add(adapter);
5345
5346         schedule_delayed_work(&adapter->func_recovery_work,
5347                               msecs_to_jiffies(1000));
5348
5349         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5350                  func_name(adapter), mc_name(adapter), adapter->port_name);
5351
5352         return 0;
5353
5354 unsetup:
5355         be_clear(adapter);
5356 stats_clean:
5357         be_stats_cleanup(adapter);
5358 ctrl_clean:
5359         be_ctrl_cleanup(adapter);
5360 free_netdev:
5361         free_netdev(netdev);
5362 rel_reg:
5363         pci_release_regions(pdev);
5364 disable_dev:
5365         pci_disable_device(pdev);
5366 do_none:
5367         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5368         return status;
5369 }
5370
5371 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5372 {
5373         struct be_adapter *adapter = pci_get_drvdata(pdev);
5374         struct net_device *netdev =  adapter->netdev;
5375
5376         if (adapter->wol_en)
5377                 be_setup_wol(adapter, true);
5378
5379         be_intr_set(adapter, false);
5380         cancel_delayed_work_sync(&adapter->func_recovery_work);
5381
5382         netif_device_detach(netdev);
5383         if (netif_running(netdev)) {
5384                 rtnl_lock();
5385                 be_close(netdev);
5386                 rtnl_unlock();
5387         }
5388         be_clear(adapter);
5389
5390         pci_save_state(pdev);
5391         pci_disable_device(pdev);
5392         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5393         return 0;
5394 }
5395
5396 static int be_resume(struct pci_dev *pdev)
5397 {
5398         int status = 0;
5399         struct be_adapter *adapter = pci_get_drvdata(pdev);
5400         struct net_device *netdev =  adapter->netdev;
5401
5402         netif_device_detach(netdev);
5403
5404         status = pci_enable_device(pdev);
5405         if (status)
5406                 return status;
5407
5408         pci_set_power_state(pdev, PCI_D0);
5409         pci_restore_state(pdev);
5410
5411         status = be_fw_wait_ready(adapter);
5412         if (status)
5413                 return status;
5414
5415         status = be_cmd_reset_function(adapter);
5416         if (status)
5417                 return status;
5418
5419         be_intr_set(adapter, true);
5420         /* tell fw we're ready to fire cmds */
5421         status = be_cmd_fw_init(adapter);
5422         if (status)
5423                 return status;
5424
5425         be_setup(adapter);
5426         if (netif_running(netdev)) {
5427                 rtnl_lock();
5428                 be_open(netdev);
5429                 rtnl_unlock();
5430         }
5431
5432         schedule_delayed_work(&adapter->func_recovery_work,
5433                               msecs_to_jiffies(1000));
5434         netif_device_attach(netdev);
5435
5436         if (adapter->wol_en)
5437                 be_setup_wol(adapter, false);
5438
5439         return 0;
5440 }
5441
5442 /*
5443  * An FLR will stop BE from DMAing any data.
5444  */
5445 static void be_shutdown(struct pci_dev *pdev)
5446 {
5447         struct be_adapter *adapter = pci_get_drvdata(pdev);
5448
5449         if (!adapter)
5450                 return;
5451
5452         be_roce_dev_shutdown(adapter);
5453         cancel_delayed_work_sync(&adapter->work);
5454         cancel_delayed_work_sync(&adapter->func_recovery_work);
5455
5456         netif_device_detach(adapter->netdev);
5457
5458         be_cmd_reset_function(adapter);
5459
5460         pci_disable_device(pdev);
5461 }
5462
5463 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5464                                             pci_channel_state_t state)
5465 {
5466         struct be_adapter *adapter = pci_get_drvdata(pdev);
5467         struct net_device *netdev =  adapter->netdev;
5468
5469         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5470
5471         if (!adapter->eeh_error) {
5472                 adapter->eeh_error = true;
5473
5474                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5475
5476                 rtnl_lock();
5477                 netif_device_detach(netdev);
5478                 if (netif_running(netdev))
5479                         be_close(netdev);
5480                 rtnl_unlock();
5481
5482                 be_clear(adapter);
5483         }
5484
5485         if (state == pci_channel_io_perm_failure)
5486                 return PCI_ERS_RESULT_DISCONNECT;
5487
5488         pci_disable_device(pdev);
5489
5490         /* The error could cause the FW to trigger a flash debug dump.
5491          * Resetting the card while flash dump is in progress
5492          * can cause it not to recover; wait for it to finish.
5493          * Wait only for first function as it is needed only once per
5494          * adapter.
5495          */
5496         if (pdev->devfn == 0)
5497                 ssleep(30);
5498
5499         return PCI_ERS_RESULT_NEED_RESET;
5500 }
5501
5502 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5503 {
5504         struct be_adapter *adapter = pci_get_drvdata(pdev);
5505         int status;
5506
5507         dev_info(&adapter->pdev->dev, "EEH reset\n");
5508
5509         status = pci_enable_device(pdev);
5510         if (status)
5511                 return PCI_ERS_RESULT_DISCONNECT;
5512
5513         pci_set_master(pdev);
5514         pci_set_power_state(pdev, PCI_D0);
5515         pci_restore_state(pdev);
5516
5517         /* Check if card is ok and fw is ready */
5518         dev_info(&adapter->pdev->dev,
5519                  "Waiting for FW to be ready after EEH reset\n");
5520         status = be_fw_wait_ready(adapter);
5521         if (status)
5522                 return PCI_ERS_RESULT_DISCONNECT;
5523
5524         pci_cleanup_aer_uncorrect_error_status(pdev);
5525         be_clear_all_error(adapter);
5526         return PCI_ERS_RESULT_RECOVERED;
5527 }
5528
5529 static void be_eeh_resume(struct pci_dev *pdev)
5530 {
5531         int status = 0;
5532         struct be_adapter *adapter = pci_get_drvdata(pdev);
5533         struct net_device *netdev =  adapter->netdev;
5534
5535         dev_info(&adapter->pdev->dev, "EEH resume\n");
5536
5537         pci_save_state(pdev);
5538
5539         status = be_cmd_reset_function(adapter);
5540         if (status)
5541                 goto err;
5542
5543         /* On some BE3 FW versions, after a HW reset,
5544          * interrupts will remain disabled for each function.
5545          * So, explicitly enable interrupts
5546          */
5547         be_intr_set(adapter, true);
5548
5549         /* tell fw we're ready to fire cmds */
5550         status = be_cmd_fw_init(adapter);
5551         if (status)
5552                 goto err;
5553
5554         status = be_setup(adapter);
5555         if (status)
5556                 goto err;
5557
5558         if (netif_running(netdev)) {
5559                 status = be_open(netdev);
5560                 if (status)
5561                         goto err;
5562         }
5563
5564         schedule_delayed_work(&adapter->func_recovery_work,
5565                               msecs_to_jiffies(1000));
5566         netif_device_attach(netdev);
5567         return;
5568 err:
5569         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5570 }
5571
5572 static const struct pci_error_handlers be_eeh_handlers = {
5573         .error_detected = be_eeh_err_detected,
5574         .slot_reset = be_eeh_reset,
5575         .resume = be_eeh_resume,
5576 };
5577
5578 static struct pci_driver be_driver = {
5579         .name = DRV_NAME,
5580         .id_table = be_dev_ids,
5581         .probe = be_probe,
5582         .remove = be_remove,
5583         .suspend = be_suspend,
5584         .resume = be_resume,
5585         .shutdown = be_shutdown,
5586         .err_handler = &be_eeh_handlers
5587 };
5588
5589 static int __init be_init_module(void)
5590 {
5591         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5592             rx_frag_size != 2048) {
5593                 printk(KERN_WARNING DRV_NAME
5594                         " : Module param rx_frag_size must be 2048/4096/8192."
5595                         " Using 2048\n");
5596                 rx_frag_size = 2048;
5597         }
5598
5599         return pci_register_driver(&be_driver);
5600 }
5601 module_init(be_init_module);
5602
5603 static void __exit be_exit_module(void)
5604 {
5605         pci_unregister_driver(&be_driver);
5606 }
5607 module_exit(be_exit_module);