drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 static unsigned int num_vfs;
  34 module_param(num_vfs, uint, S_IRUGO);
  35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  36
  37 static ushort rx_frag_size = 2048;
  38 module_param(rx_frag_size, ushort, S_IRUGO);
  39 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  40
  41 static const struct pci_device_id be_dev_ids[] = {
  42         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  44         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  46         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  50         { 0 }
  51 };
  52 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  53 /* UE Status Low CSR */
  54 static const char * const ue_status_low_desc[] = {
  55         "CEV",
  56         "CTX",
  57         "DBUF",
  58         "ERX",
  59         "Host",
  60         "MPU",
  61         "NDMA",
  62         "PTC ",
  63         "RDMA ",
  64         "RXF ",
  65         "RXIPS ",
  66         "RXULP0 ",
  67         "RXULP1 ",
  68         "RXULP2 ",
  69         "TIM ",
  70         "TPOST ",
  71         "TPRE ",
  72         "TXIPS ",
  73         "TXULP0 ",
  74         "TXULP1 ",
  75         "UC ",
  76         "WDMA ",
  77         "TXULP2 ",
  78         "HOST1 ",
  79         "P0_OB_LINK ",
  80         "P1_OB_LINK ",
  81         "HOST_GPIO ",
  82         "MBOX ",
  83         "ERX2 ",
  84         "SPARE ",
  85         "JTAG ",
  86         "MPU_INTPEND "
  87 };
  88
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 126 {
 127         struct be_dma_mem *mem = &q->dma_mem;
 128
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190
 191         val |= qid & DB_RQ_RING_ID_MASK;
 192         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 193
 194         wmb();
 195         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 196 }
 197
 198 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 199                           u16 posted)
 200 {
 201         u32 val = 0;
 202
 203         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 204         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 205
 206         wmb();
 207         iowrite32(val, adapter->db + txo->db_offset);
 208 }
 209
 210 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 211                          bool arm, bool clear_int, u16 num_popped)
 212 {
 213         u32 val = 0;
 214
 215         val |= qid & DB_EQ_RING_ID_MASK;
 216         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 217
 218         if (adapter->eeh_error)
 219                 return;
 220
 221         if (arm)
 222                 val |= 1 << DB_EQ_REARM_SHIFT;
 223         if (clear_int)
 224                 val |= 1 << DB_EQ_CLR_SHIFT;
 225         val |= 1 << DB_EQ_EVNT_SHIFT;
 226         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 227         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 228 }
 229
 230 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 231 {
 232         u32 val = 0;
 233
 234         val |= qid & DB_CQ_RING_ID_MASK;
 235         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 236                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 237
 238         if (adapter->eeh_error)
 239                 return;
 240
 241         if (arm)
 242                 val |= 1 << DB_CQ_REARM_SHIFT;
 243         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 244         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 245 }
 246
 247 static int be_mac_addr_set(struct net_device *netdev, void *p)
 248 {
 249         struct be_adapter *adapter = netdev_priv(netdev);
 250         struct device *dev = &adapter->pdev->dev;
 251         struct sockaddr *addr = p;
 252         int status;
 253         u8 mac[ETH_ALEN];
 254         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 255
 256         if (!is_valid_ether_addr(addr->sa_data))
 257                 return -EADDRNOTAVAIL;
 258
 259         /* Proceed further only if, User provided MAC is different
 260          * from active MAC
 261          */
 262         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 263                 return 0;
 264
 265         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 266          * privilege or if PF did not provision the new MAC address.
 267          * On BE3, this cmd will always fail if the VF doesn't have the
 268          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 269          * the MAC for the VF.
 270          */
 271         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 272                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 273         if (!status) {
 274                 curr_pmac_id = adapter->pmac_id[0];
 275
 276                 /* Delete the old programmed MAC. This call may fail if the
 277                  * old MAC was already deleted by the PF driver.
 278                  */
 279                 if (adapter->pmac_id[0] != old_pmac_id)
 280                         be_cmd_pmac_del(adapter, adapter->if_handle,
 281                                         old_pmac_id, 0);
 282         }
 283
 284         /* Decide if the new MAC is successfully activated only after
 285          * querying the FW
 286          */
 287         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 288                                        adapter->if_handle, true, 0);
 289         if (status)
 290                 goto err;
 291
 292         /* The MAC change did not happen, either due to lack of privilege
 293          * or PF didn't pre-provision.
 294          */
 295         if (!ether_addr_equal(addr->sa_data, mac)) {
 296                 status = -EPERM;
 297                 goto err;
 298         }
 299
 300         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 301         dev_info(dev, "MAC address changed to %pM\n", mac);
 302         return 0;
 303 err:
 304         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 305         return status;
 306 }
 307
 308 /* BE2 supports only v0 cmd */
 309 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 310 {
 311         if (BE2_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else if (BE3_chip(adapter)) {
 316                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         } else {
 320                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 321
 322                 return &cmd->hw_stats;
 323         }
 324 }
 325
 326 /* BE2 supports only v0 cmd */
 327 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 328 {
 329         if (BE2_chip(adapter)) {
 330                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else if (BE3_chip(adapter)) {
 334                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         } else {
 338                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 339
 340                 return &hw_stats->erx;
 341         }
 342 }
 343
 344 static void populate_be_v0_stats(struct be_adapter *adapter)
 345 {
 346         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 347         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 348         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 349         struct be_port_rxf_stats_v0 *port_stats =
 350                                         &rxf_stats->port[adapter->port_num];
 351         struct be_drv_stats *drvs = &adapter->drv_stats;
 352
 353         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 354         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 355         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 356         drvs->rx_control_frames = port_stats->rx_control_frames;
 357         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 358         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 359         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 360         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 361         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 362         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 363         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 364         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 365         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 366         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 367         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 368         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 369         drvs->rx_dropped_header_too_small =
 370                 port_stats->rx_dropped_header_too_small;
 371         drvs->rx_address_filtered =
 372                                         port_stats->rx_address_filtered +
 373                                         port_stats->rx_vlan_filtered;
 374         drvs->rx_alignment_symbol_errors =
 375                 port_stats->rx_alignment_symbol_errors;
 376
 377         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 378         drvs->tx_controlframes = port_stats->tx_controlframes;
 379
 380         if (adapter->port_num)
 381                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 382         else
 383                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 384         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 385         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 386         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 387         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 388         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 389         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 390         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 391 }
 392
 393 static void populate_be_v1_stats(struct be_adapter *adapter)
 394 {
 395         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 396         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 397         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 398         struct be_port_rxf_stats_v1 *port_stats =
 399                                         &rxf_stats->port[adapter->port_num];
 400         struct be_drv_stats *drvs = &adapter->drv_stats;
 401
 402         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 403         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 404         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 405         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 406         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 407         drvs->rx_control_frames = port_stats->rx_control_frames;
 408         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 409         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 410         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 411         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 412         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 413         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 414         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 415         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 416         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 417         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 418         drvs->rx_dropped_header_too_small =
 419                 port_stats->rx_dropped_header_too_small;
 420         drvs->rx_input_fifo_overflow_drop =
 421                 port_stats->rx_input_fifo_overflow_drop;
 422         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 423         drvs->rx_alignment_symbol_errors =
 424                 port_stats->rx_alignment_symbol_errors;
 425         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 426         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 427         drvs->tx_controlframes = port_stats->tx_controlframes;
 428         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 429         drvs->jabber_events = port_stats->jabber_events;
 430         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 431         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 432         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 433         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 434         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 435         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 436         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 437 }
 438
 439 static void populate_be_v2_stats(struct be_adapter *adapter)
 440 {
 441         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 442         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 443         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 444         struct be_port_rxf_stats_v2 *port_stats =
 445                                         &rxf_stats->port[adapter->port_num];
 446         struct be_drv_stats *drvs = &adapter->drv_stats;
 447
 448         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 449         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 450         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 451         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 452         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 453         drvs->rx_control_frames = port_stats->rx_control_frames;
 454         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 455         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 456         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 457         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 458         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 459         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 460         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 461         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 462         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 463         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 464         drvs->rx_dropped_header_too_small =
 465                 port_stats->rx_dropped_header_too_small;
 466         drvs->rx_input_fifo_overflow_drop =
 467                 port_stats->rx_input_fifo_overflow_drop;
 468         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 469         drvs->rx_alignment_symbol_errors =
 470                 port_stats->rx_alignment_symbol_errors;
 471         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 472         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 473         drvs->tx_controlframes = port_stats->tx_controlframes;
 474         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 475         drvs->jabber_events = port_stats->jabber_events;
 476         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 477         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 478         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 479         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 480         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 481         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 482         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 483         if (be_roce_supported(adapter)) {
 484                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 485                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 486                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 487                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 488                 drvs->roce_drops_payload_len =
 489                         port_stats->roce_drops_payload_len;
 490         }
 491 }
 492
 493 static void populate_lancer_stats(struct be_adapter *adapter)
 494 {
 495         struct be_drv_stats *drvs = &adapter->drv_stats;
 496         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 497
 498         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 499         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 500         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 501         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 502         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 503         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 504         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 505         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 506         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 507         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 508         drvs->rx_dropped_tcp_length =
 509                                 pport_stats->rx_dropped_invalid_tcp_length;
 510         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 511         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 512         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 513         drvs->rx_dropped_header_too_small =
 514                                 pport_stats->rx_dropped_header_too_small;
 515         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 516         drvs->rx_address_filtered =
 517                                         pport_stats->rx_address_filtered +
 518                                         pport_stats->rx_vlan_filtered;
 519         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 520         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 521         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 522         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 523         drvs->jabber_events = pport_stats->rx_jabbers;
 524         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 525         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 526         drvs->rx_drops_too_many_frags =
 527                                 pport_stats->rx_drops_too_many_frags_lo;
 528 }
 529
 530 static void accumulate_16bit_val(u32 *acc, u16 val)
 531 {
 532 #define lo(x)                   (x & 0xFFFF)
 533 #define hi(x)                   (x & 0xFFFF0000)
 534         bool wrapped = val < lo(*acc);
 535         u32 newacc = hi(*acc) + val;
 536
 537         if (wrapped)
 538                 newacc += 65536;
 539         ACCESS_ONCE(*acc) = newacc;
 540 }
 541
 542 static void populate_erx_stats(struct be_adapter *adapter,
 543                                struct be_rx_obj *rxo, u32 erx_stat)
 544 {
 545         if (!BEx_chip(adapter))
 546                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 547         else
 548                 /* below erx HW counter can actually wrap around after
 549                  * 65535. Driver accumulates a 32-bit value
 550                  */
 551                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 552                                      (u16)erx_stat);
 553 }
 554
 555 void be_parse_stats(struct be_adapter *adapter)
 556 {
 557         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 558         struct be_rx_obj *rxo;
 559         int i;
 560         u32 erx_stat;
 561
 562         if (lancer_chip(adapter)) {
 563                 populate_lancer_stats(adapter);
 564         } else {
 565                 if (BE2_chip(adapter))
 566                         populate_be_v0_stats(adapter);
 567                 else if (BE3_chip(adapter))
 568                         /* for BE3 */
 569                         populate_be_v1_stats(adapter);
 570                 else
 571                         populate_be_v2_stats(adapter);
 572
 573                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 574                 for_all_rx_queues(adapter, rxo, i) {
 575                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 576                         populate_erx_stats(adapter, rxo, erx_stat);
 577                 }
 578         }
 579 }
 580
 581 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 582                                                 struct rtnl_link_stats64 *stats)
 583 {
 584         struct be_adapter *adapter = netdev_priv(netdev);
 585         struct be_drv_stats *drvs = &adapter->drv_stats;
 586         struct be_rx_obj *rxo;
 587         struct be_tx_obj *txo;
 588         u64 pkts, bytes;
 589         unsigned int start;
 590         int i;
 591
 592         for_all_rx_queues(adapter, rxo, i) {
 593                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 594
 595                 do {
 596                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 597                         pkts = rx_stats(rxo)->rx_pkts;
 598                         bytes = rx_stats(rxo)->rx_bytes;
 599                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 600                 stats->rx_packets += pkts;
 601                 stats->rx_bytes += bytes;
 602                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 603                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 604                                         rx_stats(rxo)->rx_drops_no_frags;
 605         }
 606
 607         for_all_tx_queues(adapter, txo, i) {
 608                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 609
 610                 do {
 611                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 612                         pkts = tx_stats(txo)->tx_pkts;
 613                         bytes = tx_stats(txo)->tx_bytes;
 614                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 615                 stats->tx_packets += pkts;
 616                 stats->tx_bytes += bytes;
 617         }
 618
 619         /* bad pkts received */
 620         stats->rx_errors = drvs->rx_crc_errors +
 621                 drvs->rx_alignment_symbol_errors +
 622                 drvs->rx_in_range_errors +
 623                 drvs->rx_out_range_errors +
 624                 drvs->rx_frame_too_long +
 625                 drvs->rx_dropped_too_small +
 626                 drvs->rx_dropped_too_short +
 627                 drvs->rx_dropped_header_too_small +
 628                 drvs->rx_dropped_tcp_length +
 629                 drvs->rx_dropped_runt;
 630
 631         /* detailed rx errors */
 632         stats->rx_length_errors = drvs->rx_in_range_errors +
 633                 drvs->rx_out_range_errors +
 634                 drvs->rx_frame_too_long;
 635
 636         stats->rx_crc_errors = drvs->rx_crc_errors;
 637
 638         /* frame alignment errors */
 639         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 640
 641         /* receiver fifo overrun */
 642         /* drops_no_pbuf is no per i/f, it's per BE card */
 643         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 644                                 drvs->rx_input_fifo_overflow_drop +
 645                                 drvs->rx_drops_no_pbuf;
 646         return stats;
 647 }
 648
 649 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 650 {
 651         struct net_device *netdev = adapter->netdev;
 652
 653         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 654                 netif_carrier_off(netdev);
 655                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 656         }
 657
 658         if (link_status)
 659                 netif_carrier_on(netdev);
 660         else
 661                 netif_carrier_off(netdev);
 662 }
 663
 664 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 665 {
 666         struct be_tx_stats *stats = tx_stats(txo);
 667
 668         u64_stats_update_begin(&stats->sync);
 669         stats->tx_reqs++;
 670         stats->tx_bytes += skb->len;
 671         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Returns number of WRBs needed for the skb */
 676 static u32 skb_wrb_cnt(struct sk_buff *skb)
 677 {
 678         /* +1 for the header wrb */
 679         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 680 }
 681
 682 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 683 {
 684         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 685         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 686         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 687         wrb->rsvd0 = 0;
 688 }
 689
 690 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 691  * to avoid the swap and shift/mask operations in wrb_fill().
 692  */
 693 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 694 {
 695         wrb->frag_pa_hi = 0;
 696         wrb->frag_pa_lo = 0;
 697         wrb->frag_len = 0;
 698         wrb->rsvd0 = 0;
 699 }
 700
 701 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 702                                      struct sk_buff *skb)
 703 {
 704         u8 vlan_prio;
 705         u16 vlan_tag;
 706
 707         vlan_tag = skb_vlan_tag_get(skb);
 708         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 709         /* If vlan priority provided by OS is NOT in available bmap */
 710         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 711                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 712                                 adapter->recommended_prio;
 713
 714         return vlan_tag;
 715 }
 716
 717 /* Used only for IP tunnel packets */
 718 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 719 {
 720         return (inner_ip_hdr(skb)->version == 4) ?
 721                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 722 }
 723
 724 static u16 skb_ip_proto(struct sk_buff *skb)
 725 {
 726         return (ip_hdr(skb)->version == 4) ?
 727                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 728 }
 729
 730 static void be_get_wrb_params_from_skb(struct be_adapter *adapter,
 731                                        struct sk_buff *skb,
 732                                        struct be_wrb_params *wrb_params)
 733 {
 734         u16 proto;
 735
 736         if (skb_is_gso(skb)) {
 737                 BE_WRB_F_SET(wrb_params->features, LSO, 1);
 738                 wrb_params->lso_mss = skb_shinfo(skb)->gso_size;
 739                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 740                         BE_WRB_F_SET(wrb_params->features, LSO6, 1);
 741         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 742                 if (skb->encapsulation) {
 743                         BE_WRB_F_SET(wrb_params->features, IPCS, 1);
 744                         proto = skb_inner_ip_proto(skb);
 745                 } else {
 746                         proto = skb_ip_proto(skb);
 747                 }
 748                 if (proto == IPPROTO_TCP)
 749                         BE_WRB_F_SET(wrb_params->features, TCPCS, 1);
 750                 else if (proto == IPPROTO_UDP)
 751                         BE_WRB_F_SET(wrb_params->features, UDPCS, 1);
 752         }
 753
 754         if (skb_vlan_tag_present(skb)) {
 755                 BE_WRB_F_SET(wrb_params->features, VLAN, 1);
 756                 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 757         }
 758
 759         BE_WRB_F_SET(wrb_params->features, CRC, 1);
 760 }
 761
 762 static void wrb_fill_hdr(struct be_adapter *adapter,
 763                          struct be_eth_hdr_wrb *hdr,
 764                          struct be_wrb_params *wrb_params,
 765                          struct sk_buff *skb)
 766 {
 767         memset(hdr, 0, sizeof(*hdr));
 768
 769         SET_TX_WRB_HDR_BITS(crc, hdr,
 770                             BE_WRB_F_GET(wrb_params->features, CRC));
 771         SET_TX_WRB_HDR_BITS(ipcs, hdr,
 772                             BE_WRB_F_GET(wrb_params->features, IPCS));
 773         SET_TX_WRB_HDR_BITS(tcpcs, hdr,
 774                             BE_WRB_F_GET(wrb_params->features, TCPCS));
 775         SET_TX_WRB_HDR_BITS(udpcs, hdr,
 776                             BE_WRB_F_GET(wrb_params->features, UDPCS));
 777
 778         SET_TX_WRB_HDR_BITS(lso, hdr,
 779                             BE_WRB_F_GET(wrb_params->features, LSO));
 780         SET_TX_WRB_HDR_BITS(lso6, hdr,
 781                             BE_WRB_F_GET(wrb_params->features, LSO6));
 782         SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss);
 783
 784         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this
 785          * hack is not needed, the evt bit is set while ringing DB.
 786          */
 787         SET_TX_WRB_HDR_BITS(event, hdr,
 788                             BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW));
 789         SET_TX_WRB_HDR_BITS(vlan, hdr,
 790                             BE_WRB_F_GET(wrb_params->features, VLAN));
 791         SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag);
 792
 793         SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb));
 794         SET_TX_WRB_HDR_BITS(len, hdr, skb->len);
 795 }
 796
 797 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 798                           bool unmap_single)
 799 {
 800         dma_addr_t dma;
 801         u32 frag_len = le32_to_cpu(wrb->frag_len);
 802
 803
 804         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 805                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 806         if (frag_len) {
 807                 if (unmap_single)
 808                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 809                 else
 810                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 811         }
 812 }
 813
 814 /* Grab a WRB header for xmit */
 815 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo)
 816 {
 817         u16 head = txo->q.head;
 818
 819         queue_head_inc(&txo->q);
 820         return head;
 821 }
 822
 823 /* Set up the WRB header for xmit */
 824 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter,
 825                                 struct be_tx_obj *txo,
 826                                 struct be_wrb_params *wrb_params,
 827                                 struct sk_buff *skb, u16 head)
 828 {
 829         u32 num_frags = skb_wrb_cnt(skb);
 830         struct be_queue_info *txq = &txo->q;
 831         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head);
 832
 833         wrb_fill_hdr(adapter, hdr, wrb_params, skb);
 834         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 835
 836         BUG_ON(txo->sent_skb_list[head]);
 837         txo->sent_skb_list[head] = skb;
 838         txo->last_req_hdr = head;
 839         atomic_add(num_frags, &txq->used);
 840         txo->last_req_wrb_cnt = num_frags;
 841         txo->pend_wrb_cnt += num_frags;
 842 }
 843
 844 /* Setup a WRB fragment (buffer descriptor) for xmit */
 845 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,
 846                                  int len)
 847 {
 848         struct be_eth_wrb *wrb;
 849         struct be_queue_info *txq = &txo->q;
 850
 851         wrb = queue_head_node(txq);
 852         wrb_fill(wrb, busaddr, len);
 853         queue_head_inc(txq);
 854 }
 855
 856 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine
 857  * was invoked. The producer index is restored to the previous packet and the
 858  * WRBs of the current packet are unmapped. Invoked to handle tx setup errors.
 859  */
 860 static void be_xmit_restore(struct be_adapter *adapter,
 861                             struct be_tx_obj *txo, u16 head, bool map_single,
 862                             u32 copied)
 863 {
 864         struct device *dev;
 865         struct be_eth_wrb *wrb;
 866         struct be_queue_info *txq = &txo->q;
 867
 868         dev = &adapter->pdev->dev;
 869         txq->head = head;
 870
 871         /* skip the first wrb (hdr); it's not mapped */
 872         queue_head_inc(txq);
 873         while (copied) {
 874                 wrb = queue_head_node(txq);
 875                 unmap_tx_frag(dev, wrb, map_single);
 876                 map_single = false;
 877                 copied -= le32_to_cpu(wrb->frag_len);
 878                 queue_head_inc(txq);
 879         }
 880
 881         txq->head = head;
 882 }
 883
 884 /* Enqueue the given packet for transmit. This routine allocates WRBs for the
 885  * packet, dma maps the packet buffers and sets up the WRBs. Returns the number
 886  * of WRBs used up by the packet.
 887  */
 888 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 889                            struct sk_buff *skb,
 890                            struct be_wrb_params *wrb_params)
 891 {
 892         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 893         struct device *dev = &adapter->pdev->dev;
 894         struct be_queue_info *txq = &txo->q;
 895         bool map_single = false;
 896         u16 head = txq->head;
 897         dma_addr_t busaddr;
 898         int len;
 899
 900         head = be_tx_get_wrb_hdr(txo);
 901
 902         if (skb->len > skb->data_len) {
 903                 len = skb_headlen(skb);
 904
 905                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 906                 if (dma_mapping_error(dev, busaddr))
 907                         goto dma_err;
 908                 map_single = true;
 909                 be_tx_setup_wrb_frag(txo, busaddr, len);
 910                 copied += len;
 911         }
 912
 913         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 914                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 915                 len = skb_frag_size(frag);
 916
 917                 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
 918                 if (dma_mapping_error(dev, busaddr))
 919                         goto dma_err;
 920                 be_tx_setup_wrb_frag(txo, busaddr, len);
 921                 copied += len;
 922         }
 923
 924         be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);
 925
 926         be_tx_stats_update(txo, skb);
 927         return wrb_cnt;
 928
 929 dma_err:
 930         adapter->drv_stats.dma_map_errors++;
 931         be_xmit_restore(adapter, txo, head, map_single, copied);
 932         return 0;
 933 }
 934
 935 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 936 {
 937         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 938 }
 939
 940 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 941                                              struct sk_buff *skb,
 942                                              struct be_wrb_params
 943                                              *wrb_params)
 944 {
 945         u16 vlan_tag = 0;
 946
 947         skb = skb_share_check(skb, GFP_ATOMIC);
 948         if (unlikely(!skb))
 949                 return skb;
 950
 951         if (skb_vlan_tag_present(skb))
 952                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 953
 954         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 955                 if (!vlan_tag)
 956                         vlan_tag = adapter->pvid;
 957                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 958                  * skip VLAN insertion
 959                  */
 960                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
 961         }
 962
 963         if (vlan_tag) {
 964                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 965                                                 vlan_tag);
 966                 if (unlikely(!skb))
 967                         return skb;
 968                 skb->vlan_tci = 0;
 969         }
 970
 971         /* Insert the outer VLAN, if any */
 972         if (adapter->qnq_vid) {
 973                 vlan_tag = adapter->qnq_vid;
 974                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 975                                                 vlan_tag);
 976                 if (unlikely(!skb))
 977                         return skb;
 978                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
 979         }
 980
 981         return skb;
 982 }
 983
 984 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 985 {
 986         struct ethhdr *eh = (struct ethhdr *)skb->data;
 987         u16 offset = ETH_HLEN;
 988
 989         if (eh->h_proto == htons(ETH_P_IPV6)) {
 990                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 991
 992                 offset += sizeof(struct ipv6hdr);
 993                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 994                     ip6h->nexthdr != NEXTHDR_UDP) {
 995                         struct ipv6_opt_hdr *ehdr =
 996                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 997
 998                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 999                         if (ehdr->hdrlen == 0xff)
1000                                 return true;
1001                 }
1002         }
1003         return false;
1004 }
1005
1006 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
1007 {
1008         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
1009 }
1010
1011 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
1012 {
1013         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
1014 }
1015
1016 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
1017                                                   struct sk_buff *skb,
1018                                                   struct be_wrb_params
1019                                                   *wrb_params)
1020 {
1021         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1022         unsigned int eth_hdr_len;
1023         struct iphdr *ip;
1024
1025         /* For padded packets, BE HW modifies tot_len field in IP header
1026          * incorrecly when VLAN tag is inserted by HW.
1027          * For padded packets, Lancer computes incorrect checksum.
1028          */
1029         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
1030                                                 VLAN_ETH_HLEN : ETH_HLEN;
1031         if (skb->len <= 60 &&
1032             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
1033             is_ipv4_pkt(skb)) {
1034                 ip = (struct iphdr *)ip_hdr(skb);
1035                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
1036         }
1037
1038         /* If vlan tag is already inlined in the packet, skip HW VLAN
1039          * tagging in pvid-tagging mode
1040          */
1041         if (be_pvid_tagging_enabled(adapter) &&
1042             veh->h_vlan_proto == htons(ETH_P_8021Q))
1043                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1044
1045         /* HW has a bug wherein it will calculate CSUM for VLAN
1046          * pkts even though it is disabled.
1047          * Manually insert VLAN in pkt.
1048          */
1049         if (skb->ip_summed != CHECKSUM_PARTIAL &&
1050             skb_vlan_tag_present(skb)) {
1051                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1052                 if (unlikely(!skb))
1053                         goto err;
1054         }
1055
1056         /* HW may lockup when VLAN HW tagging is requested on
1057          * certain ipv6 packets. Drop such pkts if the HW workaround to
1058          * skip HW tagging is not enabled by FW.
1059          */
1060         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
1061                      (adapter->pvid || adapter->qnq_vid) &&
1062                      !qnq_async_evt_rcvd(adapter)))
1063                 goto tx_drop;
1064
1065         /* Manual VLAN tag insertion to prevent:
1066          * ASIC lockup when the ASIC inserts VLAN tag into
1067          * certain ipv6 packets. Insert VLAN tags in driver,
1068          * and set event, completion, vlan bits accordingly
1069          * in the Tx WRB.
1070          */
1071         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1072             be_vlan_tag_tx_chk(adapter, skb)) {
1073                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1074                 if (unlikely(!skb))
1075                         goto err;
1076         }
1077
1078         return skb;
1079 tx_drop:
1080         dev_kfree_skb_any(skb);
1081 err:
1082         return NULL;
1083 }
1084
1085 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1086                                            struct sk_buff *skb,
1087                                            struct be_wrb_params *wrb_params)
1088 {
1089         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1090          * less may cause a transmit stall on that port. So the work-around is
1091          * to pad short packets (<= 32 bytes) to a 36-byte length.
1092          */
1093         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1094                 if (skb_put_padto(skb, 36))
1095                         return NULL;
1096         }
1097
1098         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1099                 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params);
1100                 if (!skb)
1101                         return NULL;
1102         }
1103
1104         return skb;
1105 }
1106
1107 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1108 {
1109         struct be_queue_info *txq = &txo->q;
1110         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1111
1112         /* Mark the last request eventable if it hasn't been marked already */
1113         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1114                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1115
1116         /* compose a dummy wrb if there are odd set of wrbs to notify */
1117         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1118                 wrb_fill_dummy(queue_head_node(txq));
1119                 queue_head_inc(txq);
1120                 atomic_inc(&txq->used);
1121                 txo->pend_wrb_cnt++;
1122                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1123                                            TX_HDR_WRB_NUM_SHIFT);
1124                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1125                                           TX_HDR_WRB_NUM_SHIFT);
1126         }
1127         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1128         txo->pend_wrb_cnt = 0;
1129 }
1130
1131 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1132 {
1133         struct be_adapter *adapter = netdev_priv(netdev);
1134         u16 q_idx = skb_get_queue_mapping(skb);
1135         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1136         struct be_wrb_params wrb_params = { 0 };
1137         struct be_queue_info *txq = &txo->q;
1138         bool flush = !skb->xmit_more;
1139         u16 wrb_cnt;
1140
1141         skb = be_xmit_workarounds(adapter, skb, &wrb_params);
1142         if (unlikely(!skb))
1143                 goto drop;
1144
1145         be_get_wrb_params_from_skb(adapter, skb, &wrb_params);
1146
1147         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1148         if (unlikely(!wrb_cnt)) {
1149                 dev_kfree_skb_any(skb);
1150                 goto drop;
1151         }
1152
1153         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1154                 netif_stop_subqueue(netdev, q_idx);
1155                 tx_stats(txo)->tx_stops++;
1156         }
1157
1158         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1159                 be_xmit_flush(adapter, txo);
1160
1161         return NETDEV_TX_OK;
1162 drop:
1163         tx_stats(txo)->tx_drv_drops++;
1164         /* Flush the already enqueued tx requests */
1165         if (flush && txo->pend_wrb_cnt)
1166                 be_xmit_flush(adapter, txo);
1167
1168         return NETDEV_TX_OK;
1169 }
1170
1171 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1172 {
1173         struct be_adapter *adapter = netdev_priv(netdev);
1174         struct device *dev = &adapter->pdev->dev;
1175
1176         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1177                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1178                          BE_MIN_MTU, BE_MAX_MTU);
1179                 return -EINVAL;
1180         }
1181
1182         dev_info(dev, "MTU changed from %d to %d bytes\n",
1183                  netdev->mtu, new_mtu);
1184         netdev->mtu = new_mtu;
1185         return 0;
1186 }
1187
1188 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1189 {
1190         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1191                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1192 }
1193
1194 static int be_set_vlan_promisc(struct be_adapter *adapter)
1195 {
1196         struct device *dev = &adapter->pdev->dev;
1197         int status;
1198
1199         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1200                 return 0;
1201
1202         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1203         if (!status) {
1204                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1205                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1206         } else {
1207                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1208         }
1209         return status;
1210 }
1211
1212 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1213 {
1214         struct device *dev = &adapter->pdev->dev;
1215         int status;
1216
1217         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1218         if (!status) {
1219                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1220                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1221         }
1222         return status;
1223 }
1224
1225 /*
1226  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1227  * If the user configures more, place BE in vlan promiscuous mode.
1228  */
1229 static int be_vid_config(struct be_adapter *adapter)
1230 {
1231         struct device *dev = &adapter->pdev->dev;
1232         u16 vids[BE_NUM_VLANS_SUPPORTED];
1233         u16 num = 0, i = 0;
1234         int status = 0;
1235
1236         /* No need to further configure vids if in promiscuous mode */
1237         if (be_in_all_promisc(adapter))
1238                 return 0;
1239
1240         if (adapter->vlans_added > be_max_vlans(adapter))
1241                 return be_set_vlan_promisc(adapter);
1242
1243         /* Construct VLAN Table to give to HW */
1244         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1245                 vids[num++] = cpu_to_le16(i);
1246
1247         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1248         if (status) {
1249                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1250                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1251                 if (addl_status(status) ==
1252                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1253                         return be_set_vlan_promisc(adapter);
1254         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1255                 status = be_clear_vlan_promisc(adapter);
1256         }
1257         return status;
1258 }
1259
1260 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1261 {
1262         struct be_adapter *adapter = netdev_priv(netdev);
1263         int status = 0;
1264
1265         /* Packets with VID 0 are always received by Lancer by default */
1266         if (lancer_chip(adapter) && vid == 0)
1267                 return status;
1268
1269         if (test_bit(vid, adapter->vids))
1270                 return status;
1271
1272         set_bit(vid, adapter->vids);
1273         adapter->vlans_added++;
1274
1275         status = be_vid_config(adapter);
1276         if (status) {
1277                 adapter->vlans_added--;
1278                 clear_bit(vid, adapter->vids);
1279         }
1280
1281         return status;
1282 }
1283
1284 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1285 {
1286         struct be_adapter *adapter = netdev_priv(netdev);
1287
1288         /* Packets with VID 0 are always received by Lancer by default */
1289         if (lancer_chip(adapter) && vid == 0)
1290                 return 0;
1291
1292         clear_bit(vid, adapter->vids);
1293         adapter->vlans_added--;
1294
1295         return be_vid_config(adapter);
1296 }
1297
1298 static void be_clear_all_promisc(struct be_adapter *adapter)
1299 {
1300         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1301         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1302 }
1303
1304 static void be_set_all_promisc(struct be_adapter *adapter)
1305 {
1306         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1307         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1308 }
1309
1310 static void be_set_mc_promisc(struct be_adapter *adapter)
1311 {
1312         int status;
1313
1314         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1315                 return;
1316
1317         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1318         if (!status)
1319                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1320 }
1321
1322 static void be_set_mc_list(struct be_adapter *adapter)
1323 {
1324         int status;
1325
1326         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1327         if (!status)
1328                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1329         else
1330                 be_set_mc_promisc(adapter);
1331 }
1332
1333 static void be_set_uc_list(struct be_adapter *adapter)
1334 {
1335         struct netdev_hw_addr *ha;
1336         int i = 1; /* First slot is claimed by the Primary MAC */
1337
1338         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1339                 be_cmd_pmac_del(adapter, adapter->if_handle,
1340                                 adapter->pmac_id[i], 0);
1341
1342         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1343                 be_set_all_promisc(adapter);
1344                 return;
1345         }
1346
1347         netdev_for_each_uc_addr(ha, adapter->netdev) {
1348                 adapter->uc_macs++; /* First slot is for Primary MAC */
1349                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1350                                 &adapter->pmac_id[adapter->uc_macs], 0);
1351         }
1352 }
1353
1354 static void be_clear_uc_list(struct be_adapter *adapter)
1355 {
1356         int i;
1357
1358         for (i = 1; i < (adapter->uc_macs + 1); i++)
1359                 be_cmd_pmac_del(adapter, adapter->if_handle,
1360                                 adapter->pmac_id[i], 0);
1361         adapter->uc_macs = 0;
1362 }
1363
1364 static void be_set_rx_mode(struct net_device *netdev)
1365 {
1366         struct be_adapter *adapter = netdev_priv(netdev);
1367
1368         if (netdev->flags & IFF_PROMISC) {
1369                 be_set_all_promisc(adapter);
1370                 return;
1371         }
1372
1373         /* Interface was previously in promiscuous mode; disable it */
1374         if (be_in_all_promisc(adapter)) {
1375                 be_clear_all_promisc(adapter);
1376                 if (adapter->vlans_added)
1377                         be_vid_config(adapter);
1378         }
1379
1380         /* Enable multicast promisc if num configured exceeds what we support */
1381         if (netdev->flags & IFF_ALLMULTI ||
1382             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1383                 be_set_mc_promisc(adapter);
1384                 return;
1385         }
1386
1387         if (netdev_uc_count(netdev) != adapter->uc_macs)
1388                 be_set_uc_list(adapter);
1389
1390         be_set_mc_list(adapter);
1391 }
1392
1393 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1394 {
1395         struct be_adapter *adapter = netdev_priv(netdev);
1396         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1397         int status;
1398
1399         if (!sriov_enabled(adapter))
1400                 return -EPERM;
1401
1402         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1403                 return -EINVAL;
1404
1405         /* Proceed further only if user provided MAC is different
1406          * from active MAC
1407          */
1408         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1409                 return 0;
1410
1411         if (BEx_chip(adapter)) {
1412                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1413                                 vf + 1);
1414
1415                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1416                                          &vf_cfg->pmac_id, vf + 1);
1417         } else {
1418                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1419                                         vf + 1);
1420         }
1421
1422         if (status) {
1423                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1424                         mac, vf, status);
1425                 return be_cmd_status(status);
1426         }
1427
1428         ether_addr_copy(vf_cfg->mac_addr, mac);
1429
1430         return 0;
1431 }
1432
1433 static int be_get_vf_config(struct net_device *netdev, int vf,
1434                             struct ifla_vf_info *vi)
1435 {
1436         struct be_adapter *adapter = netdev_priv(netdev);
1437         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1438
1439         if (!sriov_enabled(adapter))
1440                 return -EPERM;
1441
1442         if (vf >= adapter->num_vfs)
1443                 return -EINVAL;
1444
1445         vi->vf = vf;
1446         vi->max_tx_rate = vf_cfg->tx_rate;
1447         vi->min_tx_rate = 0;
1448         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1449         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1450         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1451         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1452
1453         return 0;
1454 }
1455
1456 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1457 {
1458         struct be_adapter *adapter = netdev_priv(netdev);
1459         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1460         int status = 0;
1461
1462         if (!sriov_enabled(adapter))
1463                 return -EPERM;
1464
1465         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1466                 return -EINVAL;
1467
1468         if (vlan || qos) {
1469                 vlan |= qos << VLAN_PRIO_SHIFT;
1470                 if (vf_cfg->vlan_tag != vlan)
1471                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1472                                                        vf_cfg->if_handle, 0);
1473         } else {
1474                 /* Reset Transparent Vlan Tagging. */
1475                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1476                                                vf + 1, vf_cfg->if_handle, 0);
1477         }
1478
1479         if (status) {
1480                 dev_err(&adapter->pdev->dev,
1481                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1482                         vf, status);
1483                 return be_cmd_status(status);
1484         }
1485
1486         vf_cfg->vlan_tag = vlan;
1487
1488         return 0;
1489 }
1490
1491 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1492                              int min_tx_rate, int max_tx_rate)
1493 {
1494         struct be_adapter *adapter = netdev_priv(netdev);
1495         struct device *dev = &adapter->pdev->dev;
1496         int percent_rate, status = 0;
1497         u16 link_speed = 0;
1498         u8 link_status;
1499
1500         if (!sriov_enabled(adapter))
1501                 return -EPERM;
1502
1503         if (vf >= adapter->num_vfs)
1504                 return -EINVAL;
1505
1506         if (min_tx_rate)
1507                 return -EINVAL;
1508
1509         if (!max_tx_rate)
1510                 goto config_qos;
1511
1512         status = be_cmd_link_status_query(adapter, &link_speed,
1513                                           &link_status, 0);
1514         if (status)
1515                 goto err;
1516
1517         if (!link_status) {
1518                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1519                 status = -ENETDOWN;
1520                 goto err;
1521         }
1522
1523         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1524                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1525                         link_speed);
1526                 status = -EINVAL;
1527                 goto err;
1528         }
1529
1530         /* On Skyhawk the QOS setting must be done only as a % value */
1531         percent_rate = link_speed / 100;
1532         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1533                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1534                         percent_rate);
1535                 status = -EINVAL;
1536                 goto err;
1537         }
1538
1539 config_qos:
1540         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1541         if (status)
1542                 goto err;
1543
1544         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1545         return 0;
1546
1547 err:
1548         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1549                 max_tx_rate, vf);
1550         return be_cmd_status(status);
1551 }
1552
1553 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1554                                 int link_state)
1555 {
1556         struct be_adapter *adapter = netdev_priv(netdev);
1557         int status;
1558
1559         if (!sriov_enabled(adapter))
1560                 return -EPERM;
1561
1562         if (vf >= adapter->num_vfs)
1563                 return -EINVAL;
1564
1565         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1566         if (status) {
1567                 dev_err(&adapter->pdev->dev,
1568                         "Link state change on VF %d failed: %#x\n", vf, status);
1569                 return be_cmd_status(status);
1570         }
1571
1572         adapter->vf_cfg[vf].plink_tracking = link_state;
1573
1574         return 0;
1575 }
1576
1577 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1578                           ulong now)
1579 {
1580         aic->rx_pkts_prev = rx_pkts;
1581         aic->tx_reqs_prev = tx_pkts;
1582         aic->jiffies = now;
1583 }
1584
1585 static void be_eqd_update(struct be_adapter *adapter)
1586 {
1587         struct be_set_eqd set_eqd[MAX_EVT_QS];
1588         int eqd, i, num = 0, start;
1589         struct be_aic_obj *aic;
1590         struct be_eq_obj *eqo;
1591         struct be_rx_obj *rxo;
1592         struct be_tx_obj *txo;
1593         u64 rx_pkts, tx_pkts;
1594         ulong now;
1595         u32 pps, delta;
1596
1597         for_all_evt_queues(adapter, eqo, i) {
1598                 aic = &adapter->aic_obj[eqo->idx];
1599                 if (!aic->enable) {
1600                         if (aic->jiffies)
1601                                 aic->jiffies = 0;
1602                         eqd = aic->et_eqd;
1603                         goto modify_eqd;
1604                 }
1605
1606                 rxo = &adapter->rx_obj[eqo->idx];
1607                 do {
1608                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1609                         rx_pkts = rxo->stats.rx_pkts;
1610                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1611
1612                 txo = &adapter->tx_obj[eqo->idx];
1613                 do {
1614                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1615                         tx_pkts = txo->stats.tx_reqs;
1616                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1617
1618                 /* Skip, if wrapped around or first calculation */
1619                 now = jiffies;
1620                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1621                     rx_pkts < aic->rx_pkts_prev ||
1622                     tx_pkts < aic->tx_reqs_prev) {
1623                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1624                         continue;
1625                 }
1626
1627                 delta = jiffies_to_msecs(now - aic->jiffies);
1628                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1629                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1630                 eqd = (pps / 15000) << 2;
1631
1632                 if (eqd < 8)
1633                         eqd = 0;
1634                 eqd = min_t(u32, eqd, aic->max_eqd);
1635                 eqd = max_t(u32, eqd, aic->min_eqd);
1636
1637                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1638 modify_eqd:
1639                 if (eqd != aic->prev_eqd) {
1640                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1641                         set_eqd[num].eq_id = eqo->q.id;
1642                         aic->prev_eqd = eqd;
1643                         num++;
1644                 }
1645         }
1646
1647         if (num)
1648                 be_cmd_modify_eqd(adapter, set_eqd, num);
1649 }
1650
1651 static void be_rx_stats_update(struct be_rx_obj *rxo,
1652                                struct be_rx_compl_info *rxcp)
1653 {
1654         struct be_rx_stats *stats = rx_stats(rxo);
1655
1656         u64_stats_update_begin(&stats->sync);
1657         stats->rx_compl++;
1658         stats->rx_bytes += rxcp->pkt_size;
1659         stats->rx_pkts++;
1660         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1661                 stats->rx_mcast_pkts++;
1662         if (rxcp->err)
1663                 stats->rx_compl_err++;
1664         u64_stats_update_end(&stats->sync);
1665 }
1666
1667 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1668 {
1669         /* L4 checksum is not reliable for non TCP/UDP packets.
1670          * Also ignore ipcksm for ipv6 pkts
1671          */
1672         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1673                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1674 }
1675
1676 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1677 {
1678         struct be_adapter *adapter = rxo->adapter;
1679         struct be_rx_page_info *rx_page_info;
1680         struct be_queue_info *rxq = &rxo->q;
1681         u16 frag_idx = rxq->tail;
1682
1683         rx_page_info = &rxo->page_info_tbl[frag_idx];
1684         BUG_ON(!rx_page_info->page);
1685
1686         if (rx_page_info->last_frag) {
1687                 dma_unmap_page(&adapter->pdev->dev,
1688                                dma_unmap_addr(rx_page_info, bus),
1689                                adapter->big_page_size, DMA_FROM_DEVICE);
1690                 rx_page_info->last_frag = false;
1691         } else {
1692                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1693                                         dma_unmap_addr(rx_page_info, bus),
1694                                         rx_frag_size, DMA_FROM_DEVICE);
1695         }
1696
1697         queue_tail_inc(rxq);
1698         atomic_dec(&rxq->used);
1699         return rx_page_info;
1700 }
1701
1702 /* Throwaway the data in the Rx completion */
1703 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1704                                 struct be_rx_compl_info *rxcp)
1705 {
1706         struct be_rx_page_info *page_info;
1707         u16 i, num_rcvd = rxcp->num_rcvd;
1708
1709         for (i = 0; i < num_rcvd; i++) {
1710                 page_info = get_rx_page_info(rxo);
1711                 put_page(page_info->page);
1712                 memset(page_info, 0, sizeof(*page_info));
1713         }
1714 }
1715
1716 /*
1717  * skb_fill_rx_data forms a complete skb for an ether frame
1718  * indicated by rxcp.
1719  */
1720 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1721                              struct be_rx_compl_info *rxcp)
1722 {
1723         struct be_rx_page_info *page_info;
1724         u16 i, j;
1725         u16 hdr_len, curr_frag_len, remaining;
1726         u8 *start;
1727
1728         page_info = get_rx_page_info(rxo);
1729         start = page_address(page_info->page) + page_info->page_offset;
1730         prefetch(start);
1731
1732         /* Copy data in the first descriptor of this completion */
1733         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1734
1735         skb->len = curr_frag_len;
1736         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1737                 memcpy(skb->data, start, curr_frag_len);
1738                 /* Complete packet has now been moved to data */
1739                 put_page(page_info->page);
1740                 skb->data_len = 0;
1741                 skb->tail += curr_frag_len;
1742         } else {
1743                 hdr_len = ETH_HLEN;
1744                 memcpy(skb->data, start, hdr_len);
1745                 skb_shinfo(skb)->nr_frags = 1;
1746                 skb_frag_set_page(skb, 0, page_info->page);
1747                 skb_shinfo(skb)->frags[0].page_offset =
1748                                         page_info->page_offset + hdr_len;
1749                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1750                                   curr_frag_len - hdr_len);
1751                 skb->data_len = curr_frag_len - hdr_len;
1752                 skb->truesize += rx_frag_size;
1753                 skb->tail += hdr_len;
1754         }
1755         page_info->page = NULL;
1756
1757         if (rxcp->pkt_size <= rx_frag_size) {
1758                 BUG_ON(rxcp->num_rcvd != 1);
1759                 return;
1760         }
1761
1762         /* More frags present for this completion */
1763         remaining = rxcp->pkt_size - curr_frag_len;
1764         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1765                 page_info = get_rx_page_info(rxo);
1766                 curr_frag_len = min(remaining, rx_frag_size);
1767
1768                 /* Coalesce all frags from the same physical page in one slot */
1769                 if (page_info->page_offset == 0) {
1770                         /* Fresh page */
1771                         j++;
1772                         skb_frag_set_page(skb, j, page_info->page);
1773                         skb_shinfo(skb)->frags[j].page_offset =
1774                                                         page_info->page_offset;
1775                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1776                         skb_shinfo(skb)->nr_frags++;
1777                 } else {
1778                         put_page(page_info->page);
1779                 }
1780
1781                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1782                 skb->len += curr_frag_len;
1783                 skb->data_len += curr_frag_len;
1784                 skb->truesize += rx_frag_size;
1785                 remaining -= curr_frag_len;
1786                 page_info->page = NULL;
1787         }
1788         BUG_ON(j > MAX_SKB_FRAGS);
1789 }
1790
1791 /* Process the RX completion indicated by rxcp when GRO is disabled */
1792 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1793                                 struct be_rx_compl_info *rxcp)
1794 {
1795         struct be_adapter *adapter = rxo->adapter;
1796         struct net_device *netdev = adapter->netdev;
1797         struct sk_buff *skb;
1798
1799         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1800         if (unlikely(!skb)) {
1801                 rx_stats(rxo)->rx_drops_no_skbs++;
1802                 be_rx_compl_discard(rxo, rxcp);
1803                 return;
1804         }
1805
1806         skb_fill_rx_data(rxo, skb, rxcp);
1807
1808         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1809                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1810         else
1811                 skb_checksum_none_assert(skb);
1812
1813         skb->protocol = eth_type_trans(skb, netdev);
1814         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1815         if (netdev->features & NETIF_F_RXHASH)
1816                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1817
1818         skb->csum_level = rxcp->tunneled;
1819         skb_mark_napi_id(skb, napi);
1820
1821         if (rxcp->vlanf)
1822                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1823
1824         netif_receive_skb(skb);
1825 }
1826
1827 /* Process the RX completion indicated by rxcp when GRO is enabled */
1828 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1829                                     struct napi_struct *napi,
1830                                     struct be_rx_compl_info *rxcp)
1831 {
1832         struct be_adapter *adapter = rxo->adapter;
1833         struct be_rx_page_info *page_info;
1834         struct sk_buff *skb = NULL;
1835         u16 remaining, curr_frag_len;
1836         u16 i, j;
1837
1838         skb = napi_get_frags(napi);
1839         if (!skb) {
1840                 be_rx_compl_discard(rxo, rxcp);
1841                 return;
1842         }
1843
1844         remaining = rxcp->pkt_size;
1845         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1846                 page_info = get_rx_page_info(rxo);
1847
1848                 curr_frag_len = min(remaining, rx_frag_size);
1849
1850                 /* Coalesce all frags from the same physical page in one slot */
1851                 if (i == 0 || page_info->page_offset == 0) {
1852                         /* First frag or Fresh page */
1853                         j++;
1854                         skb_frag_set_page(skb, j, page_info->page);
1855                         skb_shinfo(skb)->frags[j].page_offset =
1856                                                         page_info->page_offset;
1857                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1858                 } else {
1859                         put_page(page_info->page);
1860                 }
1861                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1862                 skb->truesize += rx_frag_size;
1863                 remaining -= curr_frag_len;
1864                 memset(page_info, 0, sizeof(*page_info));
1865         }
1866         BUG_ON(j > MAX_SKB_FRAGS);
1867
1868         skb_shinfo(skb)->nr_frags = j + 1;
1869         skb->len = rxcp->pkt_size;
1870         skb->data_len = rxcp->pkt_size;
1871         skb->ip_summed = CHECKSUM_UNNECESSARY;
1872         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1873         if (adapter->netdev->features & NETIF_F_RXHASH)
1874                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1875
1876         skb->csum_level = rxcp->tunneled;
1877         skb_mark_napi_id(skb, napi);
1878
1879         if (rxcp->vlanf)
1880                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1881
1882         napi_gro_frags(napi);
1883 }
1884
1885 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1886                                  struct be_rx_compl_info *rxcp)
1887 {
1888         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1889         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1890         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1891         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1892         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1893         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1894         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1895         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1896         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1897         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1898         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1899         if (rxcp->vlanf) {
1900                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1901                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1902         }
1903         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1904         rxcp->tunneled =
1905                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1906 }
1907
1908 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1909                                  struct be_rx_compl_info *rxcp)
1910 {
1911         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1912         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1913         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1914         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1915         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1916         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1917         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1918         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1919         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1920         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1921         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1922         if (rxcp->vlanf) {
1923                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1924                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1925         }
1926         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1927         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1928 }
1929
1930 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1931 {
1932         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1933         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1934         struct be_adapter *adapter = rxo->adapter;
1935
1936         /* For checking the valid bit it is Ok to use either definition as the
1937          * valid bit is at the same position in both v0 and v1 Rx compl */
1938         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1939                 return NULL;
1940
1941         rmb();
1942         be_dws_le_to_cpu(compl, sizeof(*compl));
1943
1944         if (adapter->be3_native)
1945                 be_parse_rx_compl_v1(compl, rxcp);
1946         else
1947                 be_parse_rx_compl_v0(compl, rxcp);
1948
1949         if (rxcp->ip_frag)
1950                 rxcp->l4_csum = 0;
1951
1952         if (rxcp->vlanf) {
1953                 /* In QNQ modes, if qnq bit is not set, then the packet was
1954                  * tagged only with the transparent outer vlan-tag and must
1955                  * not be treated as a vlan packet by host
1956                  */
1957                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1958                         rxcp->vlanf = 0;
1959
1960                 if (!lancer_chip(adapter))
1961                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1962
1963                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1964                     !test_bit(rxcp->vlan_tag, adapter->vids))
1965                         rxcp->vlanf = 0;
1966         }
1967
1968         /* As the compl has been parsed, reset it; we wont touch it again */
1969         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1970
1971         queue_tail_inc(&rxo->cq);
1972         return rxcp;
1973 }
1974
1975 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1976 {
1977         u32 order = get_order(size);
1978
1979         if (order > 0)
1980                 gfp |= __GFP_COMP;
1981         return  alloc_pages(gfp, order);
1982 }
1983
1984 /*
1985  * Allocate a page, split it to fragments of size rx_frag_size and post as
1986  * receive buffers to BE
1987  */
1988 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1989 {
1990         struct be_adapter *adapter = rxo->adapter;
1991         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1992         struct be_queue_info *rxq = &rxo->q;
1993         struct page *pagep = NULL;
1994         struct device *dev = &adapter->pdev->dev;
1995         struct be_eth_rx_d *rxd;
1996         u64 page_dmaaddr = 0, frag_dmaaddr;
1997         u32 posted, page_offset = 0, notify = 0;
1998
1999         page_info = &rxo->page_info_tbl[rxq->head];
2000         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
2001                 if (!pagep) {
2002                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
2003                         if (unlikely(!pagep)) {
2004                                 rx_stats(rxo)->rx_post_fail++;
2005                                 break;
2006                         }
2007                         page_dmaaddr = dma_map_page(dev, pagep, 0,
2008                                                     adapter->big_page_size,
2009                                                     DMA_FROM_DEVICE);
2010                         if (dma_mapping_error(dev, page_dmaaddr)) {
2011                                 put_page(pagep);
2012                                 pagep = NULL;
2013                                 adapter->drv_stats.dma_map_errors++;
2014                                 break;
2015                         }
2016                         page_offset = 0;
2017                 } else {
2018                         get_page(pagep);
2019                         page_offset += rx_frag_size;
2020                 }
2021                 page_info->page_offset = page_offset;
2022                 page_info->page = pagep;
2023
2024                 rxd = queue_head_node(rxq);
2025                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2026                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2027                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2028
2029                 /* Any space left in the current big page for another frag? */
2030                 if ((page_offset + rx_frag_size + rx_frag_size) >
2031                                         adapter->big_page_size) {
2032                         pagep = NULL;
2033                         page_info->last_frag = true;
2034                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2035                 } else {
2036                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2037                 }
2038
2039                 prev_page_info = page_info;
2040                 queue_head_inc(rxq);
2041                 page_info = &rxo->page_info_tbl[rxq->head];
2042         }
2043
2044         /* Mark the last frag of a page when we break out of the above loop
2045          * with no more slots available in the RXQ
2046          */
2047         if (pagep) {
2048                 prev_page_info->last_frag = true;
2049                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2050         }
2051
2052         if (posted) {
2053                 atomic_add(posted, &rxq->used);
2054                 if (rxo->rx_post_starved)
2055                         rxo->rx_post_starved = false;
2056                 do {
2057                         notify = min(256u, posted);
2058                         be_rxq_notify(adapter, rxq->id, notify);
2059                         posted -= notify;
2060                 } while (posted);
2061         } else if (atomic_read(&rxq->used) == 0) {
2062                 /* Let be_worker replenish when memory is available */
2063                 rxo->rx_post_starved = true;
2064         }
2065 }
2066
2067 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
2068 {
2069         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
2070
2071         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2072                 return NULL;
2073
2074         rmb();
2075         be_dws_le_to_cpu(txcp, sizeof(*txcp));
2076
2077         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2078
2079         queue_tail_inc(tx_cq);
2080         return txcp;
2081 }
2082
2083 static u16 be_tx_compl_process(struct be_adapter *adapter,
2084                                struct be_tx_obj *txo, u16 last_index)
2085 {
2086         struct sk_buff **sent_skbs = txo->sent_skb_list;
2087         struct be_queue_info *txq = &txo->q;
2088         u16 frag_index, num_wrbs = 0;
2089         struct sk_buff *skb = NULL;
2090         bool unmap_skb_hdr = false;
2091         struct be_eth_wrb *wrb;
2092
2093         do {
2094                 if (sent_skbs[txq->tail]) {
2095                         /* Free skb from prev req */
2096                         if (skb)
2097                                 dev_consume_skb_any(skb);
2098                         skb = sent_skbs[txq->tail];
2099                         sent_skbs[txq->tail] = NULL;
2100                         queue_tail_inc(txq);  /* skip hdr wrb */
2101                         num_wrbs++;
2102                         unmap_skb_hdr = true;
2103                 }
2104                 wrb = queue_tail_node(txq);
2105                 frag_index = txq->tail;
2106                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2107                               (unmap_skb_hdr && skb_headlen(skb)));
2108                 unmap_skb_hdr = false;
2109                 queue_tail_inc(txq);
2110                 num_wrbs++;
2111         } while (frag_index != last_index);
2112         dev_consume_skb_any(skb);
2113
2114         return num_wrbs;
2115 }
2116
2117 /* Return the number of events in the event queue */
2118 static inline int events_get(struct be_eq_obj *eqo)
2119 {
2120         struct be_eq_entry *eqe;
2121         int num = 0;
2122
2123         do {
2124                 eqe = queue_tail_node(&eqo->q);
2125                 if (eqe->evt == 0)
2126                         break;
2127
2128                 rmb();
2129                 eqe->evt = 0;
2130                 num++;
2131                 queue_tail_inc(&eqo->q);
2132         } while (true);
2133
2134         return num;
2135 }
2136
2137 /* Leaves the EQ is disarmed state */
2138 static void be_eq_clean(struct be_eq_obj *eqo)
2139 {
2140         int num = events_get(eqo);
2141
2142         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2143 }
2144
2145 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2146 {
2147         struct be_rx_page_info *page_info;
2148         struct be_queue_info *rxq = &rxo->q;
2149         struct be_queue_info *rx_cq = &rxo->cq;
2150         struct be_rx_compl_info *rxcp;
2151         struct be_adapter *adapter = rxo->adapter;
2152         int flush_wait = 0;
2153
2154         /* Consume pending rx completions.
2155          * Wait for the flush completion (identified by zero num_rcvd)
2156          * to arrive. Notify CQ even when there are no more CQ entries
2157          * for HW to flush partially coalesced CQ entries.
2158          * In Lancer, there is no need to wait for flush compl.
2159          */
2160         for (;;) {
2161                 rxcp = be_rx_compl_get(rxo);
2162                 if (!rxcp) {
2163                         if (lancer_chip(adapter))
2164                                 break;
2165
2166                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2167                                 dev_warn(&adapter->pdev->dev,
2168                                          "did not receive flush compl\n");
2169                                 break;
2170                         }
2171                         be_cq_notify(adapter, rx_cq->id, true, 0);
2172                         mdelay(1);
2173                 } else {
2174                         be_rx_compl_discard(rxo, rxcp);
2175                         be_cq_notify(adapter, rx_cq->id, false, 1);
2176                         if (rxcp->num_rcvd == 0)
2177                                 break;
2178                 }
2179         }
2180
2181         /* After cleanup, leave the CQ in unarmed state */
2182         be_cq_notify(adapter, rx_cq->id, false, 0);
2183
2184         /* Then free posted rx buffers that were not used */
2185         while (atomic_read(&rxq->used) > 0) {
2186                 page_info = get_rx_page_info(rxo);
2187                 put_page(page_info->page);
2188                 memset(page_info, 0, sizeof(*page_info));
2189         }
2190         BUG_ON(atomic_read(&rxq->used));
2191         rxq->tail = 0;
2192         rxq->head = 0;
2193 }
2194
2195 static void be_tx_compl_clean(struct be_adapter *adapter)
2196 {
2197         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2198         struct device *dev = &adapter->pdev->dev;
2199         struct be_tx_obj *txo;
2200         struct be_queue_info *txq;
2201         struct be_eth_tx_compl *txcp;
2202         int i, pending_txqs;
2203
2204         /* Stop polling for compls when HW has been silent for 10ms */
2205         do {
2206                 pending_txqs = adapter->num_tx_qs;
2207
2208                 for_all_tx_queues(adapter, txo, i) {
2209                         cmpl = 0;
2210                         num_wrbs = 0;
2211                         txq = &txo->q;
2212                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2213                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2214                                 num_wrbs += be_tx_compl_process(adapter, txo,
2215                                                                 end_idx);
2216                                 cmpl++;
2217                         }
2218                         if (cmpl) {
2219                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2220                                 atomic_sub(num_wrbs, &txq->used);
2221                                 timeo = 0;
2222                         }
2223                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2224                                 pending_txqs--;
2225                 }
2226
2227                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2228                         break;
2229
2230                 mdelay(1);
2231         } while (true);
2232
2233         /* Free enqueued TX that was never notified to HW */
2234         for_all_tx_queues(adapter, txo, i) {
2235                 txq = &txo->q;
2236
2237                 if (atomic_read(&txq->used)) {
2238                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2239                                  i, atomic_read(&txq->used));
2240                         notified_idx = txq->tail;
2241                         end_idx = txq->tail;
2242                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2243                                   txq->len);
2244                         /* Use the tx-compl process logic to handle requests
2245                          * that were not sent to the HW.
2246                          */
2247                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2248                         atomic_sub(num_wrbs, &txq->used);
2249                         BUG_ON(atomic_read(&txq->used));
2250                         txo->pend_wrb_cnt = 0;
2251                         /* Since hw was never notified of these requests,
2252                          * reset TXQ indices
2253                          */
2254                         txq->head = notified_idx;
2255                         txq->tail = notified_idx;
2256                 }
2257         }
2258 }
2259
2260 static void be_evt_queues_destroy(struct be_adapter *adapter)
2261 {
2262         struct be_eq_obj *eqo;
2263         int i;
2264
2265         for_all_evt_queues(adapter, eqo, i) {
2266                 if (eqo->q.created) {
2267                         be_eq_clean(eqo);
2268                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2269                         napi_hash_del(&eqo->napi);
2270                         netif_napi_del(&eqo->napi);
2271                 }
2272                 be_queue_free(adapter, &eqo->q);
2273         }
2274 }
2275
2276 static int be_evt_queues_create(struct be_adapter *adapter)
2277 {
2278         struct be_queue_info *eq;
2279         struct be_eq_obj *eqo;
2280         struct be_aic_obj *aic;
2281         int i, rc;
2282
2283         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2284                                     adapter->cfg_num_qs);
2285
2286         for_all_evt_queues(adapter, eqo, i) {
2287                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2288                                BE_NAPI_WEIGHT);
2289                 napi_hash_add(&eqo->napi);
2290                 aic = &adapter->aic_obj[i];
2291                 eqo->adapter = adapter;
2292                 eqo->idx = i;
2293                 aic->max_eqd = BE_MAX_EQD;
2294                 aic->enable = true;
2295
2296                 eq = &eqo->q;
2297                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2298                                     sizeof(struct be_eq_entry));
2299                 if (rc)
2300                         return rc;
2301
2302                 rc = be_cmd_eq_create(adapter, eqo);
2303                 if (rc)
2304                         return rc;
2305         }
2306         return 0;
2307 }
2308
2309 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2310 {
2311         struct be_queue_info *q;
2312
2313         q = &adapter->mcc_obj.q;
2314         if (q->created)
2315                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2316         be_queue_free(adapter, q);
2317
2318         q = &adapter->mcc_obj.cq;
2319         if (q->created)
2320                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2321         be_queue_free(adapter, q);
2322 }
2323
2324 /* Must be called only after TX qs are created as MCC shares TX EQ */
2325 static int be_mcc_queues_create(struct be_adapter *adapter)
2326 {
2327         struct be_queue_info *q, *cq;
2328
2329         cq = &adapter->mcc_obj.cq;
2330         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2331                            sizeof(struct be_mcc_compl)))
2332                 goto err;
2333
2334         /* Use the default EQ for MCC completions */
2335         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2336                 goto mcc_cq_free;
2337
2338         q = &adapter->mcc_obj.q;
2339         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2340                 goto mcc_cq_destroy;
2341
2342         if (be_cmd_mccq_create(adapter, q, cq))
2343                 goto mcc_q_free;
2344
2345         return 0;
2346
2347 mcc_q_free:
2348         be_queue_free(adapter, q);
2349 mcc_cq_destroy:
2350         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2351 mcc_cq_free:
2352         be_queue_free(adapter, cq);
2353 err:
2354         return -1;
2355 }
2356
2357 static void be_tx_queues_destroy(struct be_adapter *adapter)
2358 {
2359         struct be_queue_info *q;
2360         struct be_tx_obj *txo;
2361         u8 i;
2362
2363         for_all_tx_queues(adapter, txo, i) {
2364                 q = &txo->q;
2365                 if (q->created)
2366                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2367                 be_queue_free(adapter, q);
2368
2369                 q = &txo->cq;
2370                 if (q->created)
2371                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2372                 be_queue_free(adapter, q);
2373         }
2374 }
2375
2376 static int be_tx_qs_create(struct be_adapter *adapter)
2377 {
2378         struct be_queue_info *cq, *eq;
2379         struct be_tx_obj *txo;
2380         int status, i;
2381
2382         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2383
2384         for_all_tx_queues(adapter, txo, i) {
2385                 cq = &txo->cq;
2386                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2387                                         sizeof(struct be_eth_tx_compl));
2388                 if (status)
2389                         return status;
2390
2391                 u64_stats_init(&txo->stats.sync);
2392                 u64_stats_init(&txo->stats.sync_compl);
2393
2394                 /* If num_evt_qs is less than num_tx_qs, then more than
2395                  * one txq share an eq
2396                  */
2397                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2398                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2399                 if (status)
2400                         return status;
2401
2402                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2403                                         sizeof(struct be_eth_wrb));
2404                 if (status)
2405                         return status;
2406
2407                 status = be_cmd_txq_create(adapter, txo);
2408                 if (status)
2409                         return status;
2410         }
2411
2412         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2413                  adapter->num_tx_qs);
2414         return 0;
2415 }
2416
2417 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2418 {
2419         struct be_queue_info *q;
2420         struct be_rx_obj *rxo;
2421         int i;
2422
2423         for_all_rx_queues(adapter, rxo, i) {
2424                 q = &rxo->cq;
2425                 if (q->created)
2426                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2427                 be_queue_free(adapter, q);
2428         }
2429 }
2430
2431 static int be_rx_cqs_create(struct be_adapter *adapter)
2432 {
2433         struct be_queue_info *eq, *cq;
2434         struct be_rx_obj *rxo;
2435         int rc, i;
2436
2437         /* We can create as many RSS rings as there are EQs. */
2438         adapter->num_rx_qs = adapter->num_evt_qs;
2439
2440         /* We'll use RSS only if atleast 2 RSS rings are supported.
2441          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2442          */
2443         if (adapter->num_rx_qs > 1)
2444                 adapter->num_rx_qs++;
2445
2446         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2447         for_all_rx_queues(adapter, rxo, i) {
2448                 rxo->adapter = adapter;
2449                 cq = &rxo->cq;
2450                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2451                                     sizeof(struct be_eth_rx_compl));
2452                 if (rc)
2453                         return rc;
2454
2455                 u64_stats_init(&rxo->stats.sync);
2456                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2457                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2458                 if (rc)
2459                         return rc;
2460         }
2461
2462         dev_info(&adapter->pdev->dev,
2463                  "created %d RSS queue(s) and 1 default RX queue\n",
2464                  adapter->num_rx_qs - 1);
2465         return 0;
2466 }
2467
2468 static irqreturn_t be_intx(int irq, void *dev)
2469 {
2470         struct be_eq_obj *eqo = dev;
2471         struct be_adapter *adapter = eqo->adapter;
2472         int num_evts = 0;
2473
2474         /* IRQ is not expected when NAPI is scheduled as the EQ
2475          * will not be armed.
2476          * But, this can happen on Lancer INTx where it takes
2477          * a while to de-assert INTx or in BE2 where occasionaly
2478          * an interrupt may be raised even when EQ is unarmed.
2479          * If NAPI is already scheduled, then counting & notifying
2480          * events will orphan them.
2481          */
2482         if (napi_schedule_prep(&eqo->napi)) {
2483                 num_evts = events_get(eqo);
2484                 __napi_schedule(&eqo->napi);
2485                 if (num_evts)
2486                         eqo->spurious_intr = 0;
2487         }
2488         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2489
2490         /* Return IRQ_HANDLED only for the the first spurious intr
2491          * after a valid intr to stop the kernel from branding
2492          * this irq as a bad one!
2493          */
2494         if (num_evts || eqo->spurious_intr++ == 0)
2495                 return IRQ_HANDLED;
2496         else
2497                 return IRQ_NONE;
2498 }
2499
2500 static irqreturn_t be_msix(int irq, void *dev)
2501 {
2502         struct be_eq_obj *eqo = dev;
2503
2504         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2505         napi_schedule(&eqo->napi);
2506         return IRQ_HANDLED;
2507 }
2508
2509 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2510 {
2511         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2512 }
2513
2514 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2515                          int budget, int polling)
2516 {
2517         struct be_adapter *adapter = rxo->adapter;
2518         struct be_queue_info *rx_cq = &rxo->cq;
2519         struct be_rx_compl_info *rxcp;
2520         u32 work_done;
2521         u32 frags_consumed = 0;
2522
2523         for (work_done = 0; work_done < budget; work_done++) {
2524                 rxcp = be_rx_compl_get(rxo);
2525                 if (!rxcp)
2526                         break;
2527
2528                 /* Is it a flush compl that has no data */
2529                 if (unlikely(rxcp->num_rcvd == 0))
2530                         goto loop_continue;
2531
2532                 /* Discard compl with partial DMA Lancer B0 */
2533                 if (unlikely(!rxcp->pkt_size)) {
2534                         be_rx_compl_discard(rxo, rxcp);
2535                         goto loop_continue;
2536                 }
2537
2538                 /* On BE drop pkts that arrive due to imperfect filtering in
2539                  * promiscuous mode on some skews
2540                  */
2541                 if (unlikely(rxcp->port != adapter->port_num &&
2542                              !lancer_chip(adapter))) {
2543                         be_rx_compl_discard(rxo, rxcp);
2544                         goto loop_continue;
2545                 }
2546
2547                 /* Don't do gro when we're busy_polling */
2548                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2549                         be_rx_compl_process_gro(rxo, napi, rxcp);
2550                 else
2551                         be_rx_compl_process(rxo, napi, rxcp);
2552
2553 loop_continue:
2554                 frags_consumed += rxcp->num_rcvd;
2555                 be_rx_stats_update(rxo, rxcp);
2556         }
2557
2558         if (work_done) {
2559                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2560
2561                 /* When an rx-obj gets into post_starved state, just
2562                  * let be_worker do the posting.
2563                  */
2564                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2565                     !rxo->rx_post_starved)
2566                         be_post_rx_frags(rxo, GFP_ATOMIC,
2567                                          max_t(u32, MAX_RX_POST,
2568                                                frags_consumed));
2569         }
2570
2571         return work_done;
2572 }
2573
2574 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2575 {
2576         switch (status) {
2577         case BE_TX_COMP_HDR_PARSE_ERR:
2578                 tx_stats(txo)->tx_hdr_parse_err++;
2579                 break;
2580         case BE_TX_COMP_NDMA_ERR:
2581                 tx_stats(txo)->tx_dma_err++;
2582                 break;
2583         case BE_TX_COMP_ACL_ERR:
2584                 tx_stats(txo)->tx_spoof_check_err++;
2585                 break;
2586         }
2587 }
2588
2589 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2590 {
2591         switch (status) {
2592         case LANCER_TX_COMP_LSO_ERR:
2593                 tx_stats(txo)->tx_tso_err++;
2594                 break;
2595         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2596         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2597                 tx_stats(txo)->tx_spoof_check_err++;
2598                 break;
2599         case LANCER_TX_COMP_QINQ_ERR:
2600                 tx_stats(txo)->tx_qinq_err++;
2601                 break;
2602         case LANCER_TX_COMP_PARITY_ERR:
2603                 tx_stats(txo)->tx_internal_parity_err++;
2604                 break;
2605         case LANCER_TX_COMP_DMA_ERR:
2606                 tx_stats(txo)->tx_dma_err++;
2607                 break;
2608         }
2609 }
2610
2611 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2612                           int idx)
2613 {
2614         struct be_eth_tx_compl *txcp;
2615         int num_wrbs = 0, work_done = 0;
2616         u32 compl_status;
2617         u16 last_idx;
2618
2619         while ((txcp = be_tx_compl_get(&txo->cq))) {
2620                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2621                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2622                 work_done++;
2623
2624                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2625                 if (compl_status) {
2626                         if (lancer_chip(adapter))
2627                                 lancer_update_tx_err(txo, compl_status);
2628                         else
2629                                 be_update_tx_err(txo, compl_status);
2630                 }
2631         }
2632
2633         if (work_done) {
2634                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2635                 atomic_sub(num_wrbs, &txo->q.used);
2636
2637                 /* As Tx wrbs have been freed up, wake up netdev queue
2638                  * if it was stopped due to lack of tx wrbs.  */
2639                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2640                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2641                         netif_wake_subqueue(adapter->netdev, idx);
2642                 }
2643
2644                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2645                 tx_stats(txo)->tx_compl += work_done;
2646                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2647         }
2648 }
2649
2650 #ifdef CONFIG_NET_RX_BUSY_POLL
2651 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2652 {
2653         bool status = true;
2654
2655         spin_lock(&eqo->lock); /* BH is already disabled */
2656         if (eqo->state & BE_EQ_LOCKED) {
2657                 WARN_ON(eqo->state & BE_EQ_NAPI);
2658                 eqo->state |= BE_EQ_NAPI_YIELD;
2659                 status = false;
2660         } else {
2661                 eqo->state = BE_EQ_NAPI;
2662         }
2663         spin_unlock(&eqo->lock);
2664         return status;
2665 }
2666
2667 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2668 {
2669         spin_lock(&eqo->lock); /* BH is already disabled */
2670
2671         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2672         eqo->state = BE_EQ_IDLE;
2673
2674         spin_unlock(&eqo->lock);
2675 }
2676
2677 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2678 {
2679         bool status = true;
2680
2681         spin_lock_bh(&eqo->lock);
2682         if (eqo->state & BE_EQ_LOCKED) {
2683                 eqo->state |= BE_EQ_POLL_YIELD;
2684                 status = false;
2685         } else {
2686                 eqo->state |= BE_EQ_POLL;
2687         }
2688         spin_unlock_bh(&eqo->lock);
2689         return status;
2690 }
2691
2692 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2693 {
2694         spin_lock_bh(&eqo->lock);
2695
2696         WARN_ON(eqo->state & (BE_EQ_NAPI));
2697         eqo->state = BE_EQ_IDLE;
2698
2699         spin_unlock_bh(&eqo->lock);
2700 }
2701
2702 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2703 {
2704         spin_lock_init(&eqo->lock);
2705         eqo->state = BE_EQ_IDLE;
2706 }
2707
2708 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2709 {
2710         local_bh_disable();
2711
2712         /* It's enough to just acquire napi lock on the eqo to stop
2713          * be_busy_poll() from processing any queueus.
2714          */
2715         while (!be_lock_napi(eqo))
2716                 mdelay(1);
2717
2718         local_bh_enable();
2719 }
2720
2721 #else /* CONFIG_NET_RX_BUSY_POLL */
2722
2723 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2724 {
2725         return true;
2726 }
2727
2728 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2729 {
2730 }
2731
2732 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2733 {
2734         return false;
2735 }
2736
2737 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2738 {
2739 }
2740
2741 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2742 {
2743 }
2744
2745 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2746 {
2747 }
2748 #endif /* CONFIG_NET_RX_BUSY_POLL */
2749
2750 int be_poll(struct napi_struct *napi, int budget)
2751 {
2752         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2753         struct be_adapter *adapter = eqo->adapter;
2754         int max_work = 0, work, i, num_evts;
2755         struct be_rx_obj *rxo;
2756         struct be_tx_obj *txo;
2757
2758         num_evts = events_get(eqo);
2759
2760         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2761                 be_process_tx(adapter, txo, i);
2762
2763         if (be_lock_napi(eqo)) {
2764                 /* This loop will iterate twice for EQ0 in which
2765                  * completions of the last RXQ (default one) are also processed
2766                  * For other EQs the loop iterates only once
2767                  */
2768                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2769                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2770                         max_work = max(work, max_work);
2771                 }
2772                 be_unlock_napi(eqo);
2773         } else {
2774                 max_work = budget;
2775         }
2776
2777         if (is_mcc_eqo(eqo))
2778                 be_process_mcc(adapter);
2779
2780         if (max_work < budget) {
2781                 napi_complete(napi);
2782                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2783         } else {
2784                 /* As we'll continue in polling mode, count and clear events */
2785                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2786         }
2787         return max_work;
2788 }
2789
2790 #ifdef CONFIG_NET_RX_BUSY_POLL
2791 static int be_busy_poll(struct napi_struct *napi)
2792 {
2793         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2794         struct be_adapter *adapter = eqo->adapter;
2795         struct be_rx_obj *rxo;
2796         int i, work = 0;
2797
2798         if (!be_lock_busy_poll(eqo))
2799                 return LL_FLUSH_BUSY;
2800
2801         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2802                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2803                 if (work)
2804                         break;
2805         }
2806
2807         be_unlock_busy_poll(eqo);
2808         return work;
2809 }
2810 #endif
2811
2812 void be_detect_error(struct be_adapter *adapter)
2813 {
2814         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2815         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2816         u32 i;
2817         bool error_detected = false;
2818         struct device *dev = &adapter->pdev->dev;
2819         struct net_device *netdev = adapter->netdev;
2820
2821         if (be_hw_error(adapter))
2822                 return;
2823
2824         if (lancer_chip(adapter)) {
2825                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2826                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2827                         sliport_err1 = ioread32(adapter->db +
2828                                                 SLIPORT_ERROR1_OFFSET);
2829                         sliport_err2 = ioread32(adapter->db +
2830                                                 SLIPORT_ERROR2_OFFSET);
2831                         adapter->hw_error = true;
2832                         /* Do not log error messages if its a FW reset */
2833                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2834                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2835                                 dev_info(dev, "Firmware update in progress\n");
2836                         } else {
2837                                 error_detected = true;
2838                                 dev_err(dev, "Error detected in the card\n");
2839                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2840                                         sliport_status);
2841                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2842                                         sliport_err1);
2843                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2844                                         sliport_err2);
2845                         }
2846                 }
2847         } else {
2848                 pci_read_config_dword(adapter->pdev,
2849                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2850                 pci_read_config_dword(adapter->pdev,
2851                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2852                 pci_read_config_dword(adapter->pdev,
2853                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2854                 pci_read_config_dword(adapter->pdev,
2855                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2856
2857                 ue_lo = (ue_lo & ~ue_lo_mask);
2858                 ue_hi = (ue_hi & ~ue_hi_mask);
2859
2860                 /* On certain platforms BE hardware can indicate spurious UEs.
2861                  * Allow HW to stop working completely in case of a real UE.
2862                  * Hence not setting the hw_error for UE detection.
2863                  */
2864
2865                 if (ue_lo || ue_hi) {
2866                         error_detected = true;
2867                         dev_err(dev,
2868                                 "Unrecoverable Error detected in the adapter");
2869                         dev_err(dev, "Please reboot server to recover");
2870                         if (skyhawk_chip(adapter))
2871                                 adapter->hw_error = true;
2872                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2873                                 if (ue_lo & 1)
2874                                         dev_err(dev, "UE: %s bit set\n",
2875                                                 ue_status_low_desc[i]);
2876                         }
2877                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2878                                 if (ue_hi & 1)
2879                                         dev_err(dev, "UE: %s bit set\n",
2880                                                 ue_status_hi_desc[i]);
2881                         }
2882                 }
2883         }
2884         if (error_detected)
2885                 netif_carrier_off(netdev);
2886 }
2887
2888 static void be_msix_disable(struct be_adapter *adapter)
2889 {
2890         if (msix_enabled(adapter)) {
2891                 pci_disable_msix(adapter->pdev);
2892                 adapter->num_msix_vec = 0;
2893                 adapter->num_msix_roce_vec = 0;
2894         }
2895 }
2896
2897 static int be_msix_enable(struct be_adapter *adapter)
2898 {
2899         int i, num_vec;
2900         struct device *dev = &adapter->pdev->dev;
2901
2902         /* If RoCE is supported, program the max number of NIC vectors that
2903          * may be configured via set-channels, along with vectors needed for
2904          * RoCe. Else, just program the number we'll use initially.
2905          */
2906         if (be_roce_supported(adapter))
2907                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2908                                 2 * num_online_cpus());
2909         else
2910                 num_vec = adapter->cfg_num_qs;
2911
2912         for (i = 0; i < num_vec; i++)
2913                 adapter->msix_entries[i].entry = i;
2914
2915         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2916                                         MIN_MSIX_VECTORS, num_vec);
2917         if (num_vec < 0)
2918                 goto fail;
2919
2920         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2921                 adapter->num_msix_roce_vec = num_vec / 2;
2922                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2923                          adapter->num_msix_roce_vec);
2924         }
2925
2926         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2927
2928         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2929                  adapter->num_msix_vec);
2930         return 0;
2931
2932 fail:
2933         dev_warn(dev, "MSIx enable failed\n");
2934
2935         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2936         if (!be_physfn(adapter))
2937                 return num_vec;
2938         return 0;
2939 }
2940
2941 static inline int be_msix_vec_get(struct be_adapter *adapter,
2942                                   struct be_eq_obj *eqo)
2943 {
2944         return adapter->msix_entries[eqo->msix_idx].vector;
2945 }
2946
2947 static int be_msix_register(struct be_adapter *adapter)
2948 {
2949         struct net_device *netdev = adapter->netdev;
2950         struct be_eq_obj *eqo;
2951         int status, i, vec;
2952
2953         for_all_evt_queues(adapter, eqo, i) {
2954                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2955                 vec = be_msix_vec_get(adapter, eqo);
2956                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2957                 if (status)
2958                         goto err_msix;
2959         }
2960
2961         return 0;
2962 err_msix:
2963         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2964                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2965         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2966                  status);
2967         be_msix_disable(adapter);
2968         return status;
2969 }
2970
2971 static int be_irq_register(struct be_adapter *adapter)
2972 {
2973         struct net_device *netdev = adapter->netdev;
2974         int status;
2975
2976         if (msix_enabled(adapter)) {
2977                 status = be_msix_register(adapter);
2978                 if (status == 0)
2979                         goto done;
2980                 /* INTx is not supported for VF */
2981                 if (!be_physfn(adapter))
2982                         return status;
2983         }
2984
2985         /* INTx: only the first EQ is used */
2986         netdev->irq = adapter->pdev->irq;
2987         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2988                              &adapter->eq_obj[0]);
2989         if (status) {
2990                 dev_err(&adapter->pdev->dev,
2991                         "INTx request IRQ failed - err %d\n", status);
2992                 return status;
2993         }
2994 done:
2995         adapter->isr_registered = true;
2996         return 0;
2997 }
2998
2999 static void be_irq_unregister(struct be_adapter *adapter)
3000 {
3001         struct net_device *netdev = adapter->netdev;
3002         struct be_eq_obj *eqo;
3003         int i;
3004
3005         if (!adapter->isr_registered)
3006                 return;
3007
3008         /* INTx */
3009         if (!msix_enabled(adapter)) {
3010                 free_irq(netdev->irq, &adapter->eq_obj[0]);
3011                 goto done;
3012         }
3013
3014         /* MSIx */
3015         for_all_evt_queues(adapter, eqo, i)
3016                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
3017
3018 done:
3019         adapter->isr_registered = false;
3020 }
3021
3022 static void be_rx_qs_destroy(struct be_adapter *adapter)
3023 {
3024         struct be_queue_info *q;
3025         struct be_rx_obj *rxo;
3026         int i;
3027
3028         for_all_rx_queues(adapter, rxo, i) {
3029                 q = &rxo->q;
3030                 if (q->created) {
3031                         be_cmd_rxq_destroy(adapter, q);
3032                         be_rx_cq_clean(rxo);
3033                 }
3034                 be_queue_free(adapter, q);
3035         }
3036 }
3037
3038 static int be_close(struct net_device *netdev)
3039 {
3040         struct be_adapter *adapter = netdev_priv(netdev);
3041         struct be_eq_obj *eqo;
3042         int i;
3043
3044         /* This protection is needed as be_close() may be called even when the
3045          * adapter is in cleared state (after eeh perm failure)
3046          */
3047         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3048                 return 0;
3049
3050         be_roce_dev_close(adapter);
3051
3052         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3053                 for_all_evt_queues(adapter, eqo, i) {
3054                         napi_disable(&eqo->napi);
3055                         be_disable_busy_poll(eqo);
3056                 }
3057                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3058         }
3059
3060         be_async_mcc_disable(adapter);
3061
3062         /* Wait for all pending tx completions to arrive so that
3063          * all tx skbs are freed.
3064          */
3065         netif_tx_disable(netdev);
3066         be_tx_compl_clean(adapter);
3067
3068         be_rx_qs_destroy(adapter);
3069         be_clear_uc_list(adapter);
3070
3071         for_all_evt_queues(adapter, eqo, i) {
3072                 if (msix_enabled(adapter))
3073                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3074                 else
3075                         synchronize_irq(netdev->irq);
3076                 be_eq_clean(eqo);
3077         }
3078
3079         be_irq_unregister(adapter);
3080
3081         return 0;
3082 }
3083
3084 static int be_rx_qs_create(struct be_adapter *adapter)
3085 {
3086         struct rss_info *rss = &adapter->rss_info;
3087         u8 rss_key[RSS_HASH_KEY_LEN];
3088         struct be_rx_obj *rxo;
3089         int rc, i, j;
3090
3091         for_all_rx_queues(adapter, rxo, i) {
3092                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3093                                     sizeof(struct be_eth_rx_d));
3094                 if (rc)
3095                         return rc;
3096         }
3097
3098         /* The FW would like the default RXQ to be created first */
3099         rxo = default_rxo(adapter);
3100         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
3101                                adapter->if_handle, false, &rxo->rss_id);
3102         if (rc)
3103                 return rc;
3104
3105         for_all_rss_queues(adapter, rxo, i) {
3106                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3107                                        rx_frag_size, adapter->if_handle,
3108                                        true, &rxo->rss_id);
3109                 if (rc)
3110                         return rc;
3111         }
3112
3113         if (be_multi_rxq(adapter)) {
3114                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
3115                         j += adapter->num_rx_qs - 1) {
3116                         for_all_rss_queues(adapter, rxo, i) {
3117                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3118                                         break;
3119                                 rss->rsstable[j + i] = rxo->rss_id;
3120                                 rss->rss_queue[j + i] = i;
3121                         }
3122                 }
3123                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3124                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3125
3126                 if (!BEx_chip(adapter))
3127                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3128                                 RSS_ENABLE_UDP_IPV6;
3129         } else {
3130                 /* Disable RSS, if only default RX Q is created */
3131                 rss->rss_flags = RSS_ENABLE_NONE;
3132         }
3133
3134         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3135         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3136                                128, rss_key);
3137         if (rc) {
3138                 rss->rss_flags = RSS_ENABLE_NONE;
3139                 return rc;
3140         }
3141
3142         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3143
3144         /* First time posting */
3145         for_all_rx_queues(adapter, rxo, i)
3146                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
3147         return 0;
3148 }
3149
3150 static int be_open(struct net_device *netdev)
3151 {
3152         struct be_adapter *adapter = netdev_priv(netdev);
3153         struct be_eq_obj *eqo;
3154         struct be_rx_obj *rxo;
3155         struct be_tx_obj *txo;
3156         u8 link_status;
3157         int status, i;
3158
3159         status = be_rx_qs_create(adapter);
3160         if (status)
3161                 goto err;
3162
3163         status = be_irq_register(adapter);
3164         if (status)
3165                 goto err;
3166
3167         for_all_rx_queues(adapter, rxo, i)
3168                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3169
3170         for_all_tx_queues(adapter, txo, i)
3171                 be_cq_notify(adapter, txo->cq.id, true, 0);
3172
3173         be_async_mcc_enable(adapter);
3174
3175         for_all_evt_queues(adapter, eqo, i) {
3176                 napi_enable(&eqo->napi);
3177                 be_enable_busy_poll(eqo);
3178                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
3179         }
3180         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3181
3182         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3183         if (!status)
3184                 be_link_status_update(adapter, link_status);
3185
3186         netif_tx_start_all_queues(netdev);
3187         be_roce_dev_open(adapter);
3188
3189 #ifdef CONFIG_BE2NET_VXLAN
3190         if (skyhawk_chip(adapter))
3191                 vxlan_get_rx_port(netdev);
3192 #endif
3193
3194         return 0;
3195 err:
3196         be_close(adapter->netdev);
3197         return -EIO;
3198 }
3199
3200 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3201 {
3202         struct be_dma_mem cmd;
3203         int status = 0;
3204         u8 mac[ETH_ALEN];
3205
3206         memset(mac, 0, ETH_ALEN);
3207
3208         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3209         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3210                                      GFP_KERNEL);
3211         if (!cmd.va)
3212                 return -ENOMEM;
3213
3214         if (enable) {
3215                 status = pci_write_config_dword(adapter->pdev,
3216                                                 PCICFG_PM_CONTROL_OFFSET,
3217                                                 PCICFG_PM_CONTROL_MASK);
3218                 if (status) {
3219                         dev_err(&adapter->pdev->dev,
3220                                 "Could not enable Wake-on-lan\n");
3221                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3222                                           cmd.dma);
3223                         return status;
3224                 }
3225                 status = be_cmd_enable_magic_wol(adapter,
3226                                                  adapter->netdev->dev_addr,
3227                                                  &cmd);
3228                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3229                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3230         } else {
3231                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3232                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3233                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3234         }
3235
3236         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3237         return status;
3238 }
3239
3240 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3241 {
3242         u32 addr;
3243
3244         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3245
3246         mac[5] = (u8)(addr & 0xFF);
3247         mac[4] = (u8)((addr >> 8) & 0xFF);
3248         mac[3] = (u8)((addr >> 16) & 0xFF);
3249         /* Use the OUI from the current MAC address */
3250         memcpy(mac, adapter->netdev->dev_addr, 3);
3251 }
3252
3253 /*
3254  * Generate a seed MAC address from the PF MAC Address using jhash.
3255  * MAC Address for VFs are assigned incrementally starting from the seed.
3256  * These addresses are programmed in the ASIC by the PF and the VF driver
3257  * queries for the MAC address during its probe.
3258  */
3259 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3260 {
3261         u32 vf;
3262         int status = 0;
3263         u8 mac[ETH_ALEN];
3264         struct be_vf_cfg *vf_cfg;
3265
3266         be_vf_eth_addr_generate(adapter, mac);
3267
3268         for_all_vfs(adapter, vf_cfg, vf) {
3269                 if (BEx_chip(adapter))
3270                         status = be_cmd_pmac_add(adapter, mac,
3271                                                  vf_cfg->if_handle,
3272                                                  &vf_cfg->pmac_id, vf + 1);
3273                 else
3274                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3275                                                 vf + 1);
3276
3277                 if (status)
3278                         dev_err(&adapter->pdev->dev,
3279                                 "Mac address assignment failed for VF %d\n",
3280                                 vf);
3281                 else
3282                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3283
3284                 mac[5] += 1;
3285         }
3286         return status;
3287 }
3288
3289 static int be_vfs_mac_query(struct be_adapter *adapter)
3290 {
3291         int status, vf;
3292         u8 mac[ETH_ALEN];
3293         struct be_vf_cfg *vf_cfg;
3294
3295         for_all_vfs(adapter, vf_cfg, vf) {
3296                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3297                                                mac, vf_cfg->if_handle,
3298                                                false, vf+1);
3299                 if (status)
3300                         return status;
3301                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3302         }
3303         return 0;
3304 }
3305
3306 static void be_vf_clear(struct be_adapter *adapter)
3307 {
3308         struct be_vf_cfg *vf_cfg;
3309         u32 vf;
3310
3311         if (pci_vfs_assigned(adapter->pdev)) {
3312                 dev_warn(&adapter->pdev->dev,
3313                          "VFs are assigned to VMs: not disabling VFs\n");
3314                 goto done;
3315         }
3316
3317         pci_disable_sriov(adapter->pdev);
3318
3319         for_all_vfs(adapter, vf_cfg, vf) {
3320                 if (BEx_chip(adapter))
3321                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3322                                         vf_cfg->pmac_id, vf + 1);
3323                 else
3324                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3325                                        vf + 1);
3326
3327                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3328         }
3329 done:
3330         kfree(adapter->vf_cfg);
3331         adapter->num_vfs = 0;
3332         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3333 }
3334
3335 static void be_clear_queues(struct be_adapter *adapter)
3336 {
3337         be_mcc_queues_destroy(adapter);
3338         be_rx_cqs_destroy(adapter);
3339         be_tx_queues_destroy(adapter);
3340         be_evt_queues_destroy(adapter);
3341 }
3342
3343 static void be_cancel_worker(struct be_adapter *adapter)
3344 {
3345         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3346                 cancel_delayed_work_sync(&adapter->work);
3347                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3348         }
3349 }
3350
3351 static void be_mac_clear(struct be_adapter *adapter)
3352 {
3353         if (adapter->pmac_id) {
3354                 be_cmd_pmac_del(adapter, adapter->if_handle,
3355                                 adapter->pmac_id[0], 0);
3356                 kfree(adapter->pmac_id);
3357                 adapter->pmac_id = NULL;
3358         }
3359 }
3360
3361 #ifdef CONFIG_BE2NET_VXLAN
3362 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3363 {
3364         struct net_device *netdev = adapter->netdev;
3365
3366         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3367                 be_cmd_manage_iface(adapter, adapter->if_handle,
3368                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3369
3370         if (adapter->vxlan_port)
3371                 be_cmd_set_vxlan_port(adapter, 0);
3372
3373         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3374         adapter->vxlan_port = 0;
3375
3376         netdev->hw_enc_features = 0;
3377         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3378         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3379 }
3380 #endif
3381
3382 static int be_clear(struct be_adapter *adapter)
3383 {
3384         be_cancel_worker(adapter);
3385
3386         if (sriov_enabled(adapter))
3387                 be_vf_clear(adapter);
3388
3389         /* Re-configure FW to distribute resources evenly across max-supported
3390          * number of VFs, only when VFs are not already enabled.
3391          */
3392         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3393                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3394                                         pci_sriov_get_totalvfs(adapter->pdev));
3395
3396 #ifdef CONFIG_BE2NET_VXLAN
3397         be_disable_vxlan_offloads(adapter);
3398 #endif
3399         /* delete the primary mac along with the uc-mac list */
3400         be_mac_clear(adapter);
3401
3402         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3403
3404         be_clear_queues(adapter);
3405
3406         be_msix_disable(adapter);
3407         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3408         return 0;
3409 }
3410
3411 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3412                         u32 cap_flags, u32 vf)
3413 {
3414         u32 en_flags;
3415         int status;
3416
3417         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3418                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3419                    BE_IF_FLAGS_RSS;
3420
3421         en_flags &= cap_flags;
3422
3423         status = be_cmd_if_create(adapter, cap_flags, en_flags,
3424                                   if_handle, vf);
3425
3426         return status;
3427 }
3428
3429 static int be_vfs_if_create(struct be_adapter *adapter)
3430 {
3431         struct be_resources res = {0};
3432         struct be_vf_cfg *vf_cfg;
3433         u32 cap_flags, vf;
3434         int status;
3435
3436         /* If a FW profile exists, then cap_flags are updated */
3437         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3438                     BE_IF_FLAGS_MULTICAST;
3439
3440         for_all_vfs(adapter, vf_cfg, vf) {
3441                 if (!BE3_chip(adapter)) {
3442                         status = be_cmd_get_profile_config(adapter, &res,
3443                                                            vf + 1);
3444                         if (!status)
3445                                 cap_flags = res.if_cap_flags;
3446                 }
3447
3448                 status = be_if_create(adapter, &vf_cfg->if_handle,
3449                                       cap_flags, vf + 1);
3450                 if (status)
3451                         return status;
3452         }
3453
3454         return 0;
3455 }
3456
3457 static int be_vf_setup_init(struct be_adapter *adapter)
3458 {
3459         struct be_vf_cfg *vf_cfg;
3460         int vf;
3461
3462         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3463                                   GFP_KERNEL);
3464         if (!adapter->vf_cfg)
3465                 return -ENOMEM;
3466
3467         for_all_vfs(adapter, vf_cfg, vf) {
3468                 vf_cfg->if_handle = -1;
3469                 vf_cfg->pmac_id = -1;
3470         }
3471         return 0;
3472 }
3473
3474 static int be_vf_setup(struct be_adapter *adapter)
3475 {
3476         struct device *dev = &adapter->pdev->dev;
3477         struct be_vf_cfg *vf_cfg;
3478         int status, old_vfs, vf;
3479         u32 privileges;
3480
3481         old_vfs = pci_num_vf(adapter->pdev);
3482
3483         status = be_vf_setup_init(adapter);
3484         if (status)
3485                 goto err;
3486
3487         if (old_vfs) {
3488                 for_all_vfs(adapter, vf_cfg, vf) {
3489                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3490                         if (status)
3491                                 goto err;
3492                 }
3493
3494                 status = be_vfs_mac_query(adapter);
3495                 if (status)
3496                         goto err;
3497         } else {
3498                 status = be_vfs_if_create(adapter);
3499                 if (status)
3500                         goto err;
3501
3502                 status = be_vf_eth_addr_config(adapter);
3503                 if (status)
3504                         goto err;
3505         }
3506
3507         for_all_vfs(adapter, vf_cfg, vf) {
3508                 /* Allow VFs to programs MAC/VLAN filters */
3509                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3510                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3511                         status = be_cmd_set_fn_privileges(adapter,
3512                                                           privileges |
3513                                                           BE_PRIV_FILTMGMT,
3514                                                           vf + 1);
3515                         if (!status)
3516                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3517                                          vf);
3518                 }
3519
3520                 /* Allow full available bandwidth */
3521                 if (!old_vfs)
3522                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3523
3524                 if (!old_vfs) {
3525                         be_cmd_enable_vf(adapter, vf + 1);
3526                         be_cmd_set_logical_link_config(adapter,
3527                                                        IFLA_VF_LINK_STATE_AUTO,
3528                                                        vf+1);
3529                 }
3530         }
3531
3532         if (!old_vfs) {
3533                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3534                 if (status) {
3535                         dev_err(dev, "SRIOV enable failed\n");
3536                         adapter->num_vfs = 0;
3537                         goto err;
3538                 }
3539         }
3540
3541         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3542         return 0;
3543 err:
3544         dev_err(dev, "VF setup failed\n");
3545         be_vf_clear(adapter);
3546         return status;
3547 }
3548
3549 /* Converting function_mode bits on BE3 to SH mc_type enums */
3550
3551 static u8 be_convert_mc_type(u32 function_mode)
3552 {
3553         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3554                 return vNIC1;
3555         else if (function_mode & QNQ_MODE)
3556                 return FLEX10;
3557         else if (function_mode & VNIC_MODE)
3558                 return vNIC2;
3559         else if (function_mode & UMC_ENABLED)
3560                 return UMC;
3561         else
3562                 return MC_NONE;
3563 }
3564
3565 /* On BE2/BE3 FW does not suggest the supported limits */
3566 static void BEx_get_resources(struct be_adapter *adapter,
3567                               struct be_resources *res)
3568 {
3569         bool use_sriov = adapter->num_vfs ? 1 : 0;
3570
3571         if (be_physfn(adapter))
3572                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3573         else
3574                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3575
3576         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3577
3578         if (be_is_mc(adapter)) {
3579                 /* Assuming that there are 4 channels per port,
3580                  * when multi-channel is enabled
3581                  */
3582                 if (be_is_qnq_mode(adapter))
3583                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3584                 else
3585                         /* In a non-qnq multichannel mode, the pvid
3586                          * takes up one vlan entry
3587                          */
3588                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3589         } else {
3590                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3591         }
3592
3593         res->max_mcast_mac = BE_MAX_MC;
3594
3595         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3596          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3597          *    *only* if it is RSS-capable.
3598          */
3599         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3600             !be_physfn(adapter) || (be_is_mc(adapter) &&
3601             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3602                 res->max_tx_qs = 1;
3603         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3604                 struct be_resources super_nic_res = {0};
3605
3606                 /* On a SuperNIC profile, the driver needs to use the
3607                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3608                  */
3609                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3610                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3611                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3612         } else {
3613                 res->max_tx_qs = BE3_MAX_TX_QS;
3614         }
3615
3616         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3617             !use_sriov && be_physfn(adapter))
3618                 res->max_rss_qs = (adapter->be3_native) ?
3619                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3620         res->max_rx_qs = res->max_rss_qs + 1;
3621
3622         if (be_physfn(adapter))
3623                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3624                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3625         else
3626                 res->max_evt_qs = 1;
3627
3628         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3629         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3630                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3631 }
3632
3633 static void be_setup_init(struct be_adapter *adapter)
3634 {
3635         adapter->vlan_prio_bmap = 0xff;
3636         adapter->phy.link_speed = -1;
3637         adapter->if_handle = -1;
3638         adapter->be3_native = false;
3639         adapter->if_flags = 0;
3640         if (be_physfn(adapter))
3641                 adapter->cmd_privileges = MAX_PRIVILEGES;
3642         else
3643                 adapter->cmd_privileges = MIN_PRIVILEGES;
3644 }
3645
3646 static int be_get_sriov_config(struct be_adapter *adapter)
3647 {
3648         struct device *dev = &adapter->pdev->dev;
3649         struct be_resources res = {0};
3650         int max_vfs, old_vfs;
3651
3652         /* Some old versions of BE3 FW don't report max_vfs value */
3653         be_cmd_get_profile_config(adapter, &res, 0);
3654
3655         if (BE3_chip(adapter) && !res.max_vfs) {
3656                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3657                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3658         }
3659
3660         adapter->pool_res = res;
3661
3662         if (!be_max_vfs(adapter)) {
3663                 if (num_vfs)
3664                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3665                 adapter->num_vfs = 0;
3666                 return 0;
3667         }
3668
3669         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3670
3671         /* validate num_vfs module param */
3672         old_vfs = pci_num_vf(adapter->pdev);
3673         if (old_vfs) {
3674                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3675                 if (old_vfs != num_vfs)
3676                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3677                 adapter->num_vfs = old_vfs;
3678         } else {
3679                 if (num_vfs > be_max_vfs(adapter)) {
3680                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3681                                  num_vfs);
3682                         dev_info(dev, "Limiting to %d VFs\n",
3683                                  be_max_vfs(adapter));
3684                 }
3685                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3686         }
3687
3688         return 0;
3689 }
3690
3691 static int be_get_resources(struct be_adapter *adapter)
3692 {
3693         struct device *dev = &adapter->pdev->dev;
3694         struct be_resources res = {0};
3695         int status;
3696
3697         if (BEx_chip(adapter)) {
3698                 BEx_get_resources(adapter, &res);
3699                 adapter->res = res;
3700         }
3701
3702         /* For Lancer, SH etc read per-function resource limits from FW.
3703          * GET_FUNC_CONFIG returns per function guaranteed limits.
3704          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3705          */
3706         if (!BEx_chip(adapter)) {
3707                 status = be_cmd_get_func_config(adapter, &res);
3708                 if (status)
3709                         return status;
3710
3711                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3712                 if (be_roce_supported(adapter))
3713                         res.max_evt_qs /= 2;
3714                 adapter->res = res;
3715         }
3716
3717         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3718                  be_max_txqs(adapter), be_max_rxqs(adapter),
3719                  be_max_rss(adapter), be_max_eqs(adapter),
3720                  be_max_vfs(adapter));
3721         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3722                  be_max_uc(adapter), be_max_mc(adapter),
3723                  be_max_vlans(adapter));
3724
3725         return 0;
3726 }
3727
3728 static void be_sriov_config(struct be_adapter *adapter)
3729 {
3730         struct device *dev = &adapter->pdev->dev;
3731         int status;
3732
3733         status = be_get_sriov_config(adapter);
3734         if (status) {
3735                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3736                 dev_err(dev, "SR-IOV cannot be enabled\n");
3737                 return;
3738         }
3739
3740         /* When the HW is in SRIOV capable configuration, the PF-pool
3741          * resources are equally distributed across the max-number of
3742          * VFs. The user may request only a subset of the max-vfs to be
3743          * enabled. Based on num_vfs, redistribute the resources across
3744          * num_vfs so that each VF will have access to more number of
3745          * resources. This facility is not available in BE3 FW.
3746          * Also, this is done by FW in Lancer chip.
3747          */
3748         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3749                 status = be_cmd_set_sriov_config(adapter,
3750                                                  adapter->pool_res,
3751                                                  adapter->num_vfs);
3752                 if (status)
3753                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3754         }
3755 }
3756
3757 static int be_get_config(struct be_adapter *adapter)
3758 {
3759         u16 profile_id;
3760         int status;
3761
3762         status = be_cmd_query_fw_cfg(adapter);
3763         if (status)
3764                 return status;
3765
3766         be_cmd_query_port_name(adapter);
3767
3768         if (be_physfn(adapter)) {
3769                 status = be_cmd_get_active_profile(adapter, &profile_id);
3770                 if (!status)
3771                         dev_info(&adapter->pdev->dev,
3772                                  "Using profile 0x%x\n", profile_id);
3773         }
3774
3775         if (!BE2_chip(adapter) && be_physfn(adapter))
3776                 be_sriov_config(adapter);
3777
3778         status = be_get_resources(adapter);
3779         if (status)
3780                 return status;
3781
3782         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3783                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3784         if (!adapter->pmac_id)
3785                 return -ENOMEM;
3786
3787         /* Sanitize cfg_num_qs based on HW and platform limits */
3788         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3789
3790         return 0;
3791 }
3792
3793 static int be_mac_setup(struct be_adapter *adapter)
3794 {
3795         u8 mac[ETH_ALEN];
3796         int status;
3797
3798         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3799                 status = be_cmd_get_perm_mac(adapter, mac);
3800                 if (status)
3801                         return status;
3802
3803                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3804                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3805         } else {
3806                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3807                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3808         }
3809
3810         /* For BE3-R VFs, the PF programs the initial MAC address */
3811         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3812                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3813                                 &adapter->pmac_id[0], 0);
3814         return 0;
3815 }
3816
3817 static void be_schedule_worker(struct be_adapter *adapter)
3818 {
3819         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3820         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3821 }
3822
3823 static int be_setup_queues(struct be_adapter *adapter)
3824 {
3825         struct net_device *netdev = adapter->netdev;
3826         int status;
3827
3828         status = be_evt_queues_create(adapter);
3829         if (status)
3830                 goto err;
3831
3832         status = be_tx_qs_create(adapter);
3833         if (status)
3834                 goto err;
3835
3836         status = be_rx_cqs_create(adapter);
3837         if (status)
3838                 goto err;
3839
3840         status = be_mcc_queues_create(adapter);
3841         if (status)
3842                 goto err;
3843
3844         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3845         if (status)
3846                 goto err;
3847
3848         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3849         if (status)
3850                 goto err;
3851
3852         return 0;
3853 err:
3854         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3855         return status;
3856 }
3857
3858 int be_update_queues(struct be_adapter *adapter)
3859 {
3860         struct net_device *netdev = adapter->netdev;
3861         int status;
3862
3863         if (netif_running(netdev))
3864                 be_close(netdev);
3865
3866         be_cancel_worker(adapter);
3867
3868         /* If any vectors have been shared with RoCE we cannot re-program
3869          * the MSIx table.
3870          */
3871         if (!adapter->num_msix_roce_vec)
3872                 be_msix_disable(adapter);
3873
3874         be_clear_queues(adapter);
3875
3876         if (!msix_enabled(adapter)) {
3877                 status = be_msix_enable(adapter);
3878                 if (status)
3879                         return status;
3880         }
3881
3882         status = be_setup_queues(adapter);
3883         if (status)
3884                 return status;
3885
3886         be_schedule_worker(adapter);
3887
3888         if (netif_running(netdev))
3889                 status = be_open(netdev);
3890
3891         return status;
3892 }
3893
3894 static inline int fw_major_num(const char *fw_ver)
3895 {
3896         int fw_major = 0, i;
3897
3898         i = sscanf(fw_ver, "%d.", &fw_major);
3899         if (i != 1)
3900                 return 0;
3901
3902         return fw_major;
3903 }
3904
3905 static int be_setup(struct be_adapter *adapter)
3906 {
3907         struct device *dev = &adapter->pdev->dev;
3908         int status;
3909
3910         be_setup_init(adapter);
3911
3912         if (!lancer_chip(adapter))
3913                 be_cmd_req_native_mode(adapter);
3914
3915         status = be_get_config(adapter);
3916         if (status)
3917                 goto err;
3918
3919         status = be_msix_enable(adapter);
3920         if (status)
3921                 goto err;
3922
3923         status = be_if_create(adapter, &adapter->if_handle,
3924                               be_if_cap_flags(adapter), 0);
3925         if (status)
3926                 goto err;
3927
3928         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3929         rtnl_lock();
3930         status = be_setup_queues(adapter);
3931         rtnl_unlock();
3932         if (status)
3933                 goto err;
3934
3935         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3936
3937         status = be_mac_setup(adapter);
3938         if (status)
3939                 goto err;
3940
3941         be_cmd_get_fw_ver(adapter);
3942         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3943
3944         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3945                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3946                         adapter->fw_ver);
3947                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3948         }
3949
3950         if (adapter->vlans_added)
3951                 be_vid_config(adapter);
3952
3953         be_set_rx_mode(adapter->netdev);
3954
3955         be_cmd_get_acpi_wol_cap(adapter);
3956
3957         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3958                                          adapter->rx_fc);
3959         if (status)
3960                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3961                                         &adapter->rx_fc);
3962
3963         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3964                  adapter->tx_fc, adapter->rx_fc);
3965
3966         if (be_physfn(adapter))
3967                 be_cmd_set_logical_link_config(adapter,
3968                                                IFLA_VF_LINK_STATE_AUTO, 0);
3969
3970         if (adapter->num_vfs)
3971                 be_vf_setup(adapter);
3972
3973         status = be_cmd_get_phy_info(adapter);
3974         if (!status && be_pause_supported(adapter))
3975                 adapter->phy.fc_autoneg = 1;
3976
3977         be_schedule_worker(adapter);
3978         adapter->flags |= BE_FLAGS_SETUP_DONE;
3979         return 0;
3980 err:
3981         be_clear(adapter);
3982         return status;
3983 }
3984
3985 #ifdef CONFIG_NET_POLL_CONTROLLER
3986 static void be_netpoll(struct net_device *netdev)
3987 {
3988         struct be_adapter *adapter = netdev_priv(netdev);
3989         struct be_eq_obj *eqo;
3990         int i;
3991
3992         for_all_evt_queues(adapter, eqo, i) {
3993                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3994                 napi_schedule(&eqo->napi);
3995         }
3996 }
3997 #endif
3998
3999 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
4000
4001 static bool phy_flashing_required(struct be_adapter *adapter)
4002 {
4003         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
4004                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
4005 }
4006
4007 static bool is_comp_in_ufi(struct be_adapter *adapter,
4008                            struct flash_section_info *fsec, int type)
4009 {
4010         int i = 0, img_type = 0;
4011         struct flash_section_info_g2 *fsec_g2 = NULL;
4012
4013         if (BE2_chip(adapter))
4014                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
4015
4016         for (i = 0; i < MAX_FLASH_COMP; i++) {
4017                 if (fsec_g2)
4018                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
4019                 else
4020                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
4021
4022                 if (img_type == type)
4023                         return true;
4024         }
4025         return false;
4026
4027 }
4028
4029 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
4030                                                 int header_size,
4031                                                 const struct firmware *fw)
4032 {
4033         struct flash_section_info *fsec = NULL;
4034         const u8 *p = fw->data;
4035
4036         p += header_size;
4037         while (p < (fw->data + fw->size)) {
4038                 fsec = (struct flash_section_info *)p;
4039                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
4040                         return fsec;
4041                 p += 32;
4042         }
4043         return NULL;
4044 }
4045
4046 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
4047                               u32 img_offset, u32 img_size, int hdr_size,
4048                               u16 img_optype, bool *crc_match)
4049 {
4050         u32 crc_offset;
4051         int status;
4052         u8 crc[4];
4053
4054         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
4055                                       img_size - 4);
4056         if (status)
4057                 return status;
4058
4059         crc_offset = hdr_size + img_offset + img_size - 4;
4060
4061         /* Skip flashing, if crc of flashed region matches */
4062         if (!memcmp(crc, p + crc_offset, 4))
4063                 *crc_match = true;
4064         else
4065                 *crc_match = false;
4066
4067         return status;
4068 }
4069
4070 static int be_flash(struct be_adapter *adapter, const u8 *img,
4071                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4072                     u32 img_offset)
4073 {
4074         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4075         struct be_cmd_write_flashrom *req = flash_cmd->va;
4076         int status;
4077
4078         while (total_bytes) {
4079                 num_bytes = min_t(u32, 32*1024, total_bytes);
4080
4081                 total_bytes -= num_bytes;
4082
4083                 if (!total_bytes) {
4084                         if (optype == OPTYPE_PHY_FW)
4085                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4086                         else
4087                                 flash_op = FLASHROM_OPER_FLASH;
4088                 } else {
4089                         if (optype == OPTYPE_PHY_FW)
4090                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4091                         else
4092                                 flash_op = FLASHROM_OPER_SAVE;
4093                 }
4094
4095                 memcpy(req->data_buf, img, num_bytes);
4096                 img += num_bytes;
4097                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4098                                                flash_op, img_offset +
4099                                                bytes_sent, num_bytes);
4100                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4101                     optype == OPTYPE_PHY_FW)
4102                         break;
4103                 else if (status)
4104                         return status;
4105
4106                 bytes_sent += num_bytes;
4107         }
4108         return 0;
4109 }
4110
4111 /* For BE2, BE3 and BE3-R */
4112 static int be_flash_BEx(struct be_adapter *adapter,
4113                         const struct firmware *fw,
4114                         struct be_dma_mem *flash_cmd, int num_of_images)
4115 {
4116         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4117         struct device *dev = &adapter->pdev->dev;
4118         struct flash_section_info *fsec = NULL;
4119         int status, i, filehdr_size, num_comp;
4120         const struct flash_comp *pflashcomp;
4121         bool crc_match;
4122         const u8 *p;
4123
4124         struct flash_comp gen3_flash_types[] = {
4125                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4126                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4127                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4128                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4129                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4130                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4131                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4132                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4133                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4134                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4135                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4136                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4137                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4138                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4139                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4140                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4141                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4142                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4143                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4144                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4145         };
4146
4147         struct flash_comp gen2_flash_types[] = {
4148                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4149                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4150                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4151                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4152                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4153                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4154                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4155                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4156                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4157                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4158                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4159                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4160                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4161                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4162                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4163                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4164         };
4165
4166         if (BE3_chip(adapter)) {
4167                 pflashcomp = gen3_flash_types;
4168                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4169                 num_comp = ARRAY_SIZE(gen3_flash_types);
4170         } else {
4171                 pflashcomp = gen2_flash_types;
4172                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4173                 num_comp = ARRAY_SIZE(gen2_flash_types);
4174                 img_hdrs_size = 0;
4175         }
4176
4177         /* Get flash section info*/
4178         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4179         if (!fsec) {
4180                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4181                 return -1;
4182         }
4183         for (i = 0; i < num_comp; i++) {
4184                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4185                         continue;
4186
4187                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4188                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4189                         continue;
4190
4191                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4192                     !phy_flashing_required(adapter))
4193                                 continue;
4194
4195                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4196                         status = be_check_flash_crc(adapter, fw->data,
4197                                                     pflashcomp[i].offset,
4198                                                     pflashcomp[i].size,
4199                                                     filehdr_size +
4200                                                     img_hdrs_size,
4201                                                     OPTYPE_REDBOOT, &crc_match);
4202                         if (status) {
4203                                 dev_err(dev,
4204                                         "Could not get CRC for 0x%x region\n",
4205                                         pflashcomp[i].optype);
4206                                 continue;
4207                         }
4208
4209                         if (crc_match)
4210                                 continue;
4211                 }
4212
4213                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4214                         img_hdrs_size;
4215                 if (p + pflashcomp[i].size > fw->data + fw->size)
4216                         return -1;
4217
4218                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4219                                   pflashcomp[i].size, 0);
4220                 if (status) {
4221                         dev_err(dev, "Flashing section type 0x%x failed\n",
4222                                 pflashcomp[i].img_type);
4223                         return status;
4224                 }
4225         }
4226         return 0;
4227 }
4228
4229 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4230 {
4231         u32 img_type = le32_to_cpu(fsec_entry.type);
4232         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4233
4234         if (img_optype != 0xFFFF)
4235                 return img_optype;
4236
4237         switch (img_type) {
4238         case IMAGE_FIRMWARE_iSCSI:
4239                 img_optype = OPTYPE_ISCSI_ACTIVE;
4240                 break;
4241         case IMAGE_BOOT_CODE:
4242                 img_optype = OPTYPE_REDBOOT;
4243                 break;
4244         case IMAGE_OPTION_ROM_ISCSI:
4245                 img_optype = OPTYPE_BIOS;
4246                 break;
4247         case IMAGE_OPTION_ROM_PXE:
4248                 img_optype = OPTYPE_PXE_BIOS;
4249                 break;
4250         case IMAGE_OPTION_ROM_FCoE:
4251                 img_optype = OPTYPE_FCOE_BIOS;
4252                 break;
4253         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4254                 img_optype = OPTYPE_ISCSI_BACKUP;
4255                 break;
4256         case IMAGE_NCSI:
4257                 img_optype = OPTYPE_NCSI_FW;
4258                 break;
4259         case IMAGE_FLASHISM_JUMPVECTOR:
4260                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4261                 break;
4262         case IMAGE_FIRMWARE_PHY:
4263                 img_optype = OPTYPE_SH_PHY_FW;
4264                 break;
4265         case IMAGE_REDBOOT_DIR:
4266                 img_optype = OPTYPE_REDBOOT_DIR;
4267                 break;
4268         case IMAGE_REDBOOT_CONFIG:
4269                 img_optype = OPTYPE_REDBOOT_CONFIG;
4270                 break;
4271         case IMAGE_UFI_DIR:
4272                 img_optype = OPTYPE_UFI_DIR;
4273                 break;
4274         default:
4275                 break;
4276         }
4277
4278         return img_optype;
4279 }
4280
4281 static int be_flash_skyhawk(struct be_adapter *adapter,
4282                             const struct firmware *fw,
4283                             struct be_dma_mem *flash_cmd, int num_of_images)
4284 {
4285         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4286         bool crc_match, old_fw_img, flash_offset_support = true;
4287         struct device *dev = &adapter->pdev->dev;
4288         struct flash_section_info *fsec = NULL;
4289         u32 img_offset, img_size, img_type;
4290         u16 img_optype, flash_optype;
4291         int status, i, filehdr_size;
4292         const u8 *p;
4293
4294         filehdr_size = sizeof(struct flash_file_hdr_g3);
4295         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4296         if (!fsec) {
4297                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4298                 return -EINVAL;
4299         }
4300
4301 retry_flash:
4302         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4303                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4304                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4305                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4306                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4307                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4308
4309                 if (img_optype == 0xFFFF)
4310                         continue;
4311
4312                 if (flash_offset_support)
4313                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4314                 else
4315                         flash_optype = img_optype;
4316
4317                 /* Don't bother verifying CRC if an old FW image is being
4318                  * flashed
4319                  */
4320                 if (old_fw_img)
4321                         goto flash;
4322
4323                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4324                                             img_size, filehdr_size +
4325                                             img_hdrs_size, flash_optype,
4326                                             &crc_match);
4327                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4328                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4329                         /* The current FW image on the card does not support
4330                          * OFFSET based flashing. Retry using older mechanism
4331                          * of OPTYPE based flashing
4332                          */
4333                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4334                                 flash_offset_support = false;
4335                                 goto retry_flash;
4336                         }
4337
4338                         /* The current FW image on the card does not recognize
4339                          * the new FLASH op_type. The FW download is partially
4340                          * complete. Reboot the server now to enable FW image
4341                          * to recognize the new FLASH op_type. To complete the
4342                          * remaining process, download the same FW again after
4343                          * the reboot.
4344                          */
4345                         dev_err(dev, "Flash incomplete. Reset the server\n");
4346                         dev_err(dev, "Download FW image again after reset\n");
4347                         return -EAGAIN;
4348                 } else if (status) {
4349                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4350                                 img_optype);
4351                         return -EFAULT;
4352                 }
4353
4354                 if (crc_match)
4355                         continue;
4356
4357 flash:
4358                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4359                 if (p + img_size > fw->data + fw->size)
4360                         return -1;
4361
4362                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4363                                   img_offset);
4364
4365                 /* The current FW image on the card does not support OFFSET
4366                  * based flashing. Retry using older mechanism of OPTYPE based
4367                  * flashing
4368                  */
4369                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4370                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4371                         flash_offset_support = false;
4372                         goto retry_flash;
4373                 }
4374
4375                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4376                  * UFI_DIR region
4377                  */
4378                 if (old_fw_img &&
4379                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4380                      (img_optype == OPTYPE_UFI_DIR &&
4381                       base_status(status) == MCC_STATUS_FAILED))) {
4382                         continue;
4383                 } else if (status) {
4384                         dev_err(dev, "Flashing section type 0x%x failed\n",
4385                                 img_type);
4386                         return -EFAULT;
4387                 }
4388         }
4389         return 0;
4390 }
4391
4392 static int lancer_fw_download(struct be_adapter *adapter,
4393                               const struct firmware *fw)
4394 {
4395 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4396 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4397         struct device *dev = &adapter->pdev->dev;
4398         struct be_dma_mem flash_cmd;
4399         const u8 *data_ptr = NULL;
4400         u8 *dest_image_ptr = NULL;
4401         size_t image_size = 0;
4402         u32 chunk_size = 0;
4403         u32 data_written = 0;
4404         u32 offset = 0;
4405         int status = 0;
4406         u8 add_status = 0;
4407         u8 change_status;
4408
4409         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4410                 dev_err(dev, "FW image size should be multiple of 4\n");
4411                 return -EINVAL;
4412         }
4413
4414         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4415                                 + LANCER_FW_DOWNLOAD_CHUNK;
4416         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4417                                           &flash_cmd.dma, GFP_KERNEL);
4418         if (!flash_cmd.va)
4419                 return -ENOMEM;
4420
4421         dest_image_ptr = flash_cmd.va +
4422                                 sizeof(struct lancer_cmd_req_write_object);
4423         image_size = fw->size;
4424         data_ptr = fw->data;
4425
4426         while (image_size) {
4427                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4428
4429                 /* Copy the image chunk content. */
4430                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4431
4432                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4433                                                  chunk_size, offset,
4434                                                  LANCER_FW_DOWNLOAD_LOCATION,
4435                                                  &data_written, &change_status,
4436                                                  &add_status);
4437                 if (status)
4438                         break;
4439
4440                 offset += data_written;
4441                 data_ptr += data_written;
4442                 image_size -= data_written;
4443         }
4444
4445         if (!status) {
4446                 /* Commit the FW written */
4447                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4448                                                  0, offset,
4449                                                  LANCER_FW_DOWNLOAD_LOCATION,
4450                                                  &data_written, &change_status,
4451                                                  &add_status);
4452         }
4453
4454         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4455         if (status) {
4456                 dev_err(dev, "Firmware load error\n");
4457                 return be_cmd_status(status);
4458         }
4459
4460         dev_info(dev, "Firmware flashed successfully\n");
4461
4462         if (change_status == LANCER_FW_RESET_NEEDED) {
4463                 dev_info(dev, "Resetting adapter to activate new FW\n");
4464                 status = lancer_physdev_ctrl(adapter,
4465                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4466                 if (status) {
4467                         dev_err(dev, "Adapter busy, could not reset FW\n");
4468                         dev_err(dev, "Reboot server to activate new FW\n");
4469                 }
4470         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4471                 dev_info(dev, "Reboot server to activate new FW\n");
4472         }
4473
4474         return 0;
4475 }
4476
4477 #define BE2_UFI         2
4478 #define BE3_UFI         3
4479 #define BE3R_UFI        10
4480 #define SH_UFI          4
4481 #define SH_P2_UFI       11
4482
4483 static int be_get_ufi_type(struct be_adapter *adapter,
4484                            struct flash_file_hdr_g3 *fhdr)
4485 {
4486         if (!fhdr) {
4487                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4488                 return -1;
4489         }
4490
4491         /* First letter of the build version is used to identify
4492          * which chip this image file is meant for.
4493          */
4494         switch (fhdr->build[0]) {
4495         case BLD_STR_UFI_TYPE_SH:
4496                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4497                                                                 SH_UFI;
4498         case BLD_STR_UFI_TYPE_BE3:
4499                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4500                                                                 BE3_UFI;
4501         case BLD_STR_UFI_TYPE_BE2:
4502                 return BE2_UFI;
4503         default:
4504                 return -1;
4505         }
4506 }
4507
4508 /* Check if the flash image file is compatible with the adapter that
4509  * is being flashed.
4510  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4511  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4512  */
4513 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4514                                        struct flash_file_hdr_g3 *fhdr)
4515 {
4516         int ufi_type = be_get_ufi_type(adapter, fhdr);
4517
4518         switch (ufi_type) {
4519         case SH_P2_UFI:
4520                 return skyhawk_chip(adapter);
4521         case SH_UFI:
4522                 return (skyhawk_chip(adapter) &&
4523                         adapter->asic_rev < ASIC_REV_P2);
4524         case BE3R_UFI:
4525                 return BE3_chip(adapter);
4526         case BE3_UFI:
4527                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4528         case BE2_UFI:
4529                 return BE2_chip(adapter);
4530         default:
4531                 return false;
4532         }
4533 }
4534
4535 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4536 {
4537         struct device *dev = &adapter->pdev->dev;
4538         struct flash_file_hdr_g3 *fhdr3;
4539         struct image_hdr *img_hdr_ptr;
4540         int status = 0, i, num_imgs;
4541         struct be_dma_mem flash_cmd;
4542
4543         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4544         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4545                 dev_err(dev, "Flash image is not compatible with adapter\n");
4546                 return -EINVAL;
4547         }
4548
4549         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4550         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4551                                           GFP_KERNEL);
4552         if (!flash_cmd.va)
4553                 return -ENOMEM;
4554
4555         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4556         for (i = 0; i < num_imgs; i++) {
4557                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4558                                 (sizeof(struct flash_file_hdr_g3) +
4559                                  i * sizeof(struct image_hdr)));
4560                 if (!BE2_chip(adapter) &&
4561                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4562                         continue;
4563
4564                 if (skyhawk_chip(adapter))
4565                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4566                                                   num_imgs);
4567                 else
4568                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4569                                               num_imgs);
4570         }
4571
4572         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4573         if (!status)
4574                 dev_info(dev, "Firmware flashed successfully\n");
4575
4576         return status;
4577 }
4578
4579 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4580 {
4581         const struct firmware *fw;
4582         int status;
4583
4584         if (!netif_running(adapter->netdev)) {
4585                 dev_err(&adapter->pdev->dev,
4586                         "Firmware load not allowed (interface is down)\n");
4587                 return -ENETDOWN;
4588         }
4589
4590         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4591         if (status)
4592                 goto fw_exit;
4593
4594         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4595
4596         if (lancer_chip(adapter))
4597                 status = lancer_fw_download(adapter, fw);
4598         else
4599                 status = be_fw_download(adapter, fw);
4600
4601         if (!status)
4602                 be_cmd_get_fw_ver(adapter);
4603
4604 fw_exit:
4605         release_firmware(fw);
4606         return status;
4607 }
4608
4609 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4610                                  u16 flags)
4611 {
4612         struct be_adapter *adapter = netdev_priv(dev);
4613         struct nlattr *attr, *br_spec;
4614         int rem;
4615         int status = 0;
4616         u16 mode = 0;
4617
4618         if (!sriov_enabled(adapter))
4619                 return -EOPNOTSUPP;
4620
4621         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4622         if (!br_spec)
4623                 return -EINVAL;
4624
4625         nla_for_each_nested(attr, br_spec, rem) {
4626                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4627                         continue;
4628
4629                 if (nla_len(attr) < sizeof(mode))
4630                         return -EINVAL;
4631
4632                 mode = nla_get_u16(attr);
4633                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4634                         return -EINVAL;
4635
4636                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4637                                                adapter->if_handle,
4638                                                mode == BRIDGE_MODE_VEPA ?
4639                                                PORT_FWD_TYPE_VEPA :
4640                                                PORT_FWD_TYPE_VEB);
4641                 if (status)
4642                         goto err;
4643
4644                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4645                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4646
4647                 return status;
4648         }
4649 err:
4650         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4651                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4652
4653         return status;
4654 }
4655
4656 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4657                                  struct net_device *dev, u32 filter_mask)
4658 {
4659         struct be_adapter *adapter = netdev_priv(dev);
4660         int status = 0;
4661         u8 hsw_mode;
4662
4663         if (!sriov_enabled(adapter))
4664                 return 0;
4665
4666         /* BE and Lancer chips support VEB mode only */
4667         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4668                 hsw_mode = PORT_FWD_TYPE_VEB;
4669         } else {
4670                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4671                                                adapter->if_handle, &hsw_mode);
4672                 if (status)
4673                         return 0;
4674         }
4675
4676         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4677                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4678                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4679                                        0, 0);
4680 }
4681
4682 #ifdef CONFIG_BE2NET_VXLAN
4683 /* VxLAN offload Notes:
4684  *
4685  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4686  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4687  * is expected to work across all types of IP tunnels once exported. Skyhawk
4688  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4689  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4690  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4691  * those other tunnels are unexported on the fly through ndo_features_check().
4692  *
4693  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4694  * adds more than one port, disable offloads and don't re-enable them again
4695  * until after all the tunnels are removed.
4696  */
4697 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4698                               __be16 port)
4699 {
4700         struct be_adapter *adapter = netdev_priv(netdev);
4701         struct device *dev = &adapter->pdev->dev;
4702         int status;
4703
4704         if (lancer_chip(adapter) || BEx_chip(adapter))
4705                 return;
4706
4707         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4708                 dev_info(dev,
4709                          "Only one UDP port supported for VxLAN offloads\n");
4710                 dev_info(dev, "Disabling VxLAN offloads\n");
4711                 adapter->vxlan_port_count++;
4712                 goto err;
4713         }
4714
4715         if (adapter->vxlan_port_count++ >= 1)
4716                 return;
4717
4718         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4719                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4720         if (status) {
4721                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4722                 goto err;
4723         }
4724
4725         status = be_cmd_set_vxlan_port(adapter, port);
4726         if (status) {
4727                 dev_warn(dev, "Failed to add VxLAN port\n");
4728                 goto err;
4729         }
4730         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4731         adapter->vxlan_port = port;
4732
4733         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4734                                    NETIF_F_TSO | NETIF_F_TSO6 |
4735                                    NETIF_F_GSO_UDP_TUNNEL;
4736         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4737         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4738
4739         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4740                  be16_to_cpu(port));
4741         return;
4742 err:
4743         be_disable_vxlan_offloads(adapter);
4744 }
4745
4746 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4747                               __be16 port)
4748 {
4749         struct be_adapter *adapter = netdev_priv(netdev);
4750
4751         if (lancer_chip(adapter) || BEx_chip(adapter))
4752                 return;
4753
4754         if (adapter->vxlan_port != port)
4755                 goto done;
4756
4757         be_disable_vxlan_offloads(adapter);
4758
4759         dev_info(&adapter->pdev->dev,
4760                  "Disabled VxLAN offloads for UDP port %d\n",
4761                  be16_to_cpu(port));
4762 done:
4763         adapter->vxlan_port_count--;
4764 }
4765
4766 static netdev_features_t be_features_check(struct sk_buff *skb,
4767                                            struct net_device *dev,
4768                                            netdev_features_t features)
4769 {
4770         struct be_adapter *adapter = netdev_priv(dev);
4771         u8 l4_hdr = 0;
4772
4773         /* The code below restricts offload features for some tunneled packets.
4774          * Offload features for normal (non tunnel) packets are unchanged.
4775          */
4776         if (!skb->encapsulation ||
4777             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4778                 return features;
4779
4780         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4781          * should disable tunnel offload features if it's not a VxLAN packet,
4782          * as tunnel offloads have been enabled only for VxLAN. This is done to
4783          * allow other tunneled traffic like GRE work fine while VxLAN
4784          * offloads are configured in Skyhawk-R.
4785          */
4786         switch (vlan_get_protocol(skb)) {
4787         case htons(ETH_P_IP):
4788                 l4_hdr = ip_hdr(skb)->protocol;
4789                 break;
4790         case htons(ETH_P_IPV6):
4791                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4792                 break;
4793         default:
4794                 return features;
4795         }
4796
4797         if (l4_hdr != IPPROTO_UDP ||
4798             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4799             skb->inner_protocol != htons(ETH_P_TEB) ||
4800             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4801             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4802                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4803
4804         return features;
4805 }
4806 #endif
4807
4808 static const struct net_device_ops be_netdev_ops = {
4809         .ndo_open               = be_open,
4810         .ndo_stop               = be_close,
4811         .ndo_start_xmit         = be_xmit,
4812         .ndo_set_rx_mode        = be_set_rx_mode,
4813         .ndo_set_mac_address    = be_mac_addr_set,
4814         .ndo_change_mtu         = be_change_mtu,
4815         .ndo_get_stats64        = be_get_stats64,
4816         .ndo_validate_addr      = eth_validate_addr,
4817         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4818         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4819         .ndo_set_vf_mac         = be_set_vf_mac,
4820         .ndo_set_vf_vlan        = be_set_vf_vlan,
4821         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4822         .ndo_get_vf_config      = be_get_vf_config,
4823         .ndo_set_vf_link_state  = be_set_vf_link_state,
4824 #ifdef CONFIG_NET_POLL_CONTROLLER
4825         .ndo_poll_controller    = be_netpoll,
4826 #endif
4827         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4828         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4829 #ifdef CONFIG_NET_RX_BUSY_POLL
4830         .ndo_busy_poll          = be_busy_poll,
4831 #endif
4832 #ifdef CONFIG_BE2NET_VXLAN
4833         .ndo_add_vxlan_port     = be_add_vxlan_port,
4834         .ndo_del_vxlan_port     = be_del_vxlan_port,
4835         .ndo_features_check     = be_features_check,
4836 #endif
4837 };
4838
4839 static void be_netdev_init(struct net_device *netdev)
4840 {
4841         struct be_adapter *adapter = netdev_priv(netdev);
4842
4843         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4844                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4845                 NETIF_F_HW_VLAN_CTAG_TX;
4846         if (be_multi_rxq(adapter))
4847                 netdev->hw_features |= NETIF_F_RXHASH;
4848
4849         netdev->features |= netdev->hw_features |
4850                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4851
4852         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4853                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4854
4855         netdev->priv_flags |= IFF_UNICAST_FLT;
4856
4857         netdev->flags |= IFF_MULTICAST;
4858
4859         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4860
4861         netdev->netdev_ops = &be_netdev_ops;
4862
4863         netdev->ethtool_ops = &be_ethtool_ops;
4864 }
4865
4866 static void be_unmap_pci_bars(struct be_adapter *adapter)
4867 {
4868         if (adapter->csr)
4869                 pci_iounmap(adapter->pdev, adapter->csr);
4870         if (adapter->db)
4871                 pci_iounmap(adapter->pdev, adapter->db);
4872 }
4873
4874 static int db_bar(struct be_adapter *adapter)
4875 {
4876         if (lancer_chip(adapter) || !be_physfn(adapter))
4877                 return 0;
4878         else
4879                 return 4;
4880 }
4881
4882 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4883 {
4884         if (skyhawk_chip(adapter)) {
4885                 adapter->roce_db.size = 4096;
4886                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4887                                                               db_bar(adapter));
4888                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4889                                                                db_bar(adapter));
4890         }
4891         return 0;
4892 }
4893
4894 static int be_map_pci_bars(struct be_adapter *adapter)
4895 {
4896         u8 __iomem *addr;
4897
4898         if (BEx_chip(adapter) && be_physfn(adapter)) {
4899                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4900                 if (!adapter->csr)
4901                         return -ENOMEM;
4902         }
4903
4904         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4905         if (!addr)
4906                 goto pci_map_err;
4907         adapter->db = addr;
4908
4909         be_roce_map_pci_bars(adapter);
4910         return 0;
4911
4912 pci_map_err:
4913         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4914         be_unmap_pci_bars(adapter);
4915         return -ENOMEM;
4916 }
4917
4918 static void be_ctrl_cleanup(struct be_adapter *adapter)
4919 {
4920         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4921
4922         be_unmap_pci_bars(adapter);
4923
4924         if (mem->va)
4925                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4926                                   mem->dma);
4927
4928         mem = &adapter->rx_filter;
4929         if (mem->va)
4930                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4931                                   mem->dma);
4932 }
4933
4934 static int be_ctrl_init(struct be_adapter *adapter)
4935 {
4936         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4937         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4938         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4939         u32 sli_intf;
4940         int status;
4941
4942         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4943         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4944                                  SLI_INTF_FAMILY_SHIFT;
4945         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4946
4947         status = be_map_pci_bars(adapter);
4948         if (status)
4949                 goto done;
4950
4951         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4952         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4953                                                 mbox_mem_alloc->size,
4954                                                 &mbox_mem_alloc->dma,
4955                                                 GFP_KERNEL);
4956         if (!mbox_mem_alloc->va) {
4957                 status = -ENOMEM;
4958                 goto unmap_pci_bars;
4959         }
4960         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4961         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4962         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4963         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4964
4965         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4966         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4967                                             rx_filter->size, &rx_filter->dma,
4968                                             GFP_KERNEL);
4969         if (!rx_filter->va) {
4970                 status = -ENOMEM;
4971                 goto free_mbox;
4972         }
4973
4974         mutex_init(&adapter->mbox_lock);
4975         spin_lock_init(&adapter->mcc_lock);
4976         spin_lock_init(&adapter->mcc_cq_lock);
4977
4978         init_completion(&adapter->et_cmd_compl);
4979         pci_save_state(adapter->pdev);
4980         return 0;
4981
4982 free_mbox:
4983         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4984                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4985
4986 unmap_pci_bars:
4987         be_unmap_pci_bars(adapter);
4988
4989 done:
4990         return status;
4991 }
4992
4993 static void be_stats_cleanup(struct be_adapter *adapter)
4994 {
4995         struct be_dma_mem *cmd = &adapter->stats_cmd;
4996
4997         if (cmd->va)
4998                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4999                                   cmd->va, cmd->dma);
5000 }
5001
5002 static int be_stats_init(struct be_adapter *adapter)
5003 {
5004         struct be_dma_mem *cmd = &adapter->stats_cmd;
5005
5006         if (lancer_chip(adapter))
5007                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5008         else if (BE2_chip(adapter))
5009                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5010         else if (BE3_chip(adapter))
5011                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5012         else
5013                 /* ALL non-BE ASICs */
5014                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5015
5016         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
5017                                       GFP_KERNEL);
5018         if (!cmd->va)
5019                 return -ENOMEM;
5020         return 0;
5021 }
5022
5023 static void be_remove(struct pci_dev *pdev)
5024 {
5025         struct be_adapter *adapter = pci_get_drvdata(pdev);
5026
5027         if (!adapter)
5028                 return;
5029
5030         be_roce_dev_remove(adapter);
5031         be_intr_set(adapter, false);
5032
5033         cancel_delayed_work_sync(&adapter->func_recovery_work);
5034
5035         unregister_netdev(adapter->netdev);
5036
5037         be_clear(adapter);
5038
5039         /* tell fw we're done with firing cmds */
5040         be_cmd_fw_clean(adapter);
5041
5042         be_stats_cleanup(adapter);
5043
5044         be_ctrl_cleanup(adapter);
5045
5046         pci_disable_pcie_error_reporting(pdev);
5047
5048         pci_release_regions(pdev);
5049         pci_disable_device(pdev);
5050
5051         free_netdev(adapter->netdev);
5052 }
5053
5054 static int be_get_initial_config(struct be_adapter *adapter)
5055 {
5056         int status, level;
5057
5058         status = be_cmd_get_cntl_attributes(adapter);
5059         if (status)
5060                 return status;
5061
5062         /* Must be a power of 2 or else MODULO will BUG_ON */
5063         adapter->be_get_temp_freq = 64;
5064
5065         if (BEx_chip(adapter)) {
5066                 level = be_cmd_get_fw_log_level(adapter);
5067                 adapter->msg_enable =
5068                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
5069         }
5070
5071         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
5072         return 0;
5073 }
5074
5075 static int lancer_recover_func(struct be_adapter *adapter)
5076 {
5077         struct device *dev = &adapter->pdev->dev;
5078         int status;
5079
5080         status = lancer_test_and_set_rdy_state(adapter);
5081         if (status)
5082                 goto err;
5083
5084         if (netif_running(adapter->netdev))
5085                 be_close(adapter->netdev);
5086
5087         be_clear(adapter);
5088
5089         be_clear_all_error(adapter);
5090
5091         status = be_setup(adapter);
5092         if (status)
5093                 goto err;
5094
5095         if (netif_running(adapter->netdev)) {
5096                 status = be_open(adapter->netdev);
5097                 if (status)
5098                         goto err;
5099         }
5100
5101         dev_err(dev, "Adapter recovery successful\n");
5102         return 0;
5103 err:
5104         if (status == -EAGAIN)
5105                 dev_err(dev, "Waiting for resource provisioning\n");
5106         else
5107                 dev_err(dev, "Adapter recovery failed\n");
5108
5109         return status;
5110 }
5111
5112 static void be_func_recovery_task(struct work_struct *work)
5113 {
5114         struct be_adapter *adapter =
5115                 container_of(work, struct be_adapter,  func_recovery_work.work);
5116         int status = 0;
5117
5118         be_detect_error(adapter);
5119
5120         if (adapter->hw_error && lancer_chip(adapter)) {
5121                 rtnl_lock();
5122                 netif_device_detach(adapter->netdev);
5123                 rtnl_unlock();
5124
5125                 status = lancer_recover_func(adapter);
5126                 if (!status)
5127                         netif_device_attach(adapter->netdev);
5128         }
5129
5130         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
5131          * no need to attempt further recovery.
5132          */
5133         if (!status || status == -EAGAIN)
5134                 schedule_delayed_work(&adapter->func_recovery_work,
5135                                       msecs_to_jiffies(1000));
5136 }
5137
5138 static void be_log_sfp_info(struct be_adapter *adapter)
5139 {
5140         int status;
5141
5142         status = be_cmd_query_sfp_info(adapter);
5143         if (!status) {
5144                 dev_err(&adapter->pdev->dev,
5145                         "Unqualified SFP+ detected on %c from %s part no: %s",
5146                         adapter->port_name, adapter->phy.vendor_name,
5147                         adapter->phy.vendor_pn);
5148         }
5149         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5150 }
5151
5152 static void be_worker(struct work_struct *work)
5153 {
5154         struct be_adapter *adapter =
5155                 container_of(work, struct be_adapter, work.work);
5156         struct be_rx_obj *rxo;
5157         int i;
5158
5159         /* when interrupts are not yet enabled, just reap any pending
5160         * mcc completions */
5161         if (!netif_running(adapter->netdev)) {
5162                 local_bh_disable();
5163                 be_process_mcc(adapter);
5164                 local_bh_enable();
5165                 goto reschedule;
5166         }
5167
5168         if (!adapter->stats_cmd_sent) {
5169                 if (lancer_chip(adapter))
5170                         lancer_cmd_get_pport_stats(adapter,
5171                                                    &adapter->stats_cmd);
5172                 else
5173                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5174         }
5175
5176         if (be_physfn(adapter) &&
5177             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5178                 be_cmd_get_die_temperature(adapter);
5179
5180         for_all_rx_queues(adapter, rxo, i) {
5181                 /* Replenish RX-queues starved due to memory
5182                  * allocation failures.
5183                  */
5184                 if (rxo->rx_post_starved)
5185                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5186         }
5187
5188         be_eqd_update(adapter);
5189
5190         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5191                 be_log_sfp_info(adapter);
5192
5193 reschedule:
5194         adapter->work_counter++;
5195         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5196 }
5197
5198 /* If any VFs are already enabled don't FLR the PF */
5199 static bool be_reset_required(struct be_adapter *adapter)
5200 {
5201         return pci_num_vf(adapter->pdev) ? false : true;
5202 }
5203
5204 static char *mc_name(struct be_adapter *adapter)
5205 {
5206         char *str = ""; /* default */
5207
5208         switch (adapter->mc_type) {
5209         case UMC:
5210                 str = "UMC";
5211                 break;
5212         case FLEX10:
5213                 str = "FLEX10";
5214                 break;
5215         case vNIC1:
5216                 str = "vNIC-1";
5217                 break;
5218         case nPAR:
5219                 str = "nPAR";
5220                 break;
5221         case UFP:
5222                 str = "UFP";
5223                 break;
5224         case vNIC2:
5225                 str = "vNIC-2";
5226                 break;
5227         default:
5228                 str = "";
5229         }
5230
5231         return str;
5232 }
5233
5234 static inline char *func_name(struct be_adapter *adapter)
5235 {
5236         return be_physfn(adapter) ? "PF" : "VF";
5237 }
5238
5239 static inline char *nic_name(struct pci_dev *pdev)
5240 {
5241         switch (pdev->device) {
5242         case OC_DEVICE_ID1:
5243                 return OC_NAME;
5244         case OC_DEVICE_ID2:
5245                 return OC_NAME_BE;
5246         case OC_DEVICE_ID3:
5247         case OC_DEVICE_ID4:
5248                 return OC_NAME_LANCER;
5249         case BE_DEVICE_ID2:
5250                 return BE3_NAME;
5251         case OC_DEVICE_ID5:
5252         case OC_DEVICE_ID6:
5253                 return OC_NAME_SH;
5254         default:
5255                 return BE_NAME;
5256         }
5257 }
5258
5259 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5260 {
5261         struct be_adapter *adapter;
5262         struct net_device *netdev;
5263         int status = 0;
5264
5265         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5266
5267         status = pci_enable_device(pdev);
5268         if (status)
5269                 goto do_none;
5270
5271         status = pci_request_regions(pdev, DRV_NAME);
5272         if (status)
5273                 goto disable_dev;
5274         pci_set_master(pdev);
5275
5276         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5277         if (!netdev) {
5278                 status = -ENOMEM;
5279                 goto rel_reg;
5280         }
5281         adapter = netdev_priv(netdev);
5282         adapter->pdev = pdev;
5283         pci_set_drvdata(pdev, adapter);
5284         adapter->netdev = netdev;
5285         SET_NETDEV_DEV(netdev, &pdev->dev);
5286
5287         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5288         if (!status) {
5289                 netdev->features |= NETIF_F_HIGHDMA;
5290         } else {
5291                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5292                 if (status) {
5293                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5294                         goto free_netdev;
5295                 }
5296         }
5297
5298         status = pci_enable_pcie_error_reporting(pdev);
5299         if (!status)
5300                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5301
5302         status = be_ctrl_init(adapter);
5303         if (status)
5304                 goto free_netdev;
5305
5306         /* sync up with fw's ready state */
5307         if (be_physfn(adapter)) {
5308                 status = be_fw_wait_ready(adapter);
5309                 if (status)
5310                         goto ctrl_clean;
5311         }
5312
5313         if (be_reset_required(adapter)) {
5314                 status = be_cmd_reset_function(adapter);
5315                 if (status)
5316                         goto ctrl_clean;
5317
5318                 /* Wait for interrupts to quiesce after an FLR */
5319                 msleep(100);
5320         }
5321
5322         /* Allow interrupts for other ULPs running on NIC function */
5323         be_intr_set(adapter, true);
5324
5325         /* tell fw we're ready to fire cmds */
5326         status = be_cmd_fw_init(adapter);
5327         if (status)
5328                 goto ctrl_clean;
5329
5330         status = be_stats_init(adapter);
5331         if (status)
5332                 goto ctrl_clean;
5333
5334         status = be_get_initial_config(adapter);
5335         if (status)
5336                 goto stats_clean;
5337
5338         INIT_DELAYED_WORK(&adapter->work, be_worker);
5339         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5340         adapter->rx_fc = true;
5341         adapter->tx_fc = true;
5342
5343         status = be_setup(adapter);
5344         if (status)
5345                 goto stats_clean;
5346
5347         be_netdev_init(netdev);
5348         status = register_netdev(netdev);
5349         if (status != 0)
5350                 goto unsetup;
5351
5352         be_roce_dev_add(adapter);
5353
5354         schedule_delayed_work(&adapter->func_recovery_work,
5355                               msecs_to_jiffies(1000));
5356
5357         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5358                  func_name(adapter), mc_name(adapter), adapter->port_name);
5359
5360         return 0;
5361
5362 unsetup:
5363         be_clear(adapter);
5364 stats_clean:
5365         be_stats_cleanup(adapter);
5366 ctrl_clean:
5367         be_ctrl_cleanup(adapter);
5368 free_netdev:
5369         free_netdev(netdev);
5370 rel_reg:
5371         pci_release_regions(pdev);
5372 disable_dev:
5373         pci_disable_device(pdev);
5374 do_none:
5375         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5376         return status;
5377 }
5378
5379 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5380 {
5381         struct be_adapter *adapter = pci_get_drvdata(pdev);
5382         struct net_device *netdev =  adapter->netdev;
5383
5384         if (adapter->wol_en)
5385                 be_setup_wol(adapter, true);
5386
5387         be_intr_set(adapter, false);
5388         cancel_delayed_work_sync(&adapter->func_recovery_work);
5389
5390         netif_device_detach(netdev);
5391         if (netif_running(netdev)) {
5392                 rtnl_lock();
5393                 be_close(netdev);
5394                 rtnl_unlock();
5395         }
5396         be_clear(adapter);
5397
5398         pci_save_state(pdev);
5399         pci_disable_device(pdev);
5400         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5401         return 0;
5402 }
5403
5404 static int be_resume(struct pci_dev *pdev)
5405 {
5406         int status = 0;
5407         struct be_adapter *adapter = pci_get_drvdata(pdev);
5408         struct net_device *netdev =  adapter->netdev;
5409
5410         netif_device_detach(netdev);
5411
5412         status = pci_enable_device(pdev);
5413         if (status)
5414                 return status;
5415
5416         pci_set_power_state(pdev, PCI_D0);
5417         pci_restore_state(pdev);
5418
5419         status = be_fw_wait_ready(adapter);
5420         if (status)
5421                 return status;
5422
5423         status = be_cmd_reset_function(adapter);
5424         if (status)
5425                 return status;
5426
5427         be_intr_set(adapter, true);
5428         /* tell fw we're ready to fire cmds */
5429         status = be_cmd_fw_init(adapter);
5430         if (status)
5431                 return status;
5432
5433         be_setup(adapter);
5434         if (netif_running(netdev)) {
5435                 rtnl_lock();
5436                 be_open(netdev);
5437                 rtnl_unlock();
5438         }
5439
5440         schedule_delayed_work(&adapter->func_recovery_work,
5441                               msecs_to_jiffies(1000));
5442         netif_device_attach(netdev);
5443
5444         if (adapter->wol_en)
5445                 be_setup_wol(adapter, false);
5446
5447         return 0;
5448 }
5449
5450 /*
5451  * An FLR will stop BE from DMAing any data.
5452  */
5453 static void be_shutdown(struct pci_dev *pdev)
5454 {
5455         struct be_adapter *adapter = pci_get_drvdata(pdev);
5456
5457         if (!adapter)
5458                 return;
5459
5460         be_roce_dev_shutdown(adapter);
5461         cancel_delayed_work_sync(&adapter->work);
5462         cancel_delayed_work_sync(&adapter->func_recovery_work);
5463
5464         netif_device_detach(adapter->netdev);
5465
5466         be_cmd_reset_function(adapter);
5467
5468         pci_disable_device(pdev);
5469 }
5470
5471 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5472                                             pci_channel_state_t state)
5473 {
5474         struct be_adapter *adapter = pci_get_drvdata(pdev);
5475         struct net_device *netdev =  adapter->netdev;
5476
5477         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5478
5479         if (!adapter->eeh_error) {
5480                 adapter->eeh_error = true;
5481
5482                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5483
5484                 rtnl_lock();
5485                 netif_device_detach(netdev);
5486                 if (netif_running(netdev))
5487                         be_close(netdev);
5488                 rtnl_unlock();
5489
5490                 be_clear(adapter);
5491         }
5492
5493         if (state == pci_channel_io_perm_failure)
5494                 return PCI_ERS_RESULT_DISCONNECT;
5495
5496         pci_disable_device(pdev);
5497
5498         /* The error could cause the FW to trigger a flash debug dump.
5499          * Resetting the card while flash dump is in progress
5500          * can cause it not to recover; wait for it to finish.
5501          * Wait only for first function as it is needed only once per
5502          * adapter.
5503          */
5504         if (pdev->devfn == 0)
5505                 ssleep(30);
5506
5507         return PCI_ERS_RESULT_NEED_RESET;
5508 }
5509
5510 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5511 {
5512         struct be_adapter *adapter = pci_get_drvdata(pdev);
5513         int status;
5514
5515         dev_info(&adapter->pdev->dev, "EEH reset\n");
5516
5517         status = pci_enable_device(pdev);
5518         if (status)
5519                 return PCI_ERS_RESULT_DISCONNECT;
5520
5521         pci_set_master(pdev);
5522         pci_set_power_state(pdev, PCI_D0);
5523         pci_restore_state(pdev);
5524
5525         /* Check if card is ok and fw is ready */
5526         dev_info(&adapter->pdev->dev,
5527                  "Waiting for FW to be ready after EEH reset\n");
5528         status = be_fw_wait_ready(adapter);
5529         if (status)
5530                 return PCI_ERS_RESULT_DISCONNECT;
5531
5532         pci_cleanup_aer_uncorrect_error_status(pdev);
5533         be_clear_all_error(adapter);
5534         return PCI_ERS_RESULT_RECOVERED;
5535 }
5536
5537 static void be_eeh_resume(struct pci_dev *pdev)
5538 {
5539         int status = 0;
5540         struct be_adapter *adapter = pci_get_drvdata(pdev);
5541         struct net_device *netdev =  adapter->netdev;
5542
5543         dev_info(&adapter->pdev->dev, "EEH resume\n");
5544
5545         pci_save_state(pdev);
5546
5547         status = be_cmd_reset_function(adapter);
5548         if (status)
5549                 goto err;
5550
5551         /* On some BE3 FW versions, after a HW reset,
5552          * interrupts will remain disabled for each function.
5553          * So, explicitly enable interrupts
5554          */
5555         be_intr_set(adapter, true);
5556
5557         /* tell fw we're ready to fire cmds */
5558         status = be_cmd_fw_init(adapter);
5559         if (status)
5560                 goto err;
5561
5562         status = be_setup(adapter);
5563         if (status)
5564                 goto err;
5565
5566         if (netif_running(netdev)) {
5567                 status = be_open(netdev);
5568                 if (status)
5569                         goto err;
5570         }
5571
5572         schedule_delayed_work(&adapter->func_recovery_work,
5573                               msecs_to_jiffies(1000));
5574         netif_device_attach(netdev);
5575         return;
5576 err:
5577         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5578 }
5579
5580 static const struct pci_error_handlers be_eeh_handlers = {
5581         .error_detected = be_eeh_err_detected,
5582         .slot_reset = be_eeh_reset,
5583         .resume = be_eeh_resume,
5584 };
5585
5586 static struct pci_driver be_driver = {
5587         .name = DRV_NAME,
5588         .id_table = be_dev_ids,
5589         .probe = be_probe,
5590         .remove = be_remove,
5591         .suspend = be_suspend,
5592         .resume = be_resume,
5593         .shutdown = be_shutdown,
5594         .err_handler = &be_eeh_handlers
5595 };
5596
5597 static int __init be_init_module(void)
5598 {
5599         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5600             rx_frag_size != 2048) {
5601                 printk(KERN_WARNING DRV_NAME
5602                         " : Module param rx_frag_size must be 2048/4096/8192."
5603                         " Using 2048\n");
5604                 rx_frag_size = 2048;
5605         }
5606
5607         return pci_register_driver(&be_driver);
5608 }
5609 module_init(be_init_module);
5610
5611 static void __exit be_exit_module(void)
5612 {
5613         pci_unregister_driver(&be_driver);
5614 }
5615 module_exit(be_exit_module);