drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 static unsigned int num_vfs;
  34 module_param(num_vfs, uint, S_IRUGO);
  35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  36
  37 static ushort rx_frag_size = 2048;
  38 module_param(rx_frag_size, ushort, S_IRUGO);
  39 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  40
  41 static const struct pci_device_id be_dev_ids[] = {
  42         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  44         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  46         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  50         { 0 }
  51 };
  52 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  53 /* UE Status Low CSR */
  54 static const char * const ue_status_low_desc[] = {
  55         "CEV",
  56         "CTX",
  57         "DBUF",
  58         "ERX",
  59         "Host",
  60         "MPU",
  61         "NDMA",
  62         "PTC ",
  63         "RDMA ",
  64         "RXF ",
  65         "RXIPS ",
  66         "RXULP0 ",
  67         "RXULP1 ",
  68         "RXULP2 ",
  69         "TIM ",
  70         "TPOST ",
  71         "TPRE ",
  72         "TXIPS ",
  73         "TXULP0 ",
  74         "TXULP1 ",
  75         "UC ",
  76         "WDMA ",
  77         "TXULP2 ",
  78         "HOST1 ",
  79         "P0_OB_LINK ",
  80         "P1_OB_LINK ",
  81         "HOST_GPIO ",
  82         "MBOX ",
  83         "ERX2 ",
  84         "SPARE ",
  85         "JTAG ",
  86         "MPU_INTPEND "
  87 };
  88
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 126 {
 127         struct be_dma_mem *mem = &q->dma_mem;
 128
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190
 191         val |= qid & DB_RQ_RING_ID_MASK;
 192         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 193
 194         wmb();
 195         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 196 }
 197
 198 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 199                           u16 posted)
 200 {
 201         u32 val = 0;
 202
 203         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 204         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 205
 206         wmb();
 207         iowrite32(val, adapter->db + txo->db_offset);
 208 }
 209
 210 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 211                          bool arm, bool clear_int, u16 num_popped)
 212 {
 213         u32 val = 0;
 214
 215         val |= qid & DB_EQ_RING_ID_MASK;
 216         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 217
 218         if (adapter->eeh_error)
 219                 return;
 220
 221         if (arm)
 222                 val |= 1 << DB_EQ_REARM_SHIFT;
 223         if (clear_int)
 224                 val |= 1 << DB_EQ_CLR_SHIFT;
 225         val |= 1 << DB_EQ_EVNT_SHIFT;
 226         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 227         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 228 }
 229
 230 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 231 {
 232         u32 val = 0;
 233
 234         val |= qid & DB_CQ_RING_ID_MASK;
 235         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 236                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 237
 238         if (adapter->eeh_error)
 239                 return;
 240
 241         if (arm)
 242                 val |= 1 << DB_CQ_REARM_SHIFT;
 243         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 244         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 245 }
 246
 247 static int be_mac_addr_set(struct net_device *netdev, void *p)
 248 {
 249         struct be_adapter *adapter = netdev_priv(netdev);
 250         struct device *dev = &adapter->pdev->dev;
 251         struct sockaddr *addr = p;
 252         int status;
 253         u8 mac[ETH_ALEN];
 254         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 255
 256         if (!is_valid_ether_addr(addr->sa_data))
 257                 return -EADDRNOTAVAIL;
 258
 259         /* Proceed further only if, User provided MAC is different
 260          * from active MAC
 261          */
 262         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 263                 return 0;
 264
 265         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 266          * privilege or if PF did not provision the new MAC address.
 267          * On BE3, this cmd will always fail if the VF doesn't have the
 268          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 269          * the MAC for the VF.
 270          */
 271         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 272                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 273         if (!status) {
 274                 curr_pmac_id = adapter->pmac_id[0];
 275
 276                 /* Delete the old programmed MAC. This call may fail if the
 277                  * old MAC was already deleted by the PF driver.
 278                  */
 279                 if (adapter->pmac_id[0] != old_pmac_id)
 280                         be_cmd_pmac_del(adapter, adapter->if_handle,
 281                                         old_pmac_id, 0);
 282         }
 283
 284         /* Decide if the new MAC is successfully activated only after
 285          * querying the FW
 286          */
 287         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 288                                        adapter->if_handle, true, 0);
 289         if (status)
 290                 goto err;
 291
 292         /* The MAC change did not happen, either due to lack of privilege
 293          * or PF didn't pre-provision.
 294          */
 295         if (!ether_addr_equal(addr->sa_data, mac)) {
 296                 status = -EPERM;
 297                 goto err;
 298         }
 299
 300         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 301         dev_info(dev, "MAC address changed to %pM\n", mac);
 302         return 0;
 303 err:
 304         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 305         return status;
 306 }
 307
 308 /* BE2 supports only v0 cmd */
 309 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 310 {
 311         if (BE2_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else if (BE3_chip(adapter)) {
 316                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         } else {
 320                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 321
 322                 return &cmd->hw_stats;
 323         }
 324 }
 325
 326 /* BE2 supports only v0 cmd */
 327 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 328 {
 329         if (BE2_chip(adapter)) {
 330                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else if (BE3_chip(adapter)) {
 334                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         } else {
 338                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 339
 340                 return &hw_stats->erx;
 341         }
 342 }
 343
 344 static void populate_be_v0_stats(struct be_adapter *adapter)
 345 {
 346         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 347         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 348         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 349         struct be_port_rxf_stats_v0 *port_stats =
 350                                         &rxf_stats->port[adapter->port_num];
 351         struct be_drv_stats *drvs = &adapter->drv_stats;
 352
 353         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 354         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 355         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 356         drvs->rx_control_frames = port_stats->rx_control_frames;
 357         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 358         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 359         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 360         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 361         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 362         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 363         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 364         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 365         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 366         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 367         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 368         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 369         drvs->rx_dropped_header_too_small =
 370                 port_stats->rx_dropped_header_too_small;
 371         drvs->rx_address_filtered =
 372                                         port_stats->rx_address_filtered +
 373                                         port_stats->rx_vlan_filtered;
 374         drvs->rx_alignment_symbol_errors =
 375                 port_stats->rx_alignment_symbol_errors;
 376
 377         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 378         drvs->tx_controlframes = port_stats->tx_controlframes;
 379
 380         if (adapter->port_num)
 381                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 382         else
 383                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 384         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 385         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 386         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 387         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 388         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 389         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 390         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 391 }
 392
 393 static void populate_be_v1_stats(struct be_adapter *adapter)
 394 {
 395         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 396         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 397         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 398         struct be_port_rxf_stats_v1 *port_stats =
 399                                         &rxf_stats->port[adapter->port_num];
 400         struct be_drv_stats *drvs = &adapter->drv_stats;
 401
 402         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 403         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 404         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 405         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 406         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 407         drvs->rx_control_frames = port_stats->rx_control_frames;
 408         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 409         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 410         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 411         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 412         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 413         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 414         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 415         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 416         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 417         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 418         drvs->rx_dropped_header_too_small =
 419                 port_stats->rx_dropped_header_too_small;
 420         drvs->rx_input_fifo_overflow_drop =
 421                 port_stats->rx_input_fifo_overflow_drop;
 422         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 423         drvs->rx_alignment_symbol_errors =
 424                 port_stats->rx_alignment_symbol_errors;
 425         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 426         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 427         drvs->tx_controlframes = port_stats->tx_controlframes;
 428         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 429         drvs->jabber_events = port_stats->jabber_events;
 430         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 431         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 432         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 433         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 434         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 435         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 436         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 437 }
 438
 439 static void populate_be_v2_stats(struct be_adapter *adapter)
 440 {
 441         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 442         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 443         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 444         struct be_port_rxf_stats_v2 *port_stats =
 445                                         &rxf_stats->port[adapter->port_num];
 446         struct be_drv_stats *drvs = &adapter->drv_stats;
 447
 448         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 449         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 450         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 451         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 452         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 453         drvs->rx_control_frames = port_stats->rx_control_frames;
 454         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 455         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 456         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 457         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 458         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 459         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 460         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 461         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 462         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 463         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 464         drvs->rx_dropped_header_too_small =
 465                 port_stats->rx_dropped_header_too_small;
 466         drvs->rx_input_fifo_overflow_drop =
 467                 port_stats->rx_input_fifo_overflow_drop;
 468         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 469         drvs->rx_alignment_symbol_errors =
 470                 port_stats->rx_alignment_symbol_errors;
 471         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 472         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 473         drvs->tx_controlframes = port_stats->tx_controlframes;
 474         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 475         drvs->jabber_events = port_stats->jabber_events;
 476         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 477         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 478         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 479         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 480         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 481         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 482         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 483         if (be_roce_supported(adapter)) {
 484                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 485                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 486                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 487                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 488                 drvs->roce_drops_payload_len =
 489                         port_stats->roce_drops_payload_len;
 490         }
 491 }
 492
 493 static void populate_lancer_stats(struct be_adapter *adapter)
 494 {
 495         struct be_drv_stats *drvs = &adapter->drv_stats;
 496         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 497
 498         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 499         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 500         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 501         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 502         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 503         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 504         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 505         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 506         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 507         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 508         drvs->rx_dropped_tcp_length =
 509                                 pport_stats->rx_dropped_invalid_tcp_length;
 510         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 511         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 512         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 513         drvs->rx_dropped_header_too_small =
 514                                 pport_stats->rx_dropped_header_too_small;
 515         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 516         drvs->rx_address_filtered =
 517                                         pport_stats->rx_address_filtered +
 518                                         pport_stats->rx_vlan_filtered;
 519         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 520         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 521         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 522         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 523         drvs->jabber_events = pport_stats->rx_jabbers;
 524         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 525         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 526         drvs->rx_drops_too_many_frags =
 527                                 pport_stats->rx_drops_too_many_frags_lo;
 528 }
 529
 530 static void accumulate_16bit_val(u32 *acc, u16 val)
 531 {
 532 #define lo(x)                   (x & 0xFFFF)
 533 #define hi(x)                   (x & 0xFFFF0000)
 534         bool wrapped = val < lo(*acc);
 535         u32 newacc = hi(*acc) + val;
 536
 537         if (wrapped)
 538                 newacc += 65536;
 539         ACCESS_ONCE(*acc) = newacc;
 540 }
 541
 542 static void populate_erx_stats(struct be_adapter *adapter,
 543                                struct be_rx_obj *rxo, u32 erx_stat)
 544 {
 545         if (!BEx_chip(adapter))
 546                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 547         else
 548                 /* below erx HW counter can actually wrap around after
 549                  * 65535. Driver accumulates a 32-bit value
 550                  */
 551                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 552                                      (u16)erx_stat);
 553 }
 554
 555 void be_parse_stats(struct be_adapter *adapter)
 556 {
 557         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 558         struct be_rx_obj *rxo;
 559         int i;
 560         u32 erx_stat;
 561
 562         if (lancer_chip(adapter)) {
 563                 populate_lancer_stats(adapter);
 564         } else {
 565                 if (BE2_chip(adapter))
 566                         populate_be_v0_stats(adapter);
 567                 else if (BE3_chip(adapter))
 568                         /* for BE3 */
 569                         populate_be_v1_stats(adapter);
 570                 else
 571                         populate_be_v2_stats(adapter);
 572
 573                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 574                 for_all_rx_queues(adapter, rxo, i) {
 575                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 576                         populate_erx_stats(adapter, rxo, erx_stat);
 577                 }
 578         }
 579 }
 580
 581 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 582                                                 struct rtnl_link_stats64 *stats)
 583 {
 584         struct be_adapter *adapter = netdev_priv(netdev);
 585         struct be_drv_stats *drvs = &adapter->drv_stats;
 586         struct be_rx_obj *rxo;
 587         struct be_tx_obj *txo;
 588         u64 pkts, bytes;
 589         unsigned int start;
 590         int i;
 591
 592         for_all_rx_queues(adapter, rxo, i) {
 593                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 594
 595                 do {
 596                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 597                         pkts = rx_stats(rxo)->rx_pkts;
 598                         bytes = rx_stats(rxo)->rx_bytes;
 599                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 600                 stats->rx_packets += pkts;
 601                 stats->rx_bytes += bytes;
 602                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 603                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 604                                         rx_stats(rxo)->rx_drops_no_frags;
 605         }
 606
 607         for_all_tx_queues(adapter, txo, i) {
 608                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 609
 610                 do {
 611                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 612                         pkts = tx_stats(txo)->tx_pkts;
 613                         bytes = tx_stats(txo)->tx_bytes;
 614                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 615                 stats->tx_packets += pkts;
 616                 stats->tx_bytes += bytes;
 617         }
 618
 619         /* bad pkts received */
 620         stats->rx_errors = drvs->rx_crc_errors +
 621                 drvs->rx_alignment_symbol_errors +
 622                 drvs->rx_in_range_errors +
 623                 drvs->rx_out_range_errors +
 624                 drvs->rx_frame_too_long +
 625                 drvs->rx_dropped_too_small +
 626                 drvs->rx_dropped_too_short +
 627                 drvs->rx_dropped_header_too_small +
 628                 drvs->rx_dropped_tcp_length +
 629                 drvs->rx_dropped_runt;
 630
 631         /* detailed rx errors */
 632         stats->rx_length_errors = drvs->rx_in_range_errors +
 633                 drvs->rx_out_range_errors +
 634                 drvs->rx_frame_too_long;
 635
 636         stats->rx_crc_errors = drvs->rx_crc_errors;
 637
 638         /* frame alignment errors */
 639         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 640
 641         /* receiver fifo overrun */
 642         /* drops_no_pbuf is no per i/f, it's per BE card */
 643         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 644                                 drvs->rx_input_fifo_overflow_drop +
 645                                 drvs->rx_drops_no_pbuf;
 646         return stats;
 647 }
 648
 649 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 650 {
 651         struct net_device *netdev = adapter->netdev;
 652
 653         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 654                 netif_carrier_off(netdev);
 655                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 656         }
 657
 658         if (link_status)
 659                 netif_carrier_on(netdev);
 660         else
 661                 netif_carrier_off(netdev);
 662 }
 663
 664 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 665 {
 666         struct be_tx_stats *stats = tx_stats(txo);
 667
 668         u64_stats_update_begin(&stats->sync);
 669         stats->tx_reqs++;
 670         stats->tx_bytes += skb->len;
 671         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Returns number of WRBs needed for the skb */
 676 static u32 skb_wrb_cnt(struct sk_buff *skb)
 677 {
 678         /* +1 for the header wrb */
 679         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 680 }
 681
 682 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 683 {
 684         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 685         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 686         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 687         wrb->rsvd0 = 0;
 688 }
 689
 690 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 691  * to avoid the swap and shift/mask operations in wrb_fill().
 692  */
 693 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 694 {
 695         wrb->frag_pa_hi = 0;
 696         wrb->frag_pa_lo = 0;
 697         wrb->frag_len = 0;
 698         wrb->rsvd0 = 0;
 699 }
 700
 701 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 702                                      struct sk_buff *skb)
 703 {
 704         u8 vlan_prio;
 705         u16 vlan_tag;
 706
 707         vlan_tag = skb_vlan_tag_get(skb);
 708         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 709         /* If vlan priority provided by OS is NOT in available bmap */
 710         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 711                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 712                                 adapter->recommended_prio;
 713
 714         return vlan_tag;
 715 }
 716
 717 /* Used only for IP tunnel packets */
 718 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 719 {
 720         return (inner_ip_hdr(skb)->version == 4) ?
 721                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 722 }
 723
 724 static u16 skb_ip_proto(struct sk_buff *skb)
 725 {
 726         return (ip_hdr(skb)->version == 4) ?
 727                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 728 }
 729
 730 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 731                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 732                          bool skip_hw_vlan)
 733 {
 734         u16 vlan_tag, proto;
 735
 736         memset(hdr, 0, sizeof(*hdr));
 737
 738         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 739
 740         if (skb_is_gso(skb)) {
 741                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 742                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 743                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 744                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 745         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 746                 if (skb->encapsulation) {
 747                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 748                         proto = skb_inner_ip_proto(skb);
 749                 } else {
 750                         proto = skb_ip_proto(skb);
 751                 }
 752                 if (proto == IPPROTO_TCP)
 753                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 754                 else if (proto == IPPROTO_UDP)
 755                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 756         }
 757
 758         if (skb_vlan_tag_present(skb)) {
 759                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 760                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 761                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 762         }
 763
 764         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 765         SET_TX_WRB_HDR_BITS(len, hdr, len);
 766
 767         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0
 768          * When this hack is not needed, the evt bit is set while ringing DB
 769          */
 770         if (skip_hw_vlan)
 771                 SET_TX_WRB_HDR_BITS(event, hdr, 1);
 772 }
 773
 774 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 775                           bool unmap_single)
 776 {
 777         dma_addr_t dma;
 778         u32 frag_len = le32_to_cpu(wrb->frag_len);
 779
 780
 781         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 782                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 783         if (frag_len) {
 784                 if (unmap_single)
 785                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 786                 else
 787                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 788         }
 789 }
 790
 791 /* Returns the number of WRBs used up by the skb */
 792 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 793                            struct sk_buff *skb, bool skip_hw_vlan)
 794 {
 795         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 796         struct device *dev = &adapter->pdev->dev;
 797         struct be_queue_info *txq = &txo->q;
 798         struct be_eth_hdr_wrb *hdr;
 799         bool map_single = false;
 800         struct be_eth_wrb *wrb;
 801         dma_addr_t busaddr;
 802         u16 head = txq->head;
 803
 804         hdr = queue_head_node(txq);
 805         wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan);
 806         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 807
 808         queue_head_inc(txq);
 809
 810         if (skb->len > skb->data_len) {
 811                 int len = skb_headlen(skb);
 812
 813                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 814                 if (dma_mapping_error(dev, busaddr))
 815                         goto dma_err;
 816                 map_single = true;
 817                 wrb = queue_head_node(txq);
 818                 wrb_fill(wrb, busaddr, len);
 819                 queue_head_inc(txq);
 820                 copied += len;
 821         }
 822
 823         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 824                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 825
 826                 busaddr = skb_frag_dma_map(dev, frag, 0,
 827                                            skb_frag_size(frag), DMA_TO_DEVICE);
 828                 if (dma_mapping_error(dev, busaddr))
 829                         goto dma_err;
 830                 wrb = queue_head_node(txq);
 831                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 832                 queue_head_inc(txq);
 833                 copied += skb_frag_size(frag);
 834         }
 835
 836         BUG_ON(txo->sent_skb_list[head]);
 837         txo->sent_skb_list[head] = skb;
 838         txo->last_req_hdr = head;
 839         atomic_add(wrb_cnt, &txq->used);
 840         txo->last_req_wrb_cnt = wrb_cnt;
 841         txo->pend_wrb_cnt += wrb_cnt;
 842
 843         be_tx_stats_update(txo, skb);
 844         return wrb_cnt;
 845
 846 dma_err:
 847         /* Bring the queue back to the state it was in before this
 848          * routine was invoked.
 849          */
 850         txq->head = head;
 851         /* skip the first wrb (hdr); it's not mapped */
 852         queue_head_inc(txq);
 853         while (copied) {
 854                 wrb = queue_head_node(txq);
 855                 unmap_tx_frag(dev, wrb, map_single);
 856                 map_single = false;
 857                 copied -= le32_to_cpu(wrb->frag_len);
 858                 adapter->drv_stats.dma_map_errors++;
 859                 queue_head_inc(txq);
 860         }
 861         txq->head = head;
 862         return 0;
 863 }
 864
 865 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 866 {
 867         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 868 }
 869
 870 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 871                                              struct sk_buff *skb,
 872                                              bool *skip_hw_vlan)
 873 {
 874         u16 vlan_tag = 0;
 875
 876         skb = skb_share_check(skb, GFP_ATOMIC);
 877         if (unlikely(!skb))
 878                 return skb;
 879
 880         if (skb_vlan_tag_present(skb))
 881                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 882
 883         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 884                 if (!vlan_tag)
 885                         vlan_tag = adapter->pvid;
 886                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 887                  * skip VLAN insertion
 888                  */
 889                 if (skip_hw_vlan)
 890                         *skip_hw_vlan = true;
 891         }
 892
 893         if (vlan_tag) {
 894                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 895                                                 vlan_tag);
 896                 if (unlikely(!skb))
 897                         return skb;
 898                 skb->vlan_tci = 0;
 899         }
 900
 901         /* Insert the outer VLAN, if any */
 902         if (adapter->qnq_vid) {
 903                 vlan_tag = adapter->qnq_vid;
 904                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 905                                                 vlan_tag);
 906                 if (unlikely(!skb))
 907                         return skb;
 908                 if (skip_hw_vlan)
 909                         *skip_hw_vlan = true;
 910         }
 911
 912         return skb;
 913 }
 914
 915 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 916 {
 917         struct ethhdr *eh = (struct ethhdr *)skb->data;
 918         u16 offset = ETH_HLEN;
 919
 920         if (eh->h_proto == htons(ETH_P_IPV6)) {
 921                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 922
 923                 offset += sizeof(struct ipv6hdr);
 924                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 925                     ip6h->nexthdr != NEXTHDR_UDP) {
 926                         struct ipv6_opt_hdr *ehdr =
 927                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 928
 929                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 930                         if (ehdr->hdrlen == 0xff)
 931                                 return true;
 932                 }
 933         }
 934         return false;
 935 }
 936
 937 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 938 {
 939         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 940 }
 941
 942 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 943 {
 944         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 945 }
 946
 947 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 948                                                   struct sk_buff *skb,
 949                                                   bool *skip_hw_vlan)
 950 {
 951         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 952         unsigned int eth_hdr_len;
 953         struct iphdr *ip;
 954
 955         /* For padded packets, BE HW modifies tot_len field in IP header
 956          * incorrecly when VLAN tag is inserted by HW.
 957          * For padded packets, Lancer computes incorrect checksum.
 958          */
 959         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 960                                                 VLAN_ETH_HLEN : ETH_HLEN;
 961         if (skb->len <= 60 &&
 962             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
 963             is_ipv4_pkt(skb)) {
 964                 ip = (struct iphdr *)ip_hdr(skb);
 965                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 966         }
 967
 968         /* If vlan tag is already inlined in the packet, skip HW VLAN
 969          * tagging in pvid-tagging mode
 970          */
 971         if (be_pvid_tagging_enabled(adapter) &&
 972             veh->h_vlan_proto == htons(ETH_P_8021Q))
 973                 *skip_hw_vlan = true;
 974
 975         /* HW has a bug wherein it will calculate CSUM for VLAN
 976          * pkts even though it is disabled.
 977          * Manually insert VLAN in pkt.
 978          */
 979         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 980             skb_vlan_tag_present(skb)) {
 981                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 982                 if (unlikely(!skb))
 983                         goto err;
 984         }
 985
 986         /* HW may lockup when VLAN HW tagging is requested on
 987          * certain ipv6 packets. Drop such pkts if the HW workaround to
 988          * skip HW tagging is not enabled by FW.
 989          */
 990         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 991                      (adapter->pvid || adapter->qnq_vid) &&
 992                      !qnq_async_evt_rcvd(adapter)))
 993                 goto tx_drop;
 994
 995         /* Manual VLAN tag insertion to prevent:
 996          * ASIC lockup when the ASIC inserts VLAN tag into
 997          * certain ipv6 packets. Insert VLAN tags in driver,
 998          * and set event, completion, vlan bits accordingly
 999          * in the Tx WRB.
1000          */
1001         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1002             be_vlan_tag_tx_chk(adapter, skb)) {
1003                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1004                 if (unlikely(!skb))
1005                         goto err;
1006         }
1007
1008         return skb;
1009 tx_drop:
1010         dev_kfree_skb_any(skb);
1011 err:
1012         return NULL;
1013 }
1014
1015 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1016                                            struct sk_buff *skb,
1017                                            bool *skip_hw_vlan)
1018 {
1019         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1020          * less may cause a transmit stall on that port. So the work-around is
1021          * to pad short packets (<= 32 bytes) to a 36-byte length.
1022          */
1023         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1024                 if (skb_put_padto(skb, 36))
1025                         return NULL;
1026         }
1027
1028         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1029                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1030                 if (!skb)
1031                         return NULL;
1032         }
1033
1034         return skb;
1035 }
1036
1037 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1038 {
1039         struct be_queue_info *txq = &txo->q;
1040         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1041
1042         /* Mark the last request eventable if it hasn't been marked already */
1043         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1044                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1045
1046         /* compose a dummy wrb if there are odd set of wrbs to notify */
1047         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1048                 wrb_fill_dummy(queue_head_node(txq));
1049                 queue_head_inc(txq);
1050                 atomic_inc(&txq->used);
1051                 txo->pend_wrb_cnt++;
1052                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1053                                            TX_HDR_WRB_NUM_SHIFT);
1054                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1055                                           TX_HDR_WRB_NUM_SHIFT);
1056         }
1057         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1058         txo->pend_wrb_cnt = 0;
1059 }
1060
1061 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1062 {
1063         bool skip_hw_vlan = false, flush = !skb->xmit_more;
1064         struct be_adapter *adapter = netdev_priv(netdev);
1065         u16 q_idx = skb_get_queue_mapping(skb);
1066         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1067         struct be_queue_info *txq = &txo->q;
1068         u16 wrb_cnt;
1069
1070         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1071         if (unlikely(!skb))
1072                 goto drop;
1073
1074         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan);
1075         if (unlikely(!wrb_cnt)) {
1076                 dev_kfree_skb_any(skb);
1077                 goto drop;
1078         }
1079
1080         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1081                 netif_stop_subqueue(netdev, q_idx);
1082                 tx_stats(txo)->tx_stops++;
1083         }
1084
1085         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1086                 be_xmit_flush(adapter, txo);
1087
1088         return NETDEV_TX_OK;
1089 drop:
1090         tx_stats(txo)->tx_drv_drops++;
1091         /* Flush the already enqueued tx requests */
1092         if (flush && txo->pend_wrb_cnt)
1093                 be_xmit_flush(adapter, txo);
1094
1095         return NETDEV_TX_OK;
1096 }
1097
1098 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1099 {
1100         struct be_adapter *adapter = netdev_priv(netdev);
1101         struct device *dev = &adapter->pdev->dev;
1102
1103         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1104                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1105                          BE_MIN_MTU, BE_MAX_MTU);
1106                 return -EINVAL;
1107         }
1108
1109         dev_info(dev, "MTU changed from %d to %d bytes\n",
1110                  netdev->mtu, new_mtu);
1111         netdev->mtu = new_mtu;
1112         return 0;
1113 }
1114
1115 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1116 {
1117         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1118                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1119 }
1120
1121 static int be_set_vlan_promisc(struct be_adapter *adapter)
1122 {
1123         struct device *dev = &adapter->pdev->dev;
1124         int status;
1125
1126         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1127                 return 0;
1128
1129         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1130         if (!status) {
1131                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1132                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1133         } else {
1134                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1135         }
1136         return status;
1137 }
1138
1139 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1140 {
1141         struct device *dev = &adapter->pdev->dev;
1142         int status;
1143
1144         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1145         if (!status) {
1146                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1147                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1148         }
1149         return status;
1150 }
1151
1152 /*
1153  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1154  * If the user configures more, place BE in vlan promiscuous mode.
1155  */
1156 static int be_vid_config(struct be_adapter *adapter)
1157 {
1158         struct device *dev = &adapter->pdev->dev;
1159         u16 vids[BE_NUM_VLANS_SUPPORTED];
1160         u16 num = 0, i = 0;
1161         int status = 0;
1162
1163         /* No need to further configure vids if in promiscuous mode */
1164         if (be_in_all_promisc(adapter))
1165                 return 0;
1166
1167         if (adapter->vlans_added > be_max_vlans(adapter))
1168                 return be_set_vlan_promisc(adapter);
1169
1170         /* Construct VLAN Table to give to HW */
1171         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1172                 vids[num++] = cpu_to_le16(i);
1173
1174         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1175         if (status) {
1176                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1177                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1178                 if (addl_status(status) ==
1179                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1180                         return be_set_vlan_promisc(adapter);
1181         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1182                 status = be_clear_vlan_promisc(adapter);
1183         }
1184         return status;
1185 }
1186
1187 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1188 {
1189         struct be_adapter *adapter = netdev_priv(netdev);
1190         int status = 0;
1191
1192         /* Packets with VID 0 are always received by Lancer by default */
1193         if (lancer_chip(adapter) && vid == 0)
1194                 return status;
1195
1196         if (test_bit(vid, adapter->vids))
1197                 return status;
1198
1199         set_bit(vid, adapter->vids);
1200         adapter->vlans_added++;
1201
1202         status = be_vid_config(adapter);
1203         if (status) {
1204                 adapter->vlans_added--;
1205                 clear_bit(vid, adapter->vids);
1206         }
1207
1208         return status;
1209 }
1210
1211 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1212 {
1213         struct be_adapter *adapter = netdev_priv(netdev);
1214
1215         /* Packets with VID 0 are always received by Lancer by default */
1216         if (lancer_chip(adapter) && vid == 0)
1217                 return 0;
1218
1219         clear_bit(vid, adapter->vids);
1220         adapter->vlans_added--;
1221
1222         return be_vid_config(adapter);
1223 }
1224
1225 static void be_clear_all_promisc(struct be_adapter *adapter)
1226 {
1227         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1228         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1229 }
1230
1231 static void be_set_all_promisc(struct be_adapter *adapter)
1232 {
1233         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1234         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1235 }
1236
1237 static void be_set_mc_promisc(struct be_adapter *adapter)
1238 {
1239         int status;
1240
1241         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1242                 return;
1243
1244         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1245         if (!status)
1246                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1247 }
1248
1249 static void be_set_mc_list(struct be_adapter *adapter)
1250 {
1251         int status;
1252
1253         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1254         if (!status)
1255                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1256         else
1257                 be_set_mc_promisc(adapter);
1258 }
1259
1260 static void be_set_uc_list(struct be_adapter *adapter)
1261 {
1262         struct netdev_hw_addr *ha;
1263         int i = 1; /* First slot is claimed by the Primary MAC */
1264
1265         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1266                 be_cmd_pmac_del(adapter, adapter->if_handle,
1267                                 adapter->pmac_id[i], 0);
1268
1269         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1270                 be_set_all_promisc(adapter);
1271                 return;
1272         }
1273
1274         netdev_for_each_uc_addr(ha, adapter->netdev) {
1275                 adapter->uc_macs++; /* First slot is for Primary MAC */
1276                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1277                                 &adapter->pmac_id[adapter->uc_macs], 0);
1278         }
1279 }
1280
1281 static void be_clear_uc_list(struct be_adapter *adapter)
1282 {
1283         int i;
1284
1285         for (i = 1; i < (adapter->uc_macs + 1); i++)
1286                 be_cmd_pmac_del(adapter, adapter->if_handle,
1287                                 adapter->pmac_id[i], 0);
1288         adapter->uc_macs = 0;
1289 }
1290
1291 static void be_set_rx_mode(struct net_device *netdev)
1292 {
1293         struct be_adapter *adapter = netdev_priv(netdev);
1294
1295         if (netdev->flags & IFF_PROMISC) {
1296                 be_set_all_promisc(adapter);
1297                 return;
1298         }
1299
1300         /* Interface was previously in promiscuous mode; disable it */
1301         if (be_in_all_promisc(adapter)) {
1302                 be_clear_all_promisc(adapter);
1303                 if (adapter->vlans_added)
1304                         be_vid_config(adapter);
1305         }
1306
1307         /* Enable multicast promisc if num configured exceeds what we support */
1308         if (netdev->flags & IFF_ALLMULTI ||
1309             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1310                 be_set_mc_promisc(adapter);
1311                 return;
1312         }
1313
1314         if (netdev_uc_count(netdev) != adapter->uc_macs)
1315                 be_set_uc_list(adapter);
1316
1317         be_set_mc_list(adapter);
1318 }
1319
1320 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1321 {
1322         struct be_adapter *adapter = netdev_priv(netdev);
1323         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1324         int status;
1325
1326         if (!sriov_enabled(adapter))
1327                 return -EPERM;
1328
1329         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1330                 return -EINVAL;
1331
1332         /* Proceed further only if user provided MAC is different
1333          * from active MAC
1334          */
1335         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1336                 return 0;
1337
1338         if (BEx_chip(adapter)) {
1339                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1340                                 vf + 1);
1341
1342                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1343                                          &vf_cfg->pmac_id, vf + 1);
1344         } else {
1345                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1346                                         vf + 1);
1347         }
1348
1349         if (status) {
1350                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1351                         mac, vf, status);
1352                 return be_cmd_status(status);
1353         }
1354
1355         ether_addr_copy(vf_cfg->mac_addr, mac);
1356
1357         return 0;
1358 }
1359
1360 static int be_get_vf_config(struct net_device *netdev, int vf,
1361                             struct ifla_vf_info *vi)
1362 {
1363         struct be_adapter *adapter = netdev_priv(netdev);
1364         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1365
1366         if (!sriov_enabled(adapter))
1367                 return -EPERM;
1368
1369         if (vf >= adapter->num_vfs)
1370                 return -EINVAL;
1371
1372         vi->vf = vf;
1373         vi->max_tx_rate = vf_cfg->tx_rate;
1374         vi->min_tx_rate = 0;
1375         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1376         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1377         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1378         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1379
1380         return 0;
1381 }
1382
1383 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1384 {
1385         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1386         u16 vids[BE_NUM_VLANS_SUPPORTED];
1387         int vf_if_id = vf_cfg->if_handle;
1388         int status;
1389
1390         /* Enable Transparent VLAN Tagging */
1391         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0);
1392         if (status)
1393                 return status;
1394
1395         /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1396         vids[0] = 0;
1397         status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1398         if (!status)
1399                 dev_info(&adapter->pdev->dev,
1400                          "Cleared guest VLANs on VF%d", vf);
1401
1402         /* After TVT is enabled, disallow VFs to program VLAN filters */
1403         if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1404                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1405                                                   ~BE_PRIV_FILTMGMT, vf + 1);
1406                 if (!status)
1407                         vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1408         }
1409         return 0;
1410 }
1411
1412 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1413 {
1414         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1415         struct device *dev = &adapter->pdev->dev;
1416         int status;
1417
1418         /* Reset Transparent VLAN Tagging. */
1419         status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1420                                        vf_cfg->if_handle, 0);
1421         if (status)
1422                 return status;
1423
1424         /* Allow VFs to program VLAN filtering */
1425         if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1426                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1427                                                   BE_PRIV_FILTMGMT, vf + 1);
1428                 if (!status) {
1429                         vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1430                         dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1431                 }
1432         }
1433
1434         dev_info(dev,
1435                  "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1436         return 0;
1437 }
1438
1439 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1440 {
1441         struct be_adapter *adapter = netdev_priv(netdev);
1442         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1443         int status;
1444
1445         if (!sriov_enabled(adapter))
1446                 return -EPERM;
1447
1448         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1449                 return -EINVAL;
1450
1451         if (vlan || qos) {
1452                 vlan |= qos << VLAN_PRIO_SHIFT;
1453                 status = be_set_vf_tvt(adapter, vf, vlan);
1454         } else {
1455                 status = be_clear_vf_tvt(adapter, vf);
1456         }
1457
1458         if (status) {
1459                 dev_err(&adapter->pdev->dev,
1460                         "VLAN %d config on VF %d failed : %#x\n", vlan, vf,
1461                         status);
1462                 return be_cmd_status(status);
1463         }
1464
1465         vf_cfg->vlan_tag = vlan;
1466         return 0;
1467 }
1468
1469 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1470                              int min_tx_rate, int max_tx_rate)
1471 {
1472         struct be_adapter *adapter = netdev_priv(netdev);
1473         struct device *dev = &adapter->pdev->dev;
1474         int percent_rate, status = 0;
1475         u16 link_speed = 0;
1476         u8 link_status;
1477
1478         if (!sriov_enabled(adapter))
1479                 return -EPERM;
1480
1481         if (vf >= adapter->num_vfs)
1482                 return -EINVAL;
1483
1484         if (min_tx_rate)
1485                 return -EINVAL;
1486
1487         if (!max_tx_rate)
1488                 goto config_qos;
1489
1490         status = be_cmd_link_status_query(adapter, &link_speed,
1491                                           &link_status, 0);
1492         if (status)
1493                 goto err;
1494
1495         if (!link_status) {
1496                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1497                 status = -ENETDOWN;
1498                 goto err;
1499         }
1500
1501         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1502                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1503                         link_speed);
1504                 status = -EINVAL;
1505                 goto err;
1506         }
1507
1508         /* On Skyhawk the QOS setting must be done only as a % value */
1509         percent_rate = link_speed / 100;
1510         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1511                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1512                         percent_rate);
1513                 status = -EINVAL;
1514                 goto err;
1515         }
1516
1517 config_qos:
1518         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1519         if (status)
1520                 goto err;
1521
1522         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1523         return 0;
1524
1525 err:
1526         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1527                 max_tx_rate, vf);
1528         return be_cmd_status(status);
1529 }
1530
1531 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1532                                 int link_state)
1533 {
1534         struct be_adapter *adapter = netdev_priv(netdev);
1535         int status;
1536
1537         if (!sriov_enabled(adapter))
1538                 return -EPERM;
1539
1540         if (vf >= adapter->num_vfs)
1541                 return -EINVAL;
1542
1543         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1544         if (status) {
1545                 dev_err(&adapter->pdev->dev,
1546                         "Link state change on VF %d failed: %#x\n", vf, status);
1547                 return be_cmd_status(status);
1548         }
1549
1550         adapter->vf_cfg[vf].plink_tracking = link_state;
1551
1552         return 0;
1553 }
1554
1555 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1556                           ulong now)
1557 {
1558         aic->rx_pkts_prev = rx_pkts;
1559         aic->tx_reqs_prev = tx_pkts;
1560         aic->jiffies = now;
1561 }
1562
1563 static void be_eqd_update(struct be_adapter *adapter)
1564 {
1565         struct be_set_eqd set_eqd[MAX_EVT_QS];
1566         int eqd, i, num = 0, start;
1567         struct be_aic_obj *aic;
1568         struct be_eq_obj *eqo;
1569         struct be_rx_obj *rxo;
1570         struct be_tx_obj *txo;
1571         u64 rx_pkts, tx_pkts;
1572         ulong now;
1573         u32 pps, delta;
1574
1575         for_all_evt_queues(adapter, eqo, i) {
1576                 aic = &adapter->aic_obj[eqo->idx];
1577                 if (!aic->enable) {
1578                         if (aic->jiffies)
1579                                 aic->jiffies = 0;
1580                         eqd = aic->et_eqd;
1581                         goto modify_eqd;
1582                 }
1583
1584                 rxo = &adapter->rx_obj[eqo->idx];
1585                 do {
1586                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1587                         rx_pkts = rxo->stats.rx_pkts;
1588                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1589
1590                 txo = &adapter->tx_obj[eqo->idx];
1591                 do {
1592                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1593                         tx_pkts = txo->stats.tx_reqs;
1594                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1595
1596                 /* Skip, if wrapped around or first calculation */
1597                 now = jiffies;
1598                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1599                     rx_pkts < aic->rx_pkts_prev ||
1600                     tx_pkts < aic->tx_reqs_prev) {
1601                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1602                         continue;
1603                 }
1604
1605                 delta = jiffies_to_msecs(now - aic->jiffies);
1606                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1607                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1608                 eqd = (pps / 15000) << 2;
1609
1610                 if (eqd < 8)
1611                         eqd = 0;
1612                 eqd = min_t(u32, eqd, aic->max_eqd);
1613                 eqd = max_t(u32, eqd, aic->min_eqd);
1614
1615                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1616 modify_eqd:
1617                 if (eqd != aic->prev_eqd) {
1618                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1619                         set_eqd[num].eq_id = eqo->q.id;
1620                         aic->prev_eqd = eqd;
1621                         num++;
1622                 }
1623         }
1624
1625         if (num)
1626                 be_cmd_modify_eqd(adapter, set_eqd, num);
1627 }
1628
1629 static void be_rx_stats_update(struct be_rx_obj *rxo,
1630                                struct be_rx_compl_info *rxcp)
1631 {
1632         struct be_rx_stats *stats = rx_stats(rxo);
1633
1634         u64_stats_update_begin(&stats->sync);
1635         stats->rx_compl++;
1636         stats->rx_bytes += rxcp->pkt_size;
1637         stats->rx_pkts++;
1638         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1639                 stats->rx_mcast_pkts++;
1640         if (rxcp->err)
1641                 stats->rx_compl_err++;
1642         u64_stats_update_end(&stats->sync);
1643 }
1644
1645 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1646 {
1647         /* L4 checksum is not reliable for non TCP/UDP packets.
1648          * Also ignore ipcksm for ipv6 pkts
1649          */
1650         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1651                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1652 }
1653
1654 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1655 {
1656         struct be_adapter *adapter = rxo->adapter;
1657         struct be_rx_page_info *rx_page_info;
1658         struct be_queue_info *rxq = &rxo->q;
1659         u16 frag_idx = rxq->tail;
1660
1661         rx_page_info = &rxo->page_info_tbl[frag_idx];
1662         BUG_ON(!rx_page_info->page);
1663
1664         if (rx_page_info->last_frag) {
1665                 dma_unmap_page(&adapter->pdev->dev,
1666                                dma_unmap_addr(rx_page_info, bus),
1667                                adapter->big_page_size, DMA_FROM_DEVICE);
1668                 rx_page_info->last_frag = false;
1669         } else {
1670                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1671                                         dma_unmap_addr(rx_page_info, bus),
1672                                         rx_frag_size, DMA_FROM_DEVICE);
1673         }
1674
1675         queue_tail_inc(rxq);
1676         atomic_dec(&rxq->used);
1677         return rx_page_info;
1678 }
1679
1680 /* Throwaway the data in the Rx completion */
1681 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1682                                 struct be_rx_compl_info *rxcp)
1683 {
1684         struct be_rx_page_info *page_info;
1685         u16 i, num_rcvd = rxcp->num_rcvd;
1686
1687         for (i = 0; i < num_rcvd; i++) {
1688                 page_info = get_rx_page_info(rxo);
1689                 put_page(page_info->page);
1690                 memset(page_info, 0, sizeof(*page_info));
1691         }
1692 }
1693
1694 /*
1695  * skb_fill_rx_data forms a complete skb for an ether frame
1696  * indicated by rxcp.
1697  */
1698 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1699                              struct be_rx_compl_info *rxcp)
1700 {
1701         struct be_rx_page_info *page_info;
1702         u16 i, j;
1703         u16 hdr_len, curr_frag_len, remaining;
1704         u8 *start;
1705
1706         page_info = get_rx_page_info(rxo);
1707         start = page_address(page_info->page) + page_info->page_offset;
1708         prefetch(start);
1709
1710         /* Copy data in the first descriptor of this completion */
1711         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1712
1713         skb->len = curr_frag_len;
1714         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1715                 memcpy(skb->data, start, curr_frag_len);
1716                 /* Complete packet has now been moved to data */
1717                 put_page(page_info->page);
1718                 skb->data_len = 0;
1719                 skb->tail += curr_frag_len;
1720         } else {
1721                 hdr_len = ETH_HLEN;
1722                 memcpy(skb->data, start, hdr_len);
1723                 skb_shinfo(skb)->nr_frags = 1;
1724                 skb_frag_set_page(skb, 0, page_info->page);
1725                 skb_shinfo(skb)->frags[0].page_offset =
1726                                         page_info->page_offset + hdr_len;
1727                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1728                                   curr_frag_len - hdr_len);
1729                 skb->data_len = curr_frag_len - hdr_len;
1730                 skb->truesize += rx_frag_size;
1731                 skb->tail += hdr_len;
1732         }
1733         page_info->page = NULL;
1734
1735         if (rxcp->pkt_size <= rx_frag_size) {
1736                 BUG_ON(rxcp->num_rcvd != 1);
1737                 return;
1738         }
1739
1740         /* More frags present for this completion */
1741         remaining = rxcp->pkt_size - curr_frag_len;
1742         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1743                 page_info = get_rx_page_info(rxo);
1744                 curr_frag_len = min(remaining, rx_frag_size);
1745
1746                 /* Coalesce all frags from the same physical page in one slot */
1747                 if (page_info->page_offset == 0) {
1748                         /* Fresh page */
1749                         j++;
1750                         skb_frag_set_page(skb, j, page_info->page);
1751                         skb_shinfo(skb)->frags[j].page_offset =
1752                                                         page_info->page_offset;
1753                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1754                         skb_shinfo(skb)->nr_frags++;
1755                 } else {
1756                         put_page(page_info->page);
1757                 }
1758
1759                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1760                 skb->len += curr_frag_len;
1761                 skb->data_len += curr_frag_len;
1762                 skb->truesize += rx_frag_size;
1763                 remaining -= curr_frag_len;
1764                 page_info->page = NULL;
1765         }
1766         BUG_ON(j > MAX_SKB_FRAGS);
1767 }
1768
1769 /* Process the RX completion indicated by rxcp when GRO is disabled */
1770 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1771                                 struct be_rx_compl_info *rxcp)
1772 {
1773         struct be_adapter *adapter = rxo->adapter;
1774         struct net_device *netdev = adapter->netdev;
1775         struct sk_buff *skb;
1776
1777         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1778         if (unlikely(!skb)) {
1779                 rx_stats(rxo)->rx_drops_no_skbs++;
1780                 be_rx_compl_discard(rxo, rxcp);
1781                 return;
1782         }
1783
1784         skb_fill_rx_data(rxo, skb, rxcp);
1785
1786         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1787                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1788         else
1789                 skb_checksum_none_assert(skb);
1790
1791         skb->protocol = eth_type_trans(skb, netdev);
1792         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1793         if (netdev->features & NETIF_F_RXHASH)
1794                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1795
1796         skb->csum_level = rxcp->tunneled;
1797         skb_mark_napi_id(skb, napi);
1798
1799         if (rxcp->vlanf)
1800                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1801
1802         netif_receive_skb(skb);
1803 }
1804
1805 /* Process the RX completion indicated by rxcp when GRO is enabled */
1806 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1807                                     struct napi_struct *napi,
1808                                     struct be_rx_compl_info *rxcp)
1809 {
1810         struct be_adapter *adapter = rxo->adapter;
1811         struct be_rx_page_info *page_info;
1812         struct sk_buff *skb = NULL;
1813         u16 remaining, curr_frag_len;
1814         u16 i, j;
1815
1816         skb = napi_get_frags(napi);
1817         if (!skb) {
1818                 be_rx_compl_discard(rxo, rxcp);
1819                 return;
1820         }
1821
1822         remaining = rxcp->pkt_size;
1823         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1824                 page_info = get_rx_page_info(rxo);
1825
1826                 curr_frag_len = min(remaining, rx_frag_size);
1827
1828                 /* Coalesce all frags from the same physical page in one slot */
1829                 if (i == 0 || page_info->page_offset == 0) {
1830                         /* First frag or Fresh page */
1831                         j++;
1832                         skb_frag_set_page(skb, j, page_info->page);
1833                         skb_shinfo(skb)->frags[j].page_offset =
1834                                                         page_info->page_offset;
1835                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1836                 } else {
1837                         put_page(page_info->page);
1838                 }
1839                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1840                 skb->truesize += rx_frag_size;
1841                 remaining -= curr_frag_len;
1842                 memset(page_info, 0, sizeof(*page_info));
1843         }
1844         BUG_ON(j > MAX_SKB_FRAGS);
1845
1846         skb_shinfo(skb)->nr_frags = j + 1;
1847         skb->len = rxcp->pkt_size;
1848         skb->data_len = rxcp->pkt_size;
1849         skb->ip_summed = CHECKSUM_UNNECESSARY;
1850         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1851         if (adapter->netdev->features & NETIF_F_RXHASH)
1852                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1853
1854         skb->csum_level = rxcp->tunneled;
1855         skb_mark_napi_id(skb, napi);
1856
1857         if (rxcp->vlanf)
1858                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1859
1860         napi_gro_frags(napi);
1861 }
1862
1863 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1864                                  struct be_rx_compl_info *rxcp)
1865 {
1866         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1867         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1868         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1869         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1870         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1871         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1872         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1873         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1874         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1875         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1876         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1877         if (rxcp->vlanf) {
1878                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1879                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1880         }
1881         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1882         rxcp->tunneled =
1883                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1884 }
1885
1886 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1887                                  struct be_rx_compl_info *rxcp)
1888 {
1889         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1890         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1891         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1892         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1893         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1894         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1895         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1896         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1897         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1898         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1899         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1900         if (rxcp->vlanf) {
1901                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1902                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1903         }
1904         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1905         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1906 }
1907
1908 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1909 {
1910         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1911         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1912         struct be_adapter *adapter = rxo->adapter;
1913
1914         /* For checking the valid bit it is Ok to use either definition as the
1915          * valid bit is at the same position in both v0 and v1 Rx compl */
1916         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1917                 return NULL;
1918
1919         rmb();
1920         be_dws_le_to_cpu(compl, sizeof(*compl));
1921
1922         if (adapter->be3_native)
1923                 be_parse_rx_compl_v1(compl, rxcp);
1924         else
1925                 be_parse_rx_compl_v0(compl, rxcp);
1926
1927         if (rxcp->ip_frag)
1928                 rxcp->l4_csum = 0;
1929
1930         if (rxcp->vlanf) {
1931                 /* In QNQ modes, if qnq bit is not set, then the packet was
1932                  * tagged only with the transparent outer vlan-tag and must
1933                  * not be treated as a vlan packet by host
1934                  */
1935                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1936                         rxcp->vlanf = 0;
1937
1938                 if (!lancer_chip(adapter))
1939                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1940
1941                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1942                     !test_bit(rxcp->vlan_tag, adapter->vids))
1943                         rxcp->vlanf = 0;
1944         }
1945
1946         /* As the compl has been parsed, reset it; we wont touch it again */
1947         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1948
1949         queue_tail_inc(&rxo->cq);
1950         return rxcp;
1951 }
1952
1953 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1954 {
1955         u32 order = get_order(size);
1956
1957         if (order > 0)
1958                 gfp |= __GFP_COMP;
1959         return  alloc_pages(gfp, order);
1960 }
1961
1962 /*
1963  * Allocate a page, split it to fragments of size rx_frag_size and post as
1964  * receive buffers to BE
1965  */
1966 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1967 {
1968         struct be_adapter *adapter = rxo->adapter;
1969         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1970         struct be_queue_info *rxq = &rxo->q;
1971         struct page *pagep = NULL;
1972         struct device *dev = &adapter->pdev->dev;
1973         struct be_eth_rx_d *rxd;
1974         u64 page_dmaaddr = 0, frag_dmaaddr;
1975         u32 posted, page_offset = 0, notify = 0;
1976
1977         page_info = &rxo->page_info_tbl[rxq->head];
1978         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1979                 if (!pagep) {
1980                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1981                         if (unlikely(!pagep)) {
1982                                 rx_stats(rxo)->rx_post_fail++;
1983                                 break;
1984                         }
1985                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1986                                                     adapter->big_page_size,
1987                                                     DMA_FROM_DEVICE);
1988                         if (dma_mapping_error(dev, page_dmaaddr)) {
1989                                 put_page(pagep);
1990                                 pagep = NULL;
1991                                 adapter->drv_stats.dma_map_errors++;
1992                                 break;
1993                         }
1994                         page_offset = 0;
1995                 } else {
1996                         get_page(pagep);
1997                         page_offset += rx_frag_size;
1998                 }
1999                 page_info->page_offset = page_offset;
2000                 page_info->page = pagep;
2001
2002                 rxd = queue_head_node(rxq);
2003                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2004                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2005                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2006
2007                 /* Any space left in the current big page for another frag? */
2008                 if ((page_offset + rx_frag_size + rx_frag_size) >
2009                                         adapter->big_page_size) {
2010                         pagep = NULL;
2011                         page_info->last_frag = true;
2012                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2013                 } else {
2014                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2015                 }
2016
2017                 prev_page_info = page_info;
2018                 queue_head_inc(rxq);
2019                 page_info = &rxo->page_info_tbl[rxq->head];
2020         }
2021
2022         /* Mark the last frag of a page when we break out of the above loop
2023          * with no more slots available in the RXQ
2024          */
2025         if (pagep) {
2026                 prev_page_info->last_frag = true;
2027                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2028         }
2029
2030         if (posted) {
2031                 atomic_add(posted, &rxq->used);
2032                 if (rxo->rx_post_starved)
2033                         rxo->rx_post_starved = false;
2034                 do {
2035                         notify = min(256u, posted);
2036                         be_rxq_notify(adapter, rxq->id, notify);
2037                         posted -= notify;
2038                 } while (posted);
2039         } else if (atomic_read(&rxq->used) == 0) {
2040                 /* Let be_worker replenish when memory is available */
2041                 rxo->rx_post_starved = true;
2042         }
2043 }
2044
2045 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
2046 {
2047         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
2048
2049         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2050                 return NULL;
2051
2052         rmb();
2053         be_dws_le_to_cpu(txcp, sizeof(*txcp));
2054
2055         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2056
2057         queue_tail_inc(tx_cq);
2058         return txcp;
2059 }
2060
2061 static u16 be_tx_compl_process(struct be_adapter *adapter,
2062                                struct be_tx_obj *txo, u16 last_index)
2063 {
2064         struct sk_buff **sent_skbs = txo->sent_skb_list;
2065         struct be_queue_info *txq = &txo->q;
2066         u16 frag_index, num_wrbs = 0;
2067         struct sk_buff *skb = NULL;
2068         bool unmap_skb_hdr = false;
2069         struct be_eth_wrb *wrb;
2070
2071         do {
2072                 if (sent_skbs[txq->tail]) {
2073                         /* Free skb from prev req */
2074                         if (skb)
2075                                 dev_consume_skb_any(skb);
2076                         skb = sent_skbs[txq->tail];
2077                         sent_skbs[txq->tail] = NULL;
2078                         queue_tail_inc(txq);  /* skip hdr wrb */
2079                         num_wrbs++;
2080                         unmap_skb_hdr = true;
2081                 }
2082                 wrb = queue_tail_node(txq);
2083                 frag_index = txq->tail;
2084                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2085                               (unmap_skb_hdr && skb_headlen(skb)));
2086                 unmap_skb_hdr = false;
2087                 queue_tail_inc(txq);
2088                 num_wrbs++;
2089         } while (frag_index != last_index);
2090         dev_consume_skb_any(skb);
2091
2092         return num_wrbs;
2093 }
2094
2095 /* Return the number of events in the event queue */
2096 static inline int events_get(struct be_eq_obj *eqo)
2097 {
2098         struct be_eq_entry *eqe;
2099         int num = 0;
2100
2101         do {
2102                 eqe = queue_tail_node(&eqo->q);
2103                 if (eqe->evt == 0)
2104                         break;
2105
2106                 rmb();
2107                 eqe->evt = 0;
2108                 num++;
2109                 queue_tail_inc(&eqo->q);
2110         } while (true);
2111
2112         return num;
2113 }
2114
2115 /* Leaves the EQ is disarmed state */
2116 static void be_eq_clean(struct be_eq_obj *eqo)
2117 {
2118         int num = events_get(eqo);
2119
2120         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2121 }
2122
2123 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2124 {
2125         struct be_rx_page_info *page_info;
2126         struct be_queue_info *rxq = &rxo->q;
2127         struct be_queue_info *rx_cq = &rxo->cq;
2128         struct be_rx_compl_info *rxcp;
2129         struct be_adapter *adapter = rxo->adapter;
2130         int flush_wait = 0;
2131
2132         /* Consume pending rx completions.
2133          * Wait for the flush completion (identified by zero num_rcvd)
2134          * to arrive. Notify CQ even when there are no more CQ entries
2135          * for HW to flush partially coalesced CQ entries.
2136          * In Lancer, there is no need to wait for flush compl.
2137          */
2138         for (;;) {
2139                 rxcp = be_rx_compl_get(rxo);
2140                 if (!rxcp) {
2141                         if (lancer_chip(adapter))
2142                                 break;
2143
2144                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2145                                 dev_warn(&adapter->pdev->dev,
2146                                          "did not receive flush compl\n");
2147                                 break;
2148                         }
2149                         be_cq_notify(adapter, rx_cq->id, true, 0);
2150                         mdelay(1);
2151                 } else {
2152                         be_rx_compl_discard(rxo, rxcp);
2153                         be_cq_notify(adapter, rx_cq->id, false, 1);
2154                         if (rxcp->num_rcvd == 0)
2155                                 break;
2156                 }
2157         }
2158
2159         /* After cleanup, leave the CQ in unarmed state */
2160         be_cq_notify(adapter, rx_cq->id, false, 0);
2161
2162         /* Then free posted rx buffers that were not used */
2163         while (atomic_read(&rxq->used) > 0) {
2164                 page_info = get_rx_page_info(rxo);
2165                 put_page(page_info->page);
2166                 memset(page_info, 0, sizeof(*page_info));
2167         }
2168         BUG_ON(atomic_read(&rxq->used));
2169         rxq->tail = 0;
2170         rxq->head = 0;
2171 }
2172
2173 static void be_tx_compl_clean(struct be_adapter *adapter)
2174 {
2175         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2176         struct device *dev = &adapter->pdev->dev;
2177         struct be_tx_obj *txo;
2178         struct be_queue_info *txq;
2179         struct be_eth_tx_compl *txcp;
2180         int i, pending_txqs;
2181
2182         /* Stop polling for compls when HW has been silent for 10ms */
2183         do {
2184                 pending_txqs = adapter->num_tx_qs;
2185
2186                 for_all_tx_queues(adapter, txo, i) {
2187                         cmpl = 0;
2188                         num_wrbs = 0;
2189                         txq = &txo->q;
2190                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2191                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2192                                 num_wrbs += be_tx_compl_process(adapter, txo,
2193                                                                 end_idx);
2194                                 cmpl++;
2195                         }
2196                         if (cmpl) {
2197                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2198                                 atomic_sub(num_wrbs, &txq->used);
2199                                 timeo = 0;
2200                         }
2201                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2202                                 pending_txqs--;
2203                 }
2204
2205                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2206                         break;
2207
2208                 mdelay(1);
2209         } while (true);
2210
2211         /* Free enqueued TX that was never notified to HW */
2212         for_all_tx_queues(adapter, txo, i) {
2213                 txq = &txo->q;
2214
2215                 if (atomic_read(&txq->used)) {
2216                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2217                                  i, atomic_read(&txq->used));
2218                         notified_idx = txq->tail;
2219                         end_idx = txq->tail;
2220                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2221                                   txq->len);
2222                         /* Use the tx-compl process logic to handle requests
2223                          * that were not sent to the HW.
2224                          */
2225                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2226                         atomic_sub(num_wrbs, &txq->used);
2227                         BUG_ON(atomic_read(&txq->used));
2228                         txo->pend_wrb_cnt = 0;
2229                         /* Since hw was never notified of these requests,
2230                          * reset TXQ indices
2231                          */
2232                         txq->head = notified_idx;
2233                         txq->tail = notified_idx;
2234                 }
2235         }
2236 }
2237
2238 static void be_evt_queues_destroy(struct be_adapter *adapter)
2239 {
2240         struct be_eq_obj *eqo;
2241         int i;
2242
2243         for_all_evt_queues(adapter, eqo, i) {
2244                 if (eqo->q.created) {
2245                         be_eq_clean(eqo);
2246                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2247                         napi_hash_del(&eqo->napi);
2248                         netif_napi_del(&eqo->napi);
2249                 }
2250                 be_queue_free(adapter, &eqo->q);
2251         }
2252 }
2253
2254 static int be_evt_queues_create(struct be_adapter *adapter)
2255 {
2256         struct be_queue_info *eq;
2257         struct be_eq_obj *eqo;
2258         struct be_aic_obj *aic;
2259         int i, rc;
2260
2261         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2262                                     adapter->cfg_num_qs);
2263
2264         for_all_evt_queues(adapter, eqo, i) {
2265                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2266                                BE_NAPI_WEIGHT);
2267                 napi_hash_add(&eqo->napi);
2268                 aic = &adapter->aic_obj[i];
2269                 eqo->adapter = adapter;
2270                 eqo->idx = i;
2271                 aic->max_eqd = BE_MAX_EQD;
2272                 aic->enable = true;
2273
2274                 eq = &eqo->q;
2275                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2276                                     sizeof(struct be_eq_entry));
2277                 if (rc)
2278                         return rc;
2279
2280                 rc = be_cmd_eq_create(adapter, eqo);
2281                 if (rc)
2282                         return rc;
2283         }
2284         return 0;
2285 }
2286
2287 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2288 {
2289         struct be_queue_info *q;
2290
2291         q = &adapter->mcc_obj.q;
2292         if (q->created)
2293                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2294         be_queue_free(adapter, q);
2295
2296         q = &adapter->mcc_obj.cq;
2297         if (q->created)
2298                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2299         be_queue_free(adapter, q);
2300 }
2301
2302 /* Must be called only after TX qs are created as MCC shares TX EQ */
2303 static int be_mcc_queues_create(struct be_adapter *adapter)
2304 {
2305         struct be_queue_info *q, *cq;
2306
2307         cq = &adapter->mcc_obj.cq;
2308         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2309                            sizeof(struct be_mcc_compl)))
2310                 goto err;
2311
2312         /* Use the default EQ for MCC completions */
2313         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2314                 goto mcc_cq_free;
2315
2316         q = &adapter->mcc_obj.q;
2317         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2318                 goto mcc_cq_destroy;
2319
2320         if (be_cmd_mccq_create(adapter, q, cq))
2321                 goto mcc_q_free;
2322
2323         return 0;
2324
2325 mcc_q_free:
2326         be_queue_free(adapter, q);
2327 mcc_cq_destroy:
2328         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2329 mcc_cq_free:
2330         be_queue_free(adapter, cq);
2331 err:
2332         return -1;
2333 }
2334
2335 static void be_tx_queues_destroy(struct be_adapter *adapter)
2336 {
2337         struct be_queue_info *q;
2338         struct be_tx_obj *txo;
2339         u8 i;
2340
2341         for_all_tx_queues(adapter, txo, i) {
2342                 q = &txo->q;
2343                 if (q->created)
2344                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2345                 be_queue_free(adapter, q);
2346
2347                 q = &txo->cq;
2348                 if (q->created)
2349                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2350                 be_queue_free(adapter, q);
2351         }
2352 }
2353
2354 static int be_tx_qs_create(struct be_adapter *adapter)
2355 {
2356         struct be_queue_info *cq, *eq;
2357         struct be_tx_obj *txo;
2358         int status, i;
2359
2360         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2361
2362         for_all_tx_queues(adapter, txo, i) {
2363                 cq = &txo->cq;
2364                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2365                                         sizeof(struct be_eth_tx_compl));
2366                 if (status)
2367                         return status;
2368
2369                 u64_stats_init(&txo->stats.sync);
2370                 u64_stats_init(&txo->stats.sync_compl);
2371
2372                 /* If num_evt_qs is less than num_tx_qs, then more than
2373                  * one txq share an eq
2374                  */
2375                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2376                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2377                 if (status)
2378                         return status;
2379
2380                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2381                                         sizeof(struct be_eth_wrb));
2382                 if (status)
2383                         return status;
2384
2385                 status = be_cmd_txq_create(adapter, txo);
2386                 if (status)
2387                         return status;
2388         }
2389
2390         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2391                  adapter->num_tx_qs);
2392         return 0;
2393 }
2394
2395 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2396 {
2397         struct be_queue_info *q;
2398         struct be_rx_obj *rxo;
2399         int i;
2400
2401         for_all_rx_queues(adapter, rxo, i) {
2402                 q = &rxo->cq;
2403                 if (q->created)
2404                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2405                 be_queue_free(adapter, q);
2406         }
2407 }
2408
2409 static int be_rx_cqs_create(struct be_adapter *adapter)
2410 {
2411         struct be_queue_info *eq, *cq;
2412         struct be_rx_obj *rxo;
2413         int rc, i;
2414
2415         /* We can create as many RSS rings as there are EQs. */
2416         adapter->num_rx_qs = adapter->num_evt_qs;
2417
2418         /* We'll use RSS only if atleast 2 RSS rings are supported.
2419          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2420          */
2421         if (adapter->num_rx_qs > 1)
2422                 adapter->num_rx_qs++;
2423
2424         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2425         for_all_rx_queues(adapter, rxo, i) {
2426                 rxo->adapter = adapter;
2427                 cq = &rxo->cq;
2428                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2429                                     sizeof(struct be_eth_rx_compl));
2430                 if (rc)
2431                         return rc;
2432
2433                 u64_stats_init(&rxo->stats.sync);
2434                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2435                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2436                 if (rc)
2437                         return rc;
2438         }
2439
2440         dev_info(&adapter->pdev->dev,
2441                  "created %d RSS queue(s) and 1 default RX queue\n",
2442                  adapter->num_rx_qs - 1);
2443         return 0;
2444 }
2445
2446 static irqreturn_t be_intx(int irq, void *dev)
2447 {
2448         struct be_eq_obj *eqo = dev;
2449         struct be_adapter *adapter = eqo->adapter;
2450         int num_evts = 0;
2451
2452         /* IRQ is not expected when NAPI is scheduled as the EQ
2453          * will not be armed.
2454          * But, this can happen on Lancer INTx where it takes
2455          * a while to de-assert INTx or in BE2 where occasionaly
2456          * an interrupt may be raised even when EQ is unarmed.
2457          * If NAPI is already scheduled, then counting & notifying
2458          * events will orphan them.
2459          */
2460         if (napi_schedule_prep(&eqo->napi)) {
2461                 num_evts = events_get(eqo);
2462                 __napi_schedule(&eqo->napi);
2463                 if (num_evts)
2464                         eqo->spurious_intr = 0;
2465         }
2466         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2467
2468         /* Return IRQ_HANDLED only for the the first spurious intr
2469          * after a valid intr to stop the kernel from branding
2470          * this irq as a bad one!
2471          */
2472         if (num_evts || eqo->spurious_intr++ == 0)
2473                 return IRQ_HANDLED;
2474         else
2475                 return IRQ_NONE;
2476 }
2477
2478 static irqreturn_t be_msix(int irq, void *dev)
2479 {
2480         struct be_eq_obj *eqo = dev;
2481
2482         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2483         napi_schedule(&eqo->napi);
2484         return IRQ_HANDLED;
2485 }
2486
2487 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2488 {
2489         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2490 }
2491
2492 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2493                          int budget, int polling)
2494 {
2495         struct be_adapter *adapter = rxo->adapter;
2496         struct be_queue_info *rx_cq = &rxo->cq;
2497         struct be_rx_compl_info *rxcp;
2498         u32 work_done;
2499         u32 frags_consumed = 0;
2500
2501         for (work_done = 0; work_done < budget; work_done++) {
2502                 rxcp = be_rx_compl_get(rxo);
2503                 if (!rxcp)
2504                         break;
2505
2506                 /* Is it a flush compl that has no data */
2507                 if (unlikely(rxcp->num_rcvd == 0))
2508                         goto loop_continue;
2509
2510                 /* Discard compl with partial DMA Lancer B0 */
2511                 if (unlikely(!rxcp->pkt_size)) {
2512                         be_rx_compl_discard(rxo, rxcp);
2513                         goto loop_continue;
2514                 }
2515
2516                 /* On BE drop pkts that arrive due to imperfect filtering in
2517                  * promiscuous mode on some skews
2518                  */
2519                 if (unlikely(rxcp->port != adapter->port_num &&
2520                              !lancer_chip(adapter))) {
2521                         be_rx_compl_discard(rxo, rxcp);
2522                         goto loop_continue;
2523                 }
2524
2525                 /* Don't do gro when we're busy_polling */
2526                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2527                         be_rx_compl_process_gro(rxo, napi, rxcp);
2528                 else
2529                         be_rx_compl_process(rxo, napi, rxcp);
2530
2531 loop_continue:
2532                 frags_consumed += rxcp->num_rcvd;
2533                 be_rx_stats_update(rxo, rxcp);
2534         }
2535
2536         if (work_done) {
2537                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2538
2539                 /* When an rx-obj gets into post_starved state, just
2540                  * let be_worker do the posting.
2541                  */
2542                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2543                     !rxo->rx_post_starved)
2544                         be_post_rx_frags(rxo, GFP_ATOMIC,
2545                                          max_t(u32, MAX_RX_POST,
2546                                                frags_consumed));
2547         }
2548
2549         return work_done;
2550 }
2551
2552 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2553 {
2554         switch (status) {
2555         case BE_TX_COMP_HDR_PARSE_ERR:
2556                 tx_stats(txo)->tx_hdr_parse_err++;
2557                 break;
2558         case BE_TX_COMP_NDMA_ERR:
2559                 tx_stats(txo)->tx_dma_err++;
2560                 break;
2561         case BE_TX_COMP_ACL_ERR:
2562                 tx_stats(txo)->tx_spoof_check_err++;
2563                 break;
2564         }
2565 }
2566
2567 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2568 {
2569         switch (status) {
2570         case LANCER_TX_COMP_LSO_ERR:
2571                 tx_stats(txo)->tx_tso_err++;
2572                 break;
2573         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2574         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2575                 tx_stats(txo)->tx_spoof_check_err++;
2576                 break;
2577         case LANCER_TX_COMP_QINQ_ERR:
2578                 tx_stats(txo)->tx_qinq_err++;
2579                 break;
2580         case LANCER_TX_COMP_PARITY_ERR:
2581                 tx_stats(txo)->tx_internal_parity_err++;
2582                 break;
2583         case LANCER_TX_COMP_DMA_ERR:
2584                 tx_stats(txo)->tx_dma_err++;
2585                 break;
2586         }
2587 }
2588
2589 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2590                           int idx)
2591 {
2592         struct be_eth_tx_compl *txcp;
2593         int num_wrbs = 0, work_done = 0;
2594         u32 compl_status;
2595         u16 last_idx;
2596
2597         while ((txcp = be_tx_compl_get(&txo->cq))) {
2598                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2599                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2600                 work_done++;
2601
2602                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2603                 if (compl_status) {
2604                         if (lancer_chip(adapter))
2605                                 lancer_update_tx_err(txo, compl_status);
2606                         else
2607                                 be_update_tx_err(txo, compl_status);
2608                 }
2609         }
2610
2611         if (work_done) {
2612                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2613                 atomic_sub(num_wrbs, &txo->q.used);
2614
2615                 /* As Tx wrbs have been freed up, wake up netdev queue
2616                  * if it was stopped due to lack of tx wrbs.  */
2617                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2618                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2619                         netif_wake_subqueue(adapter->netdev, idx);
2620                 }
2621
2622                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2623                 tx_stats(txo)->tx_compl += work_done;
2624                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2625         }
2626 }
2627
2628 #ifdef CONFIG_NET_RX_BUSY_POLL
2629 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2630 {
2631         bool status = true;
2632
2633         spin_lock(&eqo->lock); /* BH is already disabled */
2634         if (eqo->state & BE_EQ_LOCKED) {
2635                 WARN_ON(eqo->state & BE_EQ_NAPI);
2636                 eqo->state |= BE_EQ_NAPI_YIELD;
2637                 status = false;
2638         } else {
2639                 eqo->state = BE_EQ_NAPI;
2640         }
2641         spin_unlock(&eqo->lock);
2642         return status;
2643 }
2644
2645 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2646 {
2647         spin_lock(&eqo->lock); /* BH is already disabled */
2648
2649         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2650         eqo->state = BE_EQ_IDLE;
2651
2652         spin_unlock(&eqo->lock);
2653 }
2654
2655 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2656 {
2657         bool status = true;
2658
2659         spin_lock_bh(&eqo->lock);
2660         if (eqo->state & BE_EQ_LOCKED) {
2661                 eqo->state |= BE_EQ_POLL_YIELD;
2662                 status = false;
2663         } else {
2664                 eqo->state |= BE_EQ_POLL;
2665         }
2666         spin_unlock_bh(&eqo->lock);
2667         return status;
2668 }
2669
2670 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2671 {
2672         spin_lock_bh(&eqo->lock);
2673
2674         WARN_ON(eqo->state & (BE_EQ_NAPI));
2675         eqo->state = BE_EQ_IDLE;
2676
2677         spin_unlock_bh(&eqo->lock);
2678 }
2679
2680 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2681 {
2682         spin_lock_init(&eqo->lock);
2683         eqo->state = BE_EQ_IDLE;
2684 }
2685
2686 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2687 {
2688         local_bh_disable();
2689
2690         /* It's enough to just acquire napi lock on the eqo to stop
2691          * be_busy_poll() from processing any queueus.
2692          */
2693         while (!be_lock_napi(eqo))
2694                 mdelay(1);
2695
2696         local_bh_enable();
2697 }
2698
2699 #else /* CONFIG_NET_RX_BUSY_POLL */
2700
2701 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2702 {
2703         return true;
2704 }
2705
2706 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2707 {
2708 }
2709
2710 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2711 {
2712         return false;
2713 }
2714
2715 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2716 {
2717 }
2718
2719 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2720 {
2721 }
2722
2723 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2724 {
2725 }
2726 #endif /* CONFIG_NET_RX_BUSY_POLL */
2727
2728 int be_poll(struct napi_struct *napi, int budget)
2729 {
2730         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2731         struct be_adapter *adapter = eqo->adapter;
2732         int max_work = 0, work, i, num_evts;
2733         struct be_rx_obj *rxo;
2734         struct be_tx_obj *txo;
2735
2736         num_evts = events_get(eqo);
2737
2738         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2739                 be_process_tx(adapter, txo, i);
2740
2741         if (be_lock_napi(eqo)) {
2742                 /* This loop will iterate twice for EQ0 in which
2743                  * completions of the last RXQ (default one) are also processed
2744                  * For other EQs the loop iterates only once
2745                  */
2746                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2747                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2748                         max_work = max(work, max_work);
2749                 }
2750                 be_unlock_napi(eqo);
2751         } else {
2752                 max_work = budget;
2753         }
2754
2755         if (is_mcc_eqo(eqo))
2756                 be_process_mcc(adapter);
2757
2758         if (max_work < budget) {
2759                 napi_complete(napi);
2760                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2761         } else {
2762                 /* As we'll continue in polling mode, count and clear events */
2763                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2764         }
2765         return max_work;
2766 }
2767
2768 #ifdef CONFIG_NET_RX_BUSY_POLL
2769 static int be_busy_poll(struct napi_struct *napi)
2770 {
2771         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2772         struct be_adapter *adapter = eqo->adapter;
2773         struct be_rx_obj *rxo;
2774         int i, work = 0;
2775
2776         if (!be_lock_busy_poll(eqo))
2777                 return LL_FLUSH_BUSY;
2778
2779         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2780                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2781                 if (work)
2782                         break;
2783         }
2784
2785         be_unlock_busy_poll(eqo);
2786         return work;
2787 }
2788 #endif
2789
2790 void be_detect_error(struct be_adapter *adapter)
2791 {
2792         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2793         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2794         u32 i;
2795         bool error_detected = false;
2796         struct device *dev = &adapter->pdev->dev;
2797         struct net_device *netdev = adapter->netdev;
2798
2799         if (be_hw_error(adapter))
2800                 return;
2801
2802         if (lancer_chip(adapter)) {
2803                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2804                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2805                         sliport_err1 = ioread32(adapter->db +
2806                                                 SLIPORT_ERROR1_OFFSET);
2807                         sliport_err2 = ioread32(adapter->db +
2808                                                 SLIPORT_ERROR2_OFFSET);
2809                         adapter->hw_error = true;
2810                         /* Do not log error messages if its a FW reset */
2811                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2812                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2813                                 dev_info(dev, "Firmware update in progress\n");
2814                         } else {
2815                                 error_detected = true;
2816                                 dev_err(dev, "Error detected in the card\n");
2817                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2818                                         sliport_status);
2819                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2820                                         sliport_err1);
2821                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2822                                         sliport_err2);
2823                         }
2824                 }
2825         } else {
2826                 pci_read_config_dword(adapter->pdev,
2827                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2828                 pci_read_config_dword(adapter->pdev,
2829                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2830                 pci_read_config_dword(adapter->pdev,
2831                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2832                 pci_read_config_dword(adapter->pdev,
2833                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2834
2835                 ue_lo = (ue_lo & ~ue_lo_mask);
2836                 ue_hi = (ue_hi & ~ue_hi_mask);
2837
2838                 /* On certain platforms BE hardware can indicate spurious UEs.
2839                  * Allow HW to stop working completely in case of a real UE.
2840                  * Hence not setting the hw_error for UE detection.
2841                  */
2842
2843                 if (ue_lo || ue_hi) {
2844                         error_detected = true;
2845                         dev_err(dev,
2846                                 "Unrecoverable Error detected in the adapter");
2847                         dev_err(dev, "Please reboot server to recover");
2848                         if (skyhawk_chip(adapter))
2849                                 adapter->hw_error = true;
2850                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2851                                 if (ue_lo & 1)
2852                                         dev_err(dev, "UE: %s bit set\n",
2853                                                 ue_status_low_desc[i]);
2854                         }
2855                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2856                                 if (ue_hi & 1)
2857                                         dev_err(dev, "UE: %s bit set\n",
2858                                                 ue_status_hi_desc[i]);
2859                         }
2860                 }
2861         }
2862         if (error_detected)
2863                 netif_carrier_off(netdev);
2864 }
2865
2866 static void be_msix_disable(struct be_adapter *adapter)
2867 {
2868         if (msix_enabled(adapter)) {
2869                 pci_disable_msix(adapter->pdev);
2870                 adapter->num_msix_vec = 0;
2871                 adapter->num_msix_roce_vec = 0;
2872         }
2873 }
2874
2875 static int be_msix_enable(struct be_adapter *adapter)
2876 {
2877         int i, num_vec;
2878         struct device *dev = &adapter->pdev->dev;
2879
2880         /* If RoCE is supported, program the max number of NIC vectors that
2881          * may be configured via set-channels, along with vectors needed for
2882          * RoCe. Else, just program the number we'll use initially.
2883          */
2884         if (be_roce_supported(adapter))
2885                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2886                                 2 * num_online_cpus());
2887         else
2888                 num_vec = adapter->cfg_num_qs;
2889
2890         for (i = 0; i < num_vec; i++)
2891                 adapter->msix_entries[i].entry = i;
2892
2893         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2894                                         MIN_MSIX_VECTORS, num_vec);
2895         if (num_vec < 0)
2896                 goto fail;
2897
2898         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2899                 adapter->num_msix_roce_vec = num_vec / 2;
2900                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2901                          adapter->num_msix_roce_vec);
2902         }
2903
2904         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2905
2906         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2907                  adapter->num_msix_vec);
2908         return 0;
2909
2910 fail:
2911         dev_warn(dev, "MSIx enable failed\n");
2912
2913         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2914         if (!be_physfn(adapter))
2915                 return num_vec;
2916         return 0;
2917 }
2918
2919 static inline int be_msix_vec_get(struct be_adapter *adapter,
2920                                   struct be_eq_obj *eqo)
2921 {
2922         return adapter->msix_entries[eqo->msix_idx].vector;
2923 }
2924
2925 static int be_msix_register(struct be_adapter *adapter)
2926 {
2927         struct net_device *netdev = adapter->netdev;
2928         struct be_eq_obj *eqo;
2929         int status, i, vec;
2930
2931         for_all_evt_queues(adapter, eqo, i) {
2932                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2933                 vec = be_msix_vec_get(adapter, eqo);
2934                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2935                 if (status)
2936                         goto err_msix;
2937         }
2938
2939         return 0;
2940 err_msix:
2941         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2942                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2943         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2944                  status);
2945         be_msix_disable(adapter);
2946         return status;
2947 }
2948
2949 static int be_irq_register(struct be_adapter *adapter)
2950 {
2951         struct net_device *netdev = adapter->netdev;
2952         int status;
2953
2954         if (msix_enabled(adapter)) {
2955                 status = be_msix_register(adapter);
2956                 if (status == 0)
2957                         goto done;
2958                 /* INTx is not supported for VF */
2959                 if (!be_physfn(adapter))
2960                         return status;
2961         }
2962
2963         /* INTx: only the first EQ is used */
2964         netdev->irq = adapter->pdev->irq;
2965         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2966                              &adapter->eq_obj[0]);
2967         if (status) {
2968                 dev_err(&adapter->pdev->dev,
2969                         "INTx request IRQ failed - err %d\n", status);
2970                 return status;
2971         }
2972 done:
2973         adapter->isr_registered = true;
2974         return 0;
2975 }
2976
2977 static void be_irq_unregister(struct be_adapter *adapter)
2978 {
2979         struct net_device *netdev = adapter->netdev;
2980         struct be_eq_obj *eqo;
2981         int i;
2982
2983         if (!adapter->isr_registered)
2984                 return;
2985
2986         /* INTx */
2987         if (!msix_enabled(adapter)) {
2988                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2989                 goto done;
2990         }
2991
2992         /* MSIx */
2993         for_all_evt_queues(adapter, eqo, i)
2994                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2995
2996 done:
2997         adapter->isr_registered = false;
2998 }
2999
3000 static void be_rx_qs_destroy(struct be_adapter *adapter)
3001 {
3002         struct be_queue_info *q;
3003         struct be_rx_obj *rxo;
3004         int i;
3005
3006         for_all_rx_queues(adapter, rxo, i) {
3007                 q = &rxo->q;
3008                 if (q->created) {
3009                         be_cmd_rxq_destroy(adapter, q);
3010                         be_rx_cq_clean(rxo);
3011                 }
3012                 be_queue_free(adapter, q);
3013         }
3014 }
3015
3016 static int be_close(struct net_device *netdev)
3017 {
3018         struct be_adapter *adapter = netdev_priv(netdev);
3019         struct be_eq_obj *eqo;
3020         int i;
3021
3022         /* This protection is needed as be_close() may be called even when the
3023          * adapter is in cleared state (after eeh perm failure)
3024          */
3025         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3026                 return 0;
3027
3028         be_roce_dev_close(adapter);
3029
3030         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3031                 for_all_evt_queues(adapter, eqo, i) {
3032                         napi_disable(&eqo->napi);
3033                         be_disable_busy_poll(eqo);
3034                 }
3035                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3036         }
3037
3038         be_async_mcc_disable(adapter);
3039
3040         /* Wait for all pending tx completions to arrive so that
3041          * all tx skbs are freed.
3042          */
3043         netif_tx_disable(netdev);
3044         be_tx_compl_clean(adapter);
3045
3046         be_rx_qs_destroy(adapter);
3047         be_clear_uc_list(adapter);
3048
3049         for_all_evt_queues(adapter, eqo, i) {
3050                 if (msix_enabled(adapter))
3051                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3052                 else
3053                         synchronize_irq(netdev->irq);
3054                 be_eq_clean(eqo);
3055         }
3056
3057         be_irq_unregister(adapter);
3058
3059         return 0;
3060 }
3061
3062 static int be_rx_qs_create(struct be_adapter *adapter)
3063 {
3064         struct rss_info *rss = &adapter->rss_info;
3065         u8 rss_key[RSS_HASH_KEY_LEN];
3066         struct be_rx_obj *rxo;
3067         int rc, i, j;
3068
3069         for_all_rx_queues(adapter, rxo, i) {
3070                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3071                                     sizeof(struct be_eth_rx_d));
3072                 if (rc)
3073                         return rc;
3074         }
3075
3076         /* The FW would like the default RXQ to be created first */
3077         rxo = default_rxo(adapter);
3078         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
3079                                adapter->if_handle, false, &rxo->rss_id);
3080         if (rc)
3081                 return rc;
3082
3083         for_all_rss_queues(adapter, rxo, i) {
3084                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3085                                        rx_frag_size, adapter->if_handle,
3086                                        true, &rxo->rss_id);
3087                 if (rc)
3088                         return rc;
3089         }
3090
3091         if (be_multi_rxq(adapter)) {
3092                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
3093                         j += adapter->num_rx_qs - 1) {
3094                         for_all_rss_queues(adapter, rxo, i) {
3095                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3096                                         break;
3097                                 rss->rsstable[j + i] = rxo->rss_id;
3098                                 rss->rss_queue[j + i] = i;
3099                         }
3100                 }
3101                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3102                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3103
3104                 if (!BEx_chip(adapter))
3105                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3106                                 RSS_ENABLE_UDP_IPV6;
3107         } else {
3108                 /* Disable RSS, if only default RX Q is created */
3109                 rss->rss_flags = RSS_ENABLE_NONE;
3110         }
3111
3112         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3113         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3114                                128, rss_key);
3115         if (rc) {
3116                 rss->rss_flags = RSS_ENABLE_NONE;
3117                 return rc;
3118         }
3119
3120         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3121
3122         /* First time posting */
3123         for_all_rx_queues(adapter, rxo, i)
3124                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
3125         return 0;
3126 }
3127
3128 static int be_open(struct net_device *netdev)
3129 {
3130         struct be_adapter *adapter = netdev_priv(netdev);
3131         struct be_eq_obj *eqo;
3132         struct be_rx_obj *rxo;
3133         struct be_tx_obj *txo;
3134         u8 link_status;
3135         int status, i;
3136
3137         status = be_rx_qs_create(adapter);
3138         if (status)
3139                 goto err;
3140
3141         status = be_irq_register(adapter);
3142         if (status)
3143                 goto err;
3144
3145         for_all_rx_queues(adapter, rxo, i)
3146                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3147
3148         for_all_tx_queues(adapter, txo, i)
3149                 be_cq_notify(adapter, txo->cq.id, true, 0);
3150
3151         be_async_mcc_enable(adapter);
3152
3153         for_all_evt_queues(adapter, eqo, i) {
3154                 napi_enable(&eqo->napi);
3155                 be_enable_busy_poll(eqo);
3156                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
3157         }
3158         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3159
3160         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3161         if (!status)
3162                 be_link_status_update(adapter, link_status);
3163
3164         netif_tx_start_all_queues(netdev);
3165         be_roce_dev_open(adapter);
3166
3167 #ifdef CONFIG_BE2NET_VXLAN
3168         if (skyhawk_chip(adapter))
3169                 vxlan_get_rx_port(netdev);
3170 #endif
3171
3172         return 0;
3173 err:
3174         be_close(adapter->netdev);
3175         return -EIO;
3176 }
3177
3178 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3179 {
3180         struct be_dma_mem cmd;
3181         int status = 0;
3182         u8 mac[ETH_ALEN];
3183
3184         memset(mac, 0, ETH_ALEN);
3185
3186         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3187         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3188                                      GFP_KERNEL);
3189         if (!cmd.va)
3190                 return -ENOMEM;
3191
3192         if (enable) {
3193                 status = pci_write_config_dword(adapter->pdev,
3194                                                 PCICFG_PM_CONTROL_OFFSET,
3195                                                 PCICFG_PM_CONTROL_MASK);
3196                 if (status) {
3197                         dev_err(&adapter->pdev->dev,
3198                                 "Could not enable Wake-on-lan\n");
3199                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3200                                           cmd.dma);
3201                         return status;
3202                 }
3203                 status = be_cmd_enable_magic_wol(adapter,
3204                                                  adapter->netdev->dev_addr,
3205                                                  &cmd);
3206                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3207                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3208         } else {
3209                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3210                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3211                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3212         }
3213
3214         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3215         return status;
3216 }
3217
3218 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3219 {
3220         u32 addr;
3221
3222         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3223
3224         mac[5] = (u8)(addr & 0xFF);
3225         mac[4] = (u8)((addr >> 8) & 0xFF);
3226         mac[3] = (u8)((addr >> 16) & 0xFF);
3227         /* Use the OUI from the current MAC address */
3228         memcpy(mac, adapter->netdev->dev_addr, 3);
3229 }
3230
3231 /*
3232  * Generate a seed MAC address from the PF MAC Address using jhash.
3233  * MAC Address for VFs are assigned incrementally starting from the seed.
3234  * These addresses are programmed in the ASIC by the PF and the VF driver
3235  * queries for the MAC address during its probe.
3236  */
3237 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3238 {
3239         u32 vf;
3240         int status = 0;
3241         u8 mac[ETH_ALEN];
3242         struct be_vf_cfg *vf_cfg;
3243
3244         be_vf_eth_addr_generate(adapter, mac);
3245
3246         for_all_vfs(adapter, vf_cfg, vf) {
3247                 if (BEx_chip(adapter))
3248                         status = be_cmd_pmac_add(adapter, mac,
3249                                                  vf_cfg->if_handle,
3250                                                  &vf_cfg->pmac_id, vf + 1);
3251                 else
3252                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3253                                                 vf + 1);
3254
3255                 if (status)
3256                         dev_err(&adapter->pdev->dev,
3257                                 "Mac address assignment failed for VF %d\n",
3258                                 vf);
3259                 else
3260                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3261
3262                 mac[5] += 1;
3263         }
3264         return status;
3265 }
3266
3267 static int be_vfs_mac_query(struct be_adapter *adapter)
3268 {
3269         int status, vf;
3270         u8 mac[ETH_ALEN];
3271         struct be_vf_cfg *vf_cfg;
3272
3273         for_all_vfs(adapter, vf_cfg, vf) {
3274                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3275                                                mac, vf_cfg->if_handle,
3276                                                false, vf+1);
3277                 if (status)
3278                         return status;
3279                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3280         }
3281         return 0;
3282 }
3283
3284 static void be_vf_clear(struct be_adapter *adapter)
3285 {
3286         struct be_vf_cfg *vf_cfg;
3287         u32 vf;
3288
3289         if (pci_vfs_assigned(adapter->pdev)) {
3290                 dev_warn(&adapter->pdev->dev,
3291                          "VFs are assigned to VMs: not disabling VFs\n");
3292                 goto done;
3293         }
3294
3295         pci_disable_sriov(adapter->pdev);
3296
3297         for_all_vfs(adapter, vf_cfg, vf) {
3298                 if (BEx_chip(adapter))
3299                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3300                                         vf_cfg->pmac_id, vf + 1);
3301                 else
3302                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3303                                        vf + 1);
3304
3305                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3306         }
3307 done:
3308         kfree(adapter->vf_cfg);
3309         adapter->num_vfs = 0;
3310         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3311 }
3312
3313 static void be_clear_queues(struct be_adapter *adapter)
3314 {
3315         be_mcc_queues_destroy(adapter);
3316         be_rx_cqs_destroy(adapter);
3317         be_tx_queues_destroy(adapter);
3318         be_evt_queues_destroy(adapter);
3319 }
3320
3321 static void be_cancel_worker(struct be_adapter *adapter)
3322 {
3323         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3324                 cancel_delayed_work_sync(&adapter->work);
3325                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3326         }
3327 }
3328
3329 static void be_mac_clear(struct be_adapter *adapter)
3330 {
3331         if (adapter->pmac_id) {
3332                 be_cmd_pmac_del(adapter, adapter->if_handle,
3333                                 adapter->pmac_id[0], 0);
3334                 kfree(adapter->pmac_id);
3335                 adapter->pmac_id = NULL;
3336         }
3337 }
3338
3339 #ifdef CONFIG_BE2NET_VXLAN
3340 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3341 {
3342         struct net_device *netdev = adapter->netdev;
3343
3344         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3345                 be_cmd_manage_iface(adapter, adapter->if_handle,
3346                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3347
3348         if (adapter->vxlan_port)
3349                 be_cmd_set_vxlan_port(adapter, 0);
3350
3351         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3352         adapter->vxlan_port = 0;
3353
3354         netdev->hw_enc_features = 0;
3355         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3356         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3357 }
3358 #endif
3359
3360 static int be_clear(struct be_adapter *adapter)
3361 {
3362         be_cancel_worker(adapter);
3363
3364         if (sriov_enabled(adapter))
3365                 be_vf_clear(adapter);
3366
3367         /* Re-configure FW to distribute resources evenly across max-supported
3368          * number of VFs, only when VFs are not already enabled.
3369          */
3370         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3371                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3372                                         pci_sriov_get_totalvfs(adapter->pdev));
3373
3374 #ifdef CONFIG_BE2NET_VXLAN
3375         be_disable_vxlan_offloads(adapter);
3376 #endif
3377         /* delete the primary mac along with the uc-mac list */
3378         be_mac_clear(adapter);
3379
3380         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3381
3382         be_clear_queues(adapter);
3383
3384         be_msix_disable(adapter);
3385         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3386         return 0;
3387 }
3388
3389 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3390                         u32 cap_flags, u32 vf)
3391 {
3392         u32 en_flags;
3393
3394         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3395                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3396                    BE_IF_FLAGS_RSS;
3397
3398         en_flags &= cap_flags;
3399
3400         return be_cmd_if_create(adapter, cap_flags, en_flags, if_handle, vf);
3401 }
3402
3403 static int be_vfs_if_create(struct be_adapter *adapter)
3404 {
3405         struct be_resources res = {0};
3406         struct be_vf_cfg *vf_cfg;
3407         u32 cap_flags, vf;
3408         int status;
3409
3410         /* If a FW profile exists, then cap_flags are updated */
3411         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3412                     BE_IF_FLAGS_MULTICAST;
3413
3414         for_all_vfs(adapter, vf_cfg, vf) {
3415                 if (!BE3_chip(adapter)) {
3416                         status = be_cmd_get_profile_config(adapter, &res,
3417                                                            vf + 1);
3418                         if (!status) {
3419                                 cap_flags = res.if_cap_flags;
3420                                 /* Prevent VFs from enabling VLAN promiscuous
3421                                  * mode
3422                                  */
3423                                 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
3424                         }
3425                 }
3426
3427                 status = be_if_create(adapter, &vf_cfg->if_handle,
3428                                       cap_flags, vf + 1);
3429                 if (status)
3430                         return status;
3431         }
3432
3433         return 0;
3434 }
3435
3436 static int be_vf_setup_init(struct be_adapter *adapter)
3437 {
3438         struct be_vf_cfg *vf_cfg;
3439         int vf;
3440
3441         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3442                                   GFP_KERNEL);
3443         if (!adapter->vf_cfg)
3444                 return -ENOMEM;
3445
3446         for_all_vfs(adapter, vf_cfg, vf) {
3447                 vf_cfg->if_handle = -1;
3448                 vf_cfg->pmac_id = -1;
3449         }
3450         return 0;
3451 }
3452
3453 static int be_vf_setup(struct be_adapter *adapter)
3454 {
3455         struct device *dev = &adapter->pdev->dev;
3456         struct be_vf_cfg *vf_cfg;
3457         int status, old_vfs, vf;
3458
3459         old_vfs = pci_num_vf(adapter->pdev);
3460
3461         status = be_vf_setup_init(adapter);
3462         if (status)
3463                 goto err;
3464
3465         if (old_vfs) {
3466                 for_all_vfs(adapter, vf_cfg, vf) {
3467                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3468                         if (status)
3469                                 goto err;
3470                 }
3471
3472                 status = be_vfs_mac_query(adapter);
3473                 if (status)
3474                         goto err;
3475         } else {
3476                 status = be_vfs_if_create(adapter);
3477                 if (status)
3478                         goto err;
3479
3480                 status = be_vf_eth_addr_config(adapter);
3481                 if (status)
3482                         goto err;
3483         }
3484
3485         for_all_vfs(adapter, vf_cfg, vf) {
3486                 /* Allow VFs to programs MAC/VLAN filters */
3487                 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
3488                                                   vf + 1);
3489                 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
3490                         status = be_cmd_set_fn_privileges(adapter,
3491                                                           vf_cfg->privileges |
3492                                                           BE_PRIV_FILTMGMT,
3493                                                           vf + 1);
3494                         if (!status) {
3495                                 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
3496                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3497                                          vf);
3498                         }
3499                 }
3500
3501                 /* Allow full available bandwidth */
3502                 if (!old_vfs)
3503                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3504
3505                 if (!old_vfs) {
3506                         be_cmd_enable_vf(adapter, vf + 1);
3507                         be_cmd_set_logical_link_config(adapter,
3508                                                        IFLA_VF_LINK_STATE_AUTO,
3509                                                        vf+1);
3510                 }
3511         }
3512
3513         if (!old_vfs) {
3514                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3515                 if (status) {
3516                         dev_err(dev, "SRIOV enable failed\n");
3517                         adapter->num_vfs = 0;
3518                         goto err;
3519                 }
3520         }
3521
3522         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3523         return 0;
3524 err:
3525         dev_err(dev, "VF setup failed\n");
3526         be_vf_clear(adapter);
3527         return status;
3528 }
3529
3530 /* Converting function_mode bits on BE3 to SH mc_type enums */
3531
3532 static u8 be_convert_mc_type(u32 function_mode)
3533 {
3534         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3535                 return vNIC1;
3536         else if (function_mode & QNQ_MODE)
3537                 return FLEX10;
3538         else if (function_mode & VNIC_MODE)
3539                 return vNIC2;
3540         else if (function_mode & UMC_ENABLED)
3541                 return UMC;
3542         else
3543                 return MC_NONE;
3544 }
3545
3546 /* On BE2/BE3 FW does not suggest the supported limits */
3547 static void BEx_get_resources(struct be_adapter *adapter,
3548                               struct be_resources *res)
3549 {
3550         bool use_sriov = adapter->num_vfs ? 1 : 0;
3551
3552         if (be_physfn(adapter))
3553                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3554         else
3555                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3556
3557         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3558
3559         if (be_is_mc(adapter)) {
3560                 /* Assuming that there are 4 channels per port,
3561                  * when multi-channel is enabled
3562                  */
3563                 if (be_is_qnq_mode(adapter))
3564                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3565                 else
3566                         /* In a non-qnq multichannel mode, the pvid
3567                          * takes up one vlan entry
3568                          */
3569                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3570         } else {
3571                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3572         }
3573
3574         res->max_mcast_mac = BE_MAX_MC;
3575
3576         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3577          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3578          *    *only* if it is RSS-capable.
3579          */
3580         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3581             !be_physfn(adapter) || (be_is_mc(adapter) &&
3582             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3583                 res->max_tx_qs = 1;
3584         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3585                 struct be_resources super_nic_res = {0};
3586
3587                 /* On a SuperNIC profile, the driver needs to use the
3588                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3589                  */
3590                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3591                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3592                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3593         } else {
3594                 res->max_tx_qs = BE3_MAX_TX_QS;
3595         }
3596
3597         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3598             !use_sriov && be_physfn(adapter))
3599                 res->max_rss_qs = (adapter->be3_native) ?
3600                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3601         res->max_rx_qs = res->max_rss_qs + 1;
3602
3603         if (be_physfn(adapter))
3604                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3605                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3606         else
3607                 res->max_evt_qs = 1;
3608
3609         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3610         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3611                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3612 }
3613
3614 static void be_setup_init(struct be_adapter *adapter)
3615 {
3616         adapter->vlan_prio_bmap = 0xff;
3617         adapter->phy.link_speed = -1;
3618         adapter->if_handle = -1;
3619         adapter->be3_native = false;
3620         adapter->if_flags = 0;
3621         if (be_physfn(adapter))
3622                 adapter->cmd_privileges = MAX_PRIVILEGES;
3623         else
3624                 adapter->cmd_privileges = MIN_PRIVILEGES;
3625 }
3626
3627 static int be_get_sriov_config(struct be_adapter *adapter)
3628 {
3629         struct device *dev = &adapter->pdev->dev;
3630         struct be_resources res = {0};
3631         int max_vfs, old_vfs;
3632
3633         /* Some old versions of BE3 FW don't report max_vfs value */
3634         be_cmd_get_profile_config(adapter, &res, 0);
3635
3636         if (BE3_chip(adapter) && !res.max_vfs) {
3637                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3638                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3639         }
3640
3641         adapter->pool_res = res;
3642
3643         if (!be_max_vfs(adapter)) {
3644                 if (num_vfs)
3645                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3646                 adapter->num_vfs = 0;
3647                 return 0;
3648         }
3649
3650         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3651
3652         /* validate num_vfs module param */
3653         old_vfs = pci_num_vf(adapter->pdev);
3654         if (old_vfs) {
3655                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3656                 if (old_vfs != num_vfs)
3657                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3658                 adapter->num_vfs = old_vfs;
3659         } else {
3660                 if (num_vfs > be_max_vfs(adapter)) {
3661                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3662                                  num_vfs);
3663                         dev_info(dev, "Limiting to %d VFs\n",
3664                                  be_max_vfs(adapter));
3665                 }
3666                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3667         }
3668
3669         return 0;
3670 }
3671
3672 static int be_get_resources(struct be_adapter *adapter)
3673 {
3674         struct device *dev = &adapter->pdev->dev;
3675         struct be_resources res = {0};
3676         int status;
3677
3678         if (BEx_chip(adapter)) {
3679                 BEx_get_resources(adapter, &res);
3680                 adapter->res = res;
3681         }
3682
3683         /* For Lancer, SH etc read per-function resource limits from FW.
3684          * GET_FUNC_CONFIG returns per function guaranteed limits.
3685          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3686          */
3687         if (!BEx_chip(adapter)) {
3688                 status = be_cmd_get_func_config(adapter, &res);
3689                 if (status)
3690                         return status;
3691
3692                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3693                 if (be_roce_supported(adapter))
3694                         res.max_evt_qs /= 2;
3695                 adapter->res = res;
3696         }
3697
3698         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3699                  be_max_txqs(adapter), be_max_rxqs(adapter),
3700                  be_max_rss(adapter), be_max_eqs(adapter),
3701                  be_max_vfs(adapter));
3702         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3703                  be_max_uc(adapter), be_max_mc(adapter),
3704                  be_max_vlans(adapter));
3705
3706         return 0;
3707 }
3708
3709 static void be_sriov_config(struct be_adapter *adapter)
3710 {
3711         struct device *dev = &adapter->pdev->dev;
3712         int status;
3713
3714         status = be_get_sriov_config(adapter);
3715         if (status) {
3716                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3717                 dev_err(dev, "SR-IOV cannot be enabled\n");
3718                 return;
3719         }
3720
3721         /* When the HW is in SRIOV capable configuration, the PF-pool
3722          * resources are equally distributed across the max-number of
3723          * VFs. The user may request only a subset of the max-vfs to be
3724          * enabled. Based on num_vfs, redistribute the resources across
3725          * num_vfs so that each VF will have access to more number of
3726          * resources. This facility is not available in BE3 FW.
3727          * Also, this is done by FW in Lancer chip.
3728          */
3729         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3730                 status = be_cmd_set_sriov_config(adapter,
3731                                                  adapter->pool_res,
3732                                                  adapter->num_vfs);
3733                 if (status)
3734                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3735         }
3736 }
3737
3738 static int be_get_config(struct be_adapter *adapter)
3739 {
3740         u16 profile_id;
3741         int status;
3742
3743         status = be_cmd_query_fw_cfg(adapter);
3744         if (status)
3745                 return status;
3746
3747         be_cmd_query_port_name(adapter);
3748
3749         if (be_physfn(adapter)) {
3750                 status = be_cmd_get_active_profile(adapter, &profile_id);
3751                 if (!status)
3752                         dev_info(&adapter->pdev->dev,
3753                                  "Using profile 0x%x\n", profile_id);
3754         }
3755
3756         if (!BE2_chip(adapter) && be_physfn(adapter))
3757                 be_sriov_config(adapter);
3758
3759         status = be_get_resources(adapter);
3760         if (status)
3761                 return status;
3762
3763         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3764                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3765         if (!adapter->pmac_id)
3766                 return -ENOMEM;
3767
3768         /* Sanitize cfg_num_qs based on HW and platform limits */
3769         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3770
3771         return 0;
3772 }
3773
3774 static int be_mac_setup(struct be_adapter *adapter)
3775 {
3776         u8 mac[ETH_ALEN];
3777         int status;
3778
3779         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3780                 status = be_cmd_get_perm_mac(adapter, mac);
3781                 if (status)
3782                         return status;
3783
3784                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3785                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3786         } else {
3787                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3788                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3789         }
3790
3791         /* For BE3-R VFs, the PF programs the initial MAC address */
3792         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3793                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3794                                 &adapter->pmac_id[0], 0);
3795         return 0;
3796 }
3797
3798 static void be_schedule_worker(struct be_adapter *adapter)
3799 {
3800         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3801         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3802 }
3803
3804 static int be_setup_queues(struct be_adapter *adapter)
3805 {
3806         struct net_device *netdev = adapter->netdev;
3807         int status;
3808
3809         status = be_evt_queues_create(adapter);
3810         if (status)
3811                 goto err;
3812
3813         status = be_tx_qs_create(adapter);
3814         if (status)
3815                 goto err;
3816
3817         status = be_rx_cqs_create(adapter);
3818         if (status)
3819                 goto err;
3820
3821         status = be_mcc_queues_create(adapter);
3822         if (status)
3823                 goto err;
3824
3825         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3826         if (status)
3827                 goto err;
3828
3829         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3830         if (status)
3831                 goto err;
3832
3833         return 0;
3834 err:
3835         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3836         return status;
3837 }
3838
3839 int be_update_queues(struct be_adapter *adapter)
3840 {
3841         struct net_device *netdev = adapter->netdev;
3842         int status;
3843
3844         if (netif_running(netdev))
3845                 be_close(netdev);
3846
3847         be_cancel_worker(adapter);
3848
3849         /* If any vectors have been shared with RoCE we cannot re-program
3850          * the MSIx table.
3851          */
3852         if (!adapter->num_msix_roce_vec)
3853                 be_msix_disable(adapter);
3854
3855         be_clear_queues(adapter);
3856
3857         if (!msix_enabled(adapter)) {
3858                 status = be_msix_enable(adapter);
3859                 if (status)
3860                         return status;
3861         }
3862
3863         status = be_setup_queues(adapter);
3864         if (status)
3865                 return status;
3866
3867         be_schedule_worker(adapter);
3868
3869         if (netif_running(netdev))
3870                 status = be_open(netdev);
3871
3872         return status;
3873 }
3874
3875 static inline int fw_major_num(const char *fw_ver)
3876 {
3877         int fw_major = 0, i;
3878
3879         i = sscanf(fw_ver, "%d.", &fw_major);
3880         if (i != 1)
3881                 return 0;
3882
3883         return fw_major;
3884 }
3885
3886 static int be_setup(struct be_adapter *adapter)
3887 {
3888         struct device *dev = &adapter->pdev->dev;
3889         int status;
3890
3891         be_setup_init(adapter);
3892
3893         if (!lancer_chip(adapter))
3894                 be_cmd_req_native_mode(adapter);
3895
3896         status = be_get_config(adapter);
3897         if (status)
3898                 goto err;
3899
3900         status = be_msix_enable(adapter);
3901         if (status)
3902                 goto err;
3903
3904         status = be_if_create(adapter, &adapter->if_handle,
3905                               be_if_cap_flags(adapter), 0);
3906         if (status)
3907                 goto err;
3908
3909         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3910         rtnl_lock();
3911         status = be_setup_queues(adapter);
3912         rtnl_unlock();
3913         if (status)
3914                 goto err;
3915
3916         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3917
3918         status = be_mac_setup(adapter);
3919         if (status)
3920                 goto err;
3921
3922         be_cmd_get_fw_ver(adapter);
3923         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3924
3925         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3926                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3927                         adapter->fw_ver);
3928                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3929         }
3930
3931         if (adapter->vlans_added)
3932                 be_vid_config(adapter);
3933
3934         be_set_rx_mode(adapter->netdev);
3935
3936         be_cmd_get_acpi_wol_cap(adapter);
3937
3938         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3939                                          adapter->rx_fc);
3940         if (status)
3941                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3942                                         &adapter->rx_fc);
3943
3944         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3945                  adapter->tx_fc, adapter->rx_fc);
3946
3947         if (be_physfn(adapter))
3948                 be_cmd_set_logical_link_config(adapter,
3949                                                IFLA_VF_LINK_STATE_AUTO, 0);
3950
3951         if (adapter->num_vfs)
3952                 be_vf_setup(adapter);
3953
3954         status = be_cmd_get_phy_info(adapter);
3955         if (!status && be_pause_supported(adapter))
3956                 adapter->phy.fc_autoneg = 1;
3957
3958         be_schedule_worker(adapter);
3959         adapter->flags |= BE_FLAGS_SETUP_DONE;
3960         return 0;
3961 err:
3962         be_clear(adapter);
3963         return status;
3964 }
3965
3966 #ifdef CONFIG_NET_POLL_CONTROLLER
3967 static void be_netpoll(struct net_device *netdev)
3968 {
3969         struct be_adapter *adapter = netdev_priv(netdev);
3970         struct be_eq_obj *eqo;
3971         int i;
3972
3973         for_all_evt_queues(adapter, eqo, i) {
3974                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3975                 napi_schedule(&eqo->napi);
3976         }
3977 }
3978 #endif
3979
3980 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3981
3982 static bool phy_flashing_required(struct be_adapter *adapter)
3983 {
3984         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
3985                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3986 }
3987
3988 static bool is_comp_in_ufi(struct be_adapter *adapter,
3989                            struct flash_section_info *fsec, int type)
3990 {
3991         int i = 0, img_type = 0;
3992         struct flash_section_info_g2 *fsec_g2 = NULL;
3993
3994         if (BE2_chip(adapter))
3995                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3996
3997         for (i = 0; i < MAX_FLASH_COMP; i++) {
3998                 if (fsec_g2)
3999                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
4000                 else
4001                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
4002
4003                 if (img_type == type)
4004                         return true;
4005         }
4006         return false;
4007
4008 }
4009
4010 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
4011                                                 int header_size,
4012                                                 const struct firmware *fw)
4013 {
4014         struct flash_section_info *fsec = NULL;
4015         const u8 *p = fw->data;
4016
4017         p += header_size;
4018         while (p < (fw->data + fw->size)) {
4019                 fsec = (struct flash_section_info *)p;
4020                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
4021                         return fsec;
4022                 p += 32;
4023         }
4024         return NULL;
4025 }
4026
4027 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
4028                               u32 img_offset, u32 img_size, int hdr_size,
4029                               u16 img_optype, bool *crc_match)
4030 {
4031         u32 crc_offset;
4032         int status;
4033         u8 crc[4];
4034
4035         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
4036                                       img_size - 4);
4037         if (status)
4038                 return status;
4039
4040         crc_offset = hdr_size + img_offset + img_size - 4;
4041
4042         /* Skip flashing, if crc of flashed region matches */
4043         if (!memcmp(crc, p + crc_offset, 4))
4044                 *crc_match = true;
4045         else
4046                 *crc_match = false;
4047
4048         return status;
4049 }
4050
4051 static int be_flash(struct be_adapter *adapter, const u8 *img,
4052                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4053                     u32 img_offset)
4054 {
4055         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4056         struct be_cmd_write_flashrom *req = flash_cmd->va;
4057         int status;
4058
4059         while (total_bytes) {
4060                 num_bytes = min_t(u32, 32*1024, total_bytes);
4061
4062                 total_bytes -= num_bytes;
4063
4064                 if (!total_bytes) {
4065                         if (optype == OPTYPE_PHY_FW)
4066                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4067                         else
4068                                 flash_op = FLASHROM_OPER_FLASH;
4069                 } else {
4070                         if (optype == OPTYPE_PHY_FW)
4071                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4072                         else
4073                                 flash_op = FLASHROM_OPER_SAVE;
4074                 }
4075
4076                 memcpy(req->data_buf, img, num_bytes);
4077                 img += num_bytes;
4078                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4079                                                flash_op, img_offset +
4080                                                bytes_sent, num_bytes);
4081                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4082                     optype == OPTYPE_PHY_FW)
4083                         break;
4084                 else if (status)
4085                         return status;
4086
4087                 bytes_sent += num_bytes;
4088         }
4089         return 0;
4090 }
4091
4092 /* For BE2, BE3 and BE3-R */
4093 static int be_flash_BEx(struct be_adapter *adapter,
4094                         const struct firmware *fw,
4095                         struct be_dma_mem *flash_cmd, int num_of_images)
4096 {
4097         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4098         struct device *dev = &adapter->pdev->dev;
4099         struct flash_section_info *fsec = NULL;
4100         int status, i, filehdr_size, num_comp;
4101         const struct flash_comp *pflashcomp;
4102         bool crc_match;
4103         const u8 *p;
4104
4105         struct flash_comp gen3_flash_types[] = {
4106                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4107                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4108                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4109                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4110                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4111                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4112                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4113                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4114                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4115                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4116                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4117                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4118                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4119                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4120                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4121                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4122                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4123                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4124                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4125                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4126         };
4127
4128         struct flash_comp gen2_flash_types[] = {
4129                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4130                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4131                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4132                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4133                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4134                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4135                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4136                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4137                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4138                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4139                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4140                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4141                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4142                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4143                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4144                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4145         };
4146
4147         if (BE3_chip(adapter)) {
4148                 pflashcomp = gen3_flash_types;
4149                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4150                 num_comp = ARRAY_SIZE(gen3_flash_types);
4151         } else {
4152                 pflashcomp = gen2_flash_types;
4153                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4154                 num_comp = ARRAY_SIZE(gen2_flash_types);
4155                 img_hdrs_size = 0;
4156         }
4157
4158         /* Get flash section info*/
4159         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4160         if (!fsec) {
4161                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4162                 return -1;
4163         }
4164         for (i = 0; i < num_comp; i++) {
4165                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4166                         continue;
4167
4168                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4169                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4170                         continue;
4171
4172                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4173                     !phy_flashing_required(adapter))
4174                                 continue;
4175
4176                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4177                         status = be_check_flash_crc(adapter, fw->data,
4178                                                     pflashcomp[i].offset,
4179                                                     pflashcomp[i].size,
4180                                                     filehdr_size +
4181                                                     img_hdrs_size,
4182                                                     OPTYPE_REDBOOT, &crc_match);
4183                         if (status) {
4184                                 dev_err(dev,
4185                                         "Could not get CRC for 0x%x region\n",
4186                                         pflashcomp[i].optype);
4187                                 continue;
4188                         }
4189
4190                         if (crc_match)
4191                                 continue;
4192                 }
4193
4194                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4195                         img_hdrs_size;
4196                 if (p + pflashcomp[i].size > fw->data + fw->size)
4197                         return -1;
4198
4199                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4200                                   pflashcomp[i].size, 0);
4201                 if (status) {
4202                         dev_err(dev, "Flashing section type 0x%x failed\n",
4203                                 pflashcomp[i].img_type);
4204                         return status;
4205                 }
4206         }
4207         return 0;
4208 }
4209
4210 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4211 {
4212         u32 img_type = le32_to_cpu(fsec_entry.type);
4213         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4214
4215         if (img_optype != 0xFFFF)
4216                 return img_optype;
4217
4218         switch (img_type) {
4219         case IMAGE_FIRMWARE_iSCSI:
4220                 img_optype = OPTYPE_ISCSI_ACTIVE;
4221                 break;
4222         case IMAGE_BOOT_CODE:
4223                 img_optype = OPTYPE_REDBOOT;
4224                 break;
4225         case IMAGE_OPTION_ROM_ISCSI:
4226                 img_optype = OPTYPE_BIOS;
4227                 break;
4228         case IMAGE_OPTION_ROM_PXE:
4229                 img_optype = OPTYPE_PXE_BIOS;
4230                 break;
4231         case IMAGE_OPTION_ROM_FCoE:
4232                 img_optype = OPTYPE_FCOE_BIOS;
4233                 break;
4234         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4235                 img_optype = OPTYPE_ISCSI_BACKUP;
4236                 break;
4237         case IMAGE_NCSI:
4238                 img_optype = OPTYPE_NCSI_FW;
4239                 break;
4240         case IMAGE_FLASHISM_JUMPVECTOR:
4241                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4242                 break;
4243         case IMAGE_FIRMWARE_PHY:
4244                 img_optype = OPTYPE_SH_PHY_FW;
4245                 break;
4246         case IMAGE_REDBOOT_DIR:
4247                 img_optype = OPTYPE_REDBOOT_DIR;
4248                 break;
4249         case IMAGE_REDBOOT_CONFIG:
4250                 img_optype = OPTYPE_REDBOOT_CONFIG;
4251                 break;
4252         case IMAGE_UFI_DIR:
4253                 img_optype = OPTYPE_UFI_DIR;
4254                 break;
4255         default:
4256                 break;
4257         }
4258
4259         return img_optype;
4260 }
4261
4262 static int be_flash_skyhawk(struct be_adapter *adapter,
4263                             const struct firmware *fw,
4264                             struct be_dma_mem *flash_cmd, int num_of_images)
4265 {
4266         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4267         bool crc_match, old_fw_img, flash_offset_support = true;
4268         struct device *dev = &adapter->pdev->dev;
4269         struct flash_section_info *fsec = NULL;
4270         u32 img_offset, img_size, img_type;
4271         u16 img_optype, flash_optype;
4272         int status, i, filehdr_size;
4273         const u8 *p;
4274
4275         filehdr_size = sizeof(struct flash_file_hdr_g3);
4276         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4277         if (!fsec) {
4278                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4279                 return -EINVAL;
4280         }
4281
4282 retry_flash:
4283         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4284                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4285                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4286                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4287                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4288                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4289
4290                 if (img_optype == 0xFFFF)
4291                         continue;
4292
4293                 if (flash_offset_support)
4294                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4295                 else
4296                         flash_optype = img_optype;
4297
4298                 /* Don't bother verifying CRC if an old FW image is being
4299                  * flashed
4300                  */
4301                 if (old_fw_img)
4302                         goto flash;
4303
4304                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4305                                             img_size, filehdr_size +
4306                                             img_hdrs_size, flash_optype,
4307                                             &crc_match);
4308                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4309                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4310                         /* The current FW image on the card does not support
4311                          * OFFSET based flashing. Retry using older mechanism
4312                          * of OPTYPE based flashing
4313                          */
4314                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4315                                 flash_offset_support = false;
4316                                 goto retry_flash;
4317                         }
4318
4319                         /* The current FW image on the card does not recognize
4320                          * the new FLASH op_type. The FW download is partially
4321                          * complete. Reboot the server now to enable FW image
4322                          * to recognize the new FLASH op_type. To complete the
4323                          * remaining process, download the same FW again after
4324                          * the reboot.
4325                          */
4326                         dev_err(dev, "Flash incomplete. Reset the server\n");
4327                         dev_err(dev, "Download FW image again after reset\n");
4328                         return -EAGAIN;
4329                 } else if (status) {
4330                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4331                                 img_optype);
4332                         return -EFAULT;
4333                 }
4334
4335                 if (crc_match)
4336                         continue;
4337
4338 flash:
4339                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4340                 if (p + img_size > fw->data + fw->size)
4341                         return -1;
4342
4343                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4344                                   img_offset);
4345
4346                 /* The current FW image on the card does not support OFFSET
4347                  * based flashing. Retry using older mechanism of OPTYPE based
4348                  * flashing
4349                  */
4350                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4351                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4352                         flash_offset_support = false;
4353                         goto retry_flash;
4354                 }
4355
4356                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4357                  * UFI_DIR region
4358                  */
4359                 if (old_fw_img &&
4360                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4361                      (img_optype == OPTYPE_UFI_DIR &&
4362                       base_status(status) == MCC_STATUS_FAILED))) {
4363                         continue;
4364                 } else if (status) {
4365                         dev_err(dev, "Flashing section type 0x%x failed\n",
4366                                 img_type);
4367                         return -EFAULT;
4368                 }
4369         }
4370         return 0;
4371 }
4372
4373 static int lancer_fw_download(struct be_adapter *adapter,
4374                               const struct firmware *fw)
4375 {
4376 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4377 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4378         struct device *dev = &adapter->pdev->dev;
4379         struct be_dma_mem flash_cmd;
4380         const u8 *data_ptr = NULL;
4381         u8 *dest_image_ptr = NULL;
4382         size_t image_size = 0;
4383         u32 chunk_size = 0;
4384         u32 data_written = 0;
4385         u32 offset = 0;
4386         int status = 0;
4387         u8 add_status = 0;
4388         u8 change_status;
4389
4390         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4391                 dev_err(dev, "FW image size should be multiple of 4\n");
4392                 return -EINVAL;
4393         }
4394
4395         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4396                                 + LANCER_FW_DOWNLOAD_CHUNK;
4397         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4398                                           &flash_cmd.dma, GFP_KERNEL);
4399         if (!flash_cmd.va)
4400                 return -ENOMEM;
4401
4402         dest_image_ptr = flash_cmd.va +
4403                                 sizeof(struct lancer_cmd_req_write_object);
4404         image_size = fw->size;
4405         data_ptr = fw->data;
4406
4407         while (image_size) {
4408                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4409
4410                 /* Copy the image chunk content. */
4411                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4412
4413                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4414                                                  chunk_size, offset,
4415                                                  LANCER_FW_DOWNLOAD_LOCATION,
4416                                                  &data_written, &change_status,
4417                                                  &add_status);
4418                 if (status)
4419                         break;
4420
4421                 offset += data_written;
4422                 data_ptr += data_written;
4423                 image_size -= data_written;
4424         }
4425
4426         if (!status) {
4427                 /* Commit the FW written */
4428                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4429                                                  0, offset,
4430                                                  LANCER_FW_DOWNLOAD_LOCATION,
4431                                                  &data_written, &change_status,
4432                                                  &add_status);
4433         }
4434
4435         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4436         if (status) {
4437                 dev_err(dev, "Firmware load error\n");
4438                 return be_cmd_status(status);
4439         }
4440
4441         dev_info(dev, "Firmware flashed successfully\n");
4442
4443         if (change_status == LANCER_FW_RESET_NEEDED) {
4444                 dev_info(dev, "Resetting adapter to activate new FW\n");
4445                 status = lancer_physdev_ctrl(adapter,
4446                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4447                 if (status) {
4448                         dev_err(dev, "Adapter busy, could not reset FW\n");
4449                         dev_err(dev, "Reboot server to activate new FW\n");
4450                 }
4451         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4452                 dev_info(dev, "Reboot server to activate new FW\n");
4453         }
4454
4455         return 0;
4456 }
4457
4458 #define BE2_UFI         2
4459 #define BE3_UFI         3
4460 #define BE3R_UFI        10
4461 #define SH_UFI          4
4462 #define SH_P2_UFI       11
4463
4464 static int be_get_ufi_type(struct be_adapter *adapter,
4465                            struct flash_file_hdr_g3 *fhdr)
4466 {
4467         if (!fhdr) {
4468                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4469                 return -1;
4470         }
4471
4472         /* First letter of the build version is used to identify
4473          * which chip this image file is meant for.
4474          */
4475         switch (fhdr->build[0]) {
4476         case BLD_STR_UFI_TYPE_SH:
4477                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4478                                                                 SH_UFI;
4479         case BLD_STR_UFI_TYPE_BE3:
4480                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4481                                                                 BE3_UFI;
4482         case BLD_STR_UFI_TYPE_BE2:
4483                 return BE2_UFI;
4484         default:
4485                 return -1;
4486         }
4487 }
4488
4489 /* Check if the flash image file is compatible with the adapter that
4490  * is being flashed.
4491  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4492  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4493  */
4494 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4495                                        struct flash_file_hdr_g3 *fhdr)
4496 {
4497         int ufi_type = be_get_ufi_type(adapter, fhdr);
4498
4499         switch (ufi_type) {
4500         case SH_P2_UFI:
4501                 return skyhawk_chip(adapter);
4502         case SH_UFI:
4503                 return (skyhawk_chip(adapter) &&
4504                         adapter->asic_rev < ASIC_REV_P2);
4505         case BE3R_UFI:
4506                 return BE3_chip(adapter);
4507         case BE3_UFI:
4508                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4509         case BE2_UFI:
4510                 return BE2_chip(adapter);
4511         default:
4512                 return false;
4513         }
4514 }
4515
4516 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4517 {
4518         struct device *dev = &adapter->pdev->dev;
4519         struct flash_file_hdr_g3 *fhdr3;
4520         struct image_hdr *img_hdr_ptr;
4521         int status = 0, i, num_imgs;
4522         struct be_dma_mem flash_cmd;
4523
4524         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4525         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4526                 dev_err(dev, "Flash image is not compatible with adapter\n");
4527                 return -EINVAL;
4528         }
4529
4530         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4531         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4532                                           GFP_KERNEL);
4533         if (!flash_cmd.va)
4534                 return -ENOMEM;
4535
4536         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4537         for (i = 0; i < num_imgs; i++) {
4538                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4539                                 (sizeof(struct flash_file_hdr_g3) +
4540                                  i * sizeof(struct image_hdr)));
4541                 if (!BE2_chip(adapter) &&
4542                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4543                         continue;
4544
4545                 if (skyhawk_chip(adapter))
4546                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4547                                                   num_imgs);
4548                 else
4549                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4550                                               num_imgs);
4551         }
4552
4553         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4554         if (!status)
4555                 dev_info(dev, "Firmware flashed successfully\n");
4556
4557         return status;
4558 }
4559
4560 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4561 {
4562         const struct firmware *fw;
4563         int status;
4564
4565         if (!netif_running(adapter->netdev)) {
4566                 dev_err(&adapter->pdev->dev,
4567                         "Firmware load not allowed (interface is down)\n");
4568                 return -ENETDOWN;
4569         }
4570
4571         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4572         if (status)
4573                 goto fw_exit;
4574
4575         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4576
4577         if (lancer_chip(adapter))
4578                 status = lancer_fw_download(adapter, fw);
4579         else
4580                 status = be_fw_download(adapter, fw);
4581
4582         if (!status)
4583                 be_cmd_get_fw_ver(adapter);
4584
4585 fw_exit:
4586         release_firmware(fw);
4587         return status;
4588 }
4589
4590 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4591                                  u16 flags)
4592 {
4593         struct be_adapter *adapter = netdev_priv(dev);
4594         struct nlattr *attr, *br_spec;
4595         int rem;
4596         int status = 0;
4597         u16 mode = 0;
4598
4599         if (!sriov_enabled(adapter))
4600                 return -EOPNOTSUPP;
4601
4602         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4603         if (!br_spec)
4604                 return -EINVAL;
4605
4606         nla_for_each_nested(attr, br_spec, rem) {
4607                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4608                         continue;
4609
4610                 if (nla_len(attr) < sizeof(mode))
4611                         return -EINVAL;
4612
4613                 mode = nla_get_u16(attr);
4614                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4615                         return -EINVAL;
4616
4617                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4618                                                adapter->if_handle,
4619                                                mode == BRIDGE_MODE_VEPA ?
4620                                                PORT_FWD_TYPE_VEPA :
4621                                                PORT_FWD_TYPE_VEB);
4622                 if (status)
4623                         goto err;
4624
4625                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4626                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4627
4628                 return status;
4629         }
4630 err:
4631         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4632                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4633
4634         return status;
4635 }
4636
4637 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4638                                  struct net_device *dev, u32 filter_mask)
4639 {
4640         struct be_adapter *adapter = netdev_priv(dev);
4641         int status = 0;
4642         u8 hsw_mode;
4643
4644         if (!sriov_enabled(adapter))
4645                 return 0;
4646
4647         /* BE and Lancer chips support VEB mode only */
4648         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4649                 hsw_mode = PORT_FWD_TYPE_VEB;
4650         } else {
4651                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4652                                                adapter->if_handle, &hsw_mode);
4653                 if (status)
4654                         return 0;
4655         }
4656
4657         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4658                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4659                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4660                                        0, 0);
4661 }
4662
4663 #ifdef CONFIG_BE2NET_VXLAN
4664 /* VxLAN offload Notes:
4665  *
4666  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4667  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4668  * is expected to work across all types of IP tunnels once exported. Skyhawk
4669  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4670  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4671  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4672  * those other tunnels are unexported on the fly through ndo_features_check().
4673  *
4674  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4675  * adds more than one port, disable offloads and don't re-enable them again
4676  * until after all the tunnels are removed.
4677  */
4678 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4679                               __be16 port)
4680 {
4681         struct be_adapter *adapter = netdev_priv(netdev);
4682         struct device *dev = &adapter->pdev->dev;
4683         int status;
4684
4685         if (lancer_chip(adapter) || BEx_chip(adapter))
4686                 return;
4687
4688         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4689                 dev_info(dev,
4690                          "Only one UDP port supported for VxLAN offloads\n");
4691                 dev_info(dev, "Disabling VxLAN offloads\n");
4692                 adapter->vxlan_port_count++;
4693                 goto err;
4694         }
4695
4696         if (adapter->vxlan_port_count++ >= 1)
4697                 return;
4698
4699         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4700                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4701         if (status) {
4702                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4703                 goto err;
4704         }
4705
4706         status = be_cmd_set_vxlan_port(adapter, port);
4707         if (status) {
4708                 dev_warn(dev, "Failed to add VxLAN port\n");
4709                 goto err;
4710         }
4711         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4712         adapter->vxlan_port = port;
4713
4714         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4715                                    NETIF_F_TSO | NETIF_F_TSO6 |
4716                                    NETIF_F_GSO_UDP_TUNNEL;
4717         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4718         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4719
4720         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4721                  be16_to_cpu(port));
4722         return;
4723 err:
4724         be_disable_vxlan_offloads(adapter);
4725 }
4726
4727 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4728                               __be16 port)
4729 {
4730         struct be_adapter *adapter = netdev_priv(netdev);
4731
4732         if (lancer_chip(adapter) || BEx_chip(adapter))
4733                 return;
4734
4735         if (adapter->vxlan_port != port)
4736                 goto done;
4737
4738         be_disable_vxlan_offloads(adapter);
4739
4740         dev_info(&adapter->pdev->dev,
4741                  "Disabled VxLAN offloads for UDP port %d\n",
4742                  be16_to_cpu(port));
4743 done:
4744         adapter->vxlan_port_count--;
4745 }
4746
4747 static netdev_features_t be_features_check(struct sk_buff *skb,
4748                                            struct net_device *dev,
4749                                            netdev_features_t features)
4750 {
4751         struct be_adapter *adapter = netdev_priv(dev);
4752         u8 l4_hdr = 0;
4753
4754         /* The code below restricts offload features for some tunneled packets.
4755          * Offload features for normal (non tunnel) packets are unchanged.
4756          */
4757         if (!skb->encapsulation ||
4758             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4759                 return features;
4760
4761         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4762          * should disable tunnel offload features if it's not a VxLAN packet,
4763          * as tunnel offloads have been enabled only for VxLAN. This is done to
4764          * allow other tunneled traffic like GRE work fine while VxLAN
4765          * offloads are configured in Skyhawk-R.
4766          */
4767         switch (vlan_get_protocol(skb)) {
4768         case htons(ETH_P_IP):
4769                 l4_hdr = ip_hdr(skb)->protocol;
4770                 break;
4771         case htons(ETH_P_IPV6):
4772                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4773                 break;
4774         default:
4775                 return features;
4776         }
4777
4778         if (l4_hdr != IPPROTO_UDP ||
4779             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4780             skb->inner_protocol != htons(ETH_P_TEB) ||
4781             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4782             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4783                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4784
4785         return features;
4786 }
4787 #endif
4788
4789 static const struct net_device_ops be_netdev_ops = {
4790         .ndo_open               = be_open,
4791         .ndo_stop               = be_close,
4792         .ndo_start_xmit         = be_xmit,
4793         .ndo_set_rx_mode        = be_set_rx_mode,
4794         .ndo_set_mac_address    = be_mac_addr_set,
4795         .ndo_change_mtu         = be_change_mtu,
4796         .ndo_get_stats64        = be_get_stats64,
4797         .ndo_validate_addr      = eth_validate_addr,
4798         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4799         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4800         .ndo_set_vf_mac         = be_set_vf_mac,
4801         .ndo_set_vf_vlan        = be_set_vf_vlan,
4802         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4803         .ndo_get_vf_config      = be_get_vf_config,
4804         .ndo_set_vf_link_state  = be_set_vf_link_state,
4805 #ifdef CONFIG_NET_POLL_CONTROLLER
4806         .ndo_poll_controller    = be_netpoll,
4807 #endif
4808         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4809         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4810 #ifdef CONFIG_NET_RX_BUSY_POLL
4811         .ndo_busy_poll          = be_busy_poll,
4812 #endif
4813 #ifdef CONFIG_BE2NET_VXLAN
4814         .ndo_add_vxlan_port     = be_add_vxlan_port,
4815         .ndo_del_vxlan_port     = be_del_vxlan_port,
4816         .ndo_features_check     = be_features_check,
4817 #endif
4818 };
4819
4820 static void be_netdev_init(struct net_device *netdev)
4821 {
4822         struct be_adapter *adapter = netdev_priv(netdev);
4823
4824         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4825                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4826                 NETIF_F_HW_VLAN_CTAG_TX;
4827         if (be_multi_rxq(adapter))
4828                 netdev->hw_features |= NETIF_F_RXHASH;
4829
4830         netdev->features |= netdev->hw_features |
4831                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4832
4833         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4834                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4835
4836         netdev->priv_flags |= IFF_UNICAST_FLT;
4837
4838         netdev->flags |= IFF_MULTICAST;
4839
4840         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4841
4842         netdev->netdev_ops = &be_netdev_ops;
4843
4844         netdev->ethtool_ops = &be_ethtool_ops;
4845 }
4846
4847 static void be_unmap_pci_bars(struct be_adapter *adapter)
4848 {
4849         if (adapter->csr)
4850                 pci_iounmap(adapter->pdev, adapter->csr);
4851         if (adapter->db)
4852                 pci_iounmap(adapter->pdev, adapter->db);
4853 }
4854
4855 static int db_bar(struct be_adapter *adapter)
4856 {
4857         if (lancer_chip(adapter) || !be_physfn(adapter))
4858                 return 0;
4859         else
4860                 return 4;
4861 }
4862
4863 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4864 {
4865         if (skyhawk_chip(adapter)) {
4866                 adapter->roce_db.size = 4096;
4867                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4868                                                               db_bar(adapter));
4869                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4870                                                                db_bar(adapter));
4871         }
4872         return 0;
4873 }
4874
4875 static int be_map_pci_bars(struct be_adapter *adapter)
4876 {
4877         u8 __iomem *addr;
4878
4879         if (BEx_chip(adapter) && be_physfn(adapter)) {
4880                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4881                 if (!adapter->csr)
4882                         return -ENOMEM;
4883         }
4884
4885         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4886         if (!addr)
4887                 goto pci_map_err;
4888         adapter->db = addr;
4889
4890         be_roce_map_pci_bars(adapter);
4891         return 0;
4892
4893 pci_map_err:
4894         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4895         be_unmap_pci_bars(adapter);
4896         return -ENOMEM;
4897 }
4898
4899 static void be_ctrl_cleanup(struct be_adapter *adapter)
4900 {
4901         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4902
4903         be_unmap_pci_bars(adapter);
4904
4905         if (mem->va)
4906                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4907                                   mem->dma);
4908
4909         mem = &adapter->rx_filter;
4910         if (mem->va)
4911                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4912                                   mem->dma);
4913 }
4914
4915 static int be_ctrl_init(struct be_adapter *adapter)
4916 {
4917         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4918         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4919         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4920         u32 sli_intf;
4921         int status;
4922
4923         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4924         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4925                                  SLI_INTF_FAMILY_SHIFT;
4926         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4927
4928         status = be_map_pci_bars(adapter);
4929         if (status)
4930                 goto done;
4931
4932         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4933         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4934                                                 mbox_mem_alloc->size,
4935                                                 &mbox_mem_alloc->dma,
4936                                                 GFP_KERNEL);
4937         if (!mbox_mem_alloc->va) {
4938                 status = -ENOMEM;
4939                 goto unmap_pci_bars;
4940         }
4941         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4942         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4943         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4944         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4945
4946         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4947         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4948                                             rx_filter->size, &rx_filter->dma,
4949                                             GFP_KERNEL);
4950         if (!rx_filter->va) {
4951                 status = -ENOMEM;
4952                 goto free_mbox;
4953         }
4954
4955         mutex_init(&adapter->mbox_lock);
4956         spin_lock_init(&adapter->mcc_lock);
4957         spin_lock_init(&adapter->mcc_cq_lock);
4958
4959         init_completion(&adapter->et_cmd_compl);
4960         pci_save_state(adapter->pdev);
4961         return 0;
4962
4963 free_mbox:
4964         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4965                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4966
4967 unmap_pci_bars:
4968         be_unmap_pci_bars(adapter);
4969
4970 done:
4971         return status;
4972 }
4973
4974 static void be_stats_cleanup(struct be_adapter *adapter)
4975 {
4976         struct be_dma_mem *cmd = &adapter->stats_cmd;
4977
4978         if (cmd->va)
4979                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4980                                   cmd->va, cmd->dma);
4981 }
4982
4983 static int be_stats_init(struct be_adapter *adapter)
4984 {
4985         struct be_dma_mem *cmd = &adapter->stats_cmd;
4986
4987         if (lancer_chip(adapter))
4988                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4989         else if (BE2_chip(adapter))
4990                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4991         else if (BE3_chip(adapter))
4992                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4993         else
4994                 /* ALL non-BE ASICs */
4995                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4996
4997         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4998                                       GFP_KERNEL);
4999         if (!cmd->va)
5000                 return -ENOMEM;
5001         return 0;
5002 }
5003
5004 static void be_remove(struct pci_dev *pdev)
5005 {
5006         struct be_adapter *adapter = pci_get_drvdata(pdev);
5007
5008         if (!adapter)
5009                 return;
5010
5011         be_roce_dev_remove(adapter);
5012         be_intr_set(adapter, false);
5013
5014         cancel_delayed_work_sync(&adapter->func_recovery_work);
5015
5016         unregister_netdev(adapter->netdev);
5017
5018         be_clear(adapter);
5019
5020         /* tell fw we're done with firing cmds */
5021         be_cmd_fw_clean(adapter);
5022
5023         be_stats_cleanup(adapter);
5024
5025         be_ctrl_cleanup(adapter);
5026
5027         pci_disable_pcie_error_reporting(pdev);
5028
5029         pci_release_regions(pdev);
5030         pci_disable_device(pdev);
5031
5032         free_netdev(adapter->netdev);
5033 }
5034
5035 static int be_get_initial_config(struct be_adapter *adapter)
5036 {
5037         int status, level;
5038
5039         status = be_cmd_get_cntl_attributes(adapter);
5040         if (status)
5041                 return status;
5042
5043         /* Must be a power of 2 or else MODULO will BUG_ON */
5044         adapter->be_get_temp_freq = 64;
5045
5046         if (BEx_chip(adapter)) {
5047                 level = be_cmd_get_fw_log_level(adapter);
5048                 adapter->msg_enable =
5049                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
5050         }
5051
5052         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
5053         return 0;
5054 }
5055
5056 static int lancer_recover_func(struct be_adapter *adapter)
5057 {
5058         struct device *dev = &adapter->pdev->dev;
5059         int status;
5060
5061         status = lancer_test_and_set_rdy_state(adapter);
5062         if (status)
5063                 goto err;
5064
5065         if (netif_running(adapter->netdev))
5066                 be_close(adapter->netdev);
5067
5068         be_clear(adapter);
5069
5070         be_clear_all_error(adapter);
5071
5072         status = be_setup(adapter);
5073         if (status)
5074                 goto err;
5075
5076         if (netif_running(adapter->netdev)) {
5077                 status = be_open(adapter->netdev);
5078                 if (status)
5079                         goto err;
5080         }
5081
5082         dev_err(dev, "Adapter recovery successful\n");
5083         return 0;
5084 err:
5085         if (status == -EAGAIN)
5086                 dev_err(dev, "Waiting for resource provisioning\n");
5087         else
5088                 dev_err(dev, "Adapter recovery failed\n");
5089
5090         return status;
5091 }
5092
5093 static void be_func_recovery_task(struct work_struct *work)
5094 {
5095         struct be_adapter *adapter =
5096                 container_of(work, struct be_adapter,  func_recovery_work.work);
5097         int status = 0;
5098
5099         be_detect_error(adapter);
5100
5101         if (adapter->hw_error && lancer_chip(adapter)) {
5102                 rtnl_lock();
5103                 netif_device_detach(adapter->netdev);
5104                 rtnl_unlock();
5105
5106                 status = lancer_recover_func(adapter);
5107                 if (!status)
5108                         netif_device_attach(adapter->netdev);
5109         }
5110
5111         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
5112          * no need to attempt further recovery.
5113          */
5114         if (!status || status == -EAGAIN)
5115                 schedule_delayed_work(&adapter->func_recovery_work,
5116                                       msecs_to_jiffies(1000));
5117 }
5118
5119 static void be_log_sfp_info(struct be_adapter *adapter)
5120 {
5121         int status;
5122
5123         status = be_cmd_query_sfp_info(adapter);
5124         if (!status) {
5125                 dev_err(&adapter->pdev->dev,
5126                         "Unqualified SFP+ detected on %c from %s part no: %s",
5127                         adapter->port_name, adapter->phy.vendor_name,
5128                         adapter->phy.vendor_pn);
5129         }
5130         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5131 }
5132
5133 static void be_worker(struct work_struct *work)
5134 {
5135         struct be_adapter *adapter =
5136                 container_of(work, struct be_adapter, work.work);
5137         struct be_rx_obj *rxo;
5138         int i;
5139
5140         /* when interrupts are not yet enabled, just reap any pending
5141         * mcc completions */
5142         if (!netif_running(adapter->netdev)) {
5143                 local_bh_disable();
5144                 be_process_mcc(adapter);
5145                 local_bh_enable();
5146                 goto reschedule;
5147         }
5148
5149         if (!adapter->stats_cmd_sent) {
5150                 if (lancer_chip(adapter))
5151                         lancer_cmd_get_pport_stats(adapter,
5152                                                    &adapter->stats_cmd);
5153                 else
5154                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5155         }
5156
5157         if (be_physfn(adapter) &&
5158             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5159                 be_cmd_get_die_temperature(adapter);
5160
5161         for_all_rx_queues(adapter, rxo, i) {
5162                 /* Replenish RX-queues starved due to memory
5163                  * allocation failures.
5164                  */
5165                 if (rxo->rx_post_starved)
5166                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5167         }
5168
5169         be_eqd_update(adapter);
5170
5171         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5172                 be_log_sfp_info(adapter);
5173
5174 reschedule:
5175         adapter->work_counter++;
5176         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5177 }
5178
5179 /* If any VFs are already enabled don't FLR the PF */
5180 static bool be_reset_required(struct be_adapter *adapter)
5181 {
5182         return pci_num_vf(adapter->pdev) ? false : true;
5183 }
5184
5185 static char *mc_name(struct be_adapter *adapter)
5186 {
5187         char *str = ""; /* default */
5188
5189         switch (adapter->mc_type) {
5190         case UMC:
5191                 str = "UMC";
5192                 break;
5193         case FLEX10:
5194                 str = "FLEX10";
5195                 break;
5196         case vNIC1:
5197                 str = "vNIC-1";
5198                 break;
5199         case nPAR:
5200                 str = "nPAR";
5201                 break;
5202         case UFP:
5203                 str = "UFP";
5204                 break;
5205         case vNIC2:
5206                 str = "vNIC-2";
5207                 break;
5208         default:
5209                 str = "";
5210         }
5211
5212         return str;
5213 }
5214
5215 static inline char *func_name(struct be_adapter *adapter)
5216 {
5217         return be_physfn(adapter) ? "PF" : "VF";
5218 }
5219
5220 static inline char *nic_name(struct pci_dev *pdev)
5221 {
5222         switch (pdev->device) {
5223         case OC_DEVICE_ID1:
5224                 return OC_NAME;
5225         case OC_DEVICE_ID2:
5226                 return OC_NAME_BE;
5227         case OC_DEVICE_ID3:
5228         case OC_DEVICE_ID4:
5229                 return OC_NAME_LANCER;
5230         case BE_DEVICE_ID2:
5231                 return BE3_NAME;
5232         case OC_DEVICE_ID5:
5233         case OC_DEVICE_ID6:
5234                 return OC_NAME_SH;
5235         default:
5236                 return BE_NAME;
5237         }
5238 }
5239
5240 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5241 {
5242         struct be_adapter *adapter;
5243         struct net_device *netdev;
5244         int status = 0;
5245
5246         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5247
5248         status = pci_enable_device(pdev);
5249         if (status)
5250                 goto do_none;
5251
5252         status = pci_request_regions(pdev, DRV_NAME);
5253         if (status)
5254                 goto disable_dev;
5255         pci_set_master(pdev);
5256
5257         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5258         if (!netdev) {
5259                 status = -ENOMEM;
5260                 goto rel_reg;
5261         }
5262         adapter = netdev_priv(netdev);
5263         adapter->pdev = pdev;
5264         pci_set_drvdata(pdev, adapter);
5265         adapter->netdev = netdev;
5266         SET_NETDEV_DEV(netdev, &pdev->dev);
5267
5268         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5269         if (!status) {
5270                 netdev->features |= NETIF_F_HIGHDMA;
5271         } else {
5272                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5273                 if (status) {
5274                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5275                         goto free_netdev;
5276                 }
5277         }
5278
5279         status = pci_enable_pcie_error_reporting(pdev);
5280         if (!status)
5281                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5282
5283         status = be_ctrl_init(adapter);
5284         if (status)
5285                 goto free_netdev;
5286
5287         /* sync up with fw's ready state */
5288         if (be_physfn(adapter)) {
5289                 status = be_fw_wait_ready(adapter);
5290                 if (status)
5291                         goto ctrl_clean;
5292         }
5293
5294         if (be_reset_required(adapter)) {
5295                 status = be_cmd_reset_function(adapter);
5296                 if (status)
5297                         goto ctrl_clean;
5298
5299                 /* Wait for interrupts to quiesce after an FLR */
5300                 msleep(100);
5301         }
5302
5303         /* Allow interrupts for other ULPs running on NIC function */
5304         be_intr_set(adapter, true);
5305
5306         /* tell fw we're ready to fire cmds */
5307         status = be_cmd_fw_init(adapter);
5308         if (status)
5309                 goto ctrl_clean;
5310
5311         status = be_stats_init(adapter);
5312         if (status)
5313                 goto ctrl_clean;
5314
5315         status = be_get_initial_config(adapter);
5316         if (status)
5317                 goto stats_clean;
5318
5319         INIT_DELAYED_WORK(&adapter->work, be_worker);
5320         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5321         adapter->rx_fc = true;
5322         adapter->tx_fc = true;
5323
5324         status = be_setup(adapter);
5325         if (status)
5326                 goto stats_clean;
5327
5328         be_netdev_init(netdev);
5329         status = register_netdev(netdev);
5330         if (status != 0)
5331                 goto unsetup;
5332
5333         be_roce_dev_add(adapter);
5334
5335         schedule_delayed_work(&adapter->func_recovery_work,
5336                               msecs_to_jiffies(1000));
5337
5338         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5339                  func_name(adapter), mc_name(adapter), adapter->port_name);
5340
5341         return 0;
5342
5343 unsetup:
5344         be_clear(adapter);
5345 stats_clean:
5346         be_stats_cleanup(adapter);
5347 ctrl_clean:
5348         be_ctrl_cleanup(adapter);
5349 free_netdev:
5350         free_netdev(netdev);
5351 rel_reg:
5352         pci_release_regions(pdev);
5353 disable_dev:
5354         pci_disable_device(pdev);
5355 do_none:
5356         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5357         return status;
5358 }
5359
5360 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5361 {
5362         struct be_adapter *adapter = pci_get_drvdata(pdev);
5363         struct net_device *netdev =  adapter->netdev;
5364
5365         if (adapter->wol_en)
5366                 be_setup_wol(adapter, true);
5367
5368         be_intr_set(adapter, false);
5369         cancel_delayed_work_sync(&adapter->func_recovery_work);
5370
5371         netif_device_detach(netdev);
5372         if (netif_running(netdev)) {
5373                 rtnl_lock();
5374                 be_close(netdev);
5375                 rtnl_unlock();
5376         }
5377         be_clear(adapter);
5378
5379         pci_save_state(pdev);
5380         pci_disable_device(pdev);
5381         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5382         return 0;
5383 }
5384
5385 static int be_resume(struct pci_dev *pdev)
5386 {
5387         int status = 0;
5388         struct be_adapter *adapter = pci_get_drvdata(pdev);
5389         struct net_device *netdev =  adapter->netdev;
5390
5391         netif_device_detach(netdev);
5392
5393         status = pci_enable_device(pdev);
5394         if (status)
5395                 return status;
5396
5397         pci_set_power_state(pdev, PCI_D0);
5398         pci_restore_state(pdev);
5399
5400         status = be_fw_wait_ready(adapter);
5401         if (status)
5402                 return status;
5403
5404         status = be_cmd_reset_function(adapter);
5405         if (status)
5406                 return status;
5407
5408         be_intr_set(adapter, true);
5409         /* tell fw we're ready to fire cmds */
5410         status = be_cmd_fw_init(adapter);
5411         if (status)
5412                 return status;
5413
5414         be_setup(adapter);
5415         if (netif_running(netdev)) {
5416                 rtnl_lock();
5417                 be_open(netdev);
5418                 rtnl_unlock();
5419         }
5420
5421         schedule_delayed_work(&adapter->func_recovery_work,
5422                               msecs_to_jiffies(1000));
5423         netif_device_attach(netdev);
5424
5425         if (adapter->wol_en)
5426                 be_setup_wol(adapter, false);
5427
5428         return 0;
5429 }
5430
5431 /*
5432  * An FLR will stop BE from DMAing any data.
5433  */
5434 static void be_shutdown(struct pci_dev *pdev)
5435 {
5436         struct be_adapter *adapter = pci_get_drvdata(pdev);
5437
5438         if (!adapter)
5439                 return;
5440
5441         be_roce_dev_shutdown(adapter);
5442         cancel_delayed_work_sync(&adapter->work);
5443         cancel_delayed_work_sync(&adapter->func_recovery_work);
5444
5445         netif_device_detach(adapter->netdev);
5446
5447         be_cmd_reset_function(adapter);
5448
5449         pci_disable_device(pdev);
5450 }
5451
5452 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5453                                             pci_channel_state_t state)
5454 {
5455         struct be_adapter *adapter = pci_get_drvdata(pdev);
5456         struct net_device *netdev =  adapter->netdev;
5457
5458         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5459
5460         if (!adapter->eeh_error) {
5461                 adapter->eeh_error = true;
5462
5463                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5464
5465                 rtnl_lock();
5466                 netif_device_detach(netdev);
5467                 if (netif_running(netdev))
5468                         be_close(netdev);
5469                 rtnl_unlock();
5470
5471                 be_clear(adapter);
5472         }
5473
5474         if (state == pci_channel_io_perm_failure)
5475                 return PCI_ERS_RESULT_DISCONNECT;
5476
5477         pci_disable_device(pdev);
5478
5479         /* The error could cause the FW to trigger a flash debug dump.
5480          * Resetting the card while flash dump is in progress
5481          * can cause it not to recover; wait for it to finish.
5482          * Wait only for first function as it is needed only once per
5483          * adapter.
5484          */
5485         if (pdev->devfn == 0)
5486                 ssleep(30);
5487
5488         return PCI_ERS_RESULT_NEED_RESET;
5489 }
5490
5491 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5492 {
5493         struct be_adapter *adapter = pci_get_drvdata(pdev);
5494         int status;
5495
5496         dev_info(&adapter->pdev->dev, "EEH reset\n");
5497
5498         status = pci_enable_device(pdev);
5499         if (status)
5500                 return PCI_ERS_RESULT_DISCONNECT;
5501
5502         pci_set_master(pdev);
5503         pci_set_power_state(pdev, PCI_D0);
5504         pci_restore_state(pdev);
5505
5506         /* Check if card is ok and fw is ready */
5507         dev_info(&adapter->pdev->dev,
5508                  "Waiting for FW to be ready after EEH reset\n");
5509         status = be_fw_wait_ready(adapter);
5510         if (status)
5511                 return PCI_ERS_RESULT_DISCONNECT;
5512
5513         pci_cleanup_aer_uncorrect_error_status(pdev);
5514         be_clear_all_error(adapter);
5515         return PCI_ERS_RESULT_RECOVERED;
5516 }
5517
5518 static void be_eeh_resume(struct pci_dev *pdev)
5519 {
5520         int status = 0;
5521         struct be_adapter *adapter = pci_get_drvdata(pdev);
5522         struct net_device *netdev =  adapter->netdev;
5523
5524         dev_info(&adapter->pdev->dev, "EEH resume\n");
5525
5526         pci_save_state(pdev);
5527
5528         status = be_cmd_reset_function(adapter);
5529         if (status)
5530                 goto err;
5531
5532         /* On some BE3 FW versions, after a HW reset,
5533          * interrupts will remain disabled for each function.
5534          * So, explicitly enable interrupts
5535          */
5536         be_intr_set(adapter, true);
5537
5538         /* tell fw we're ready to fire cmds */
5539         status = be_cmd_fw_init(adapter);
5540         if (status)
5541                 goto err;
5542
5543         status = be_setup(adapter);
5544         if (status)
5545                 goto err;
5546
5547         if (netif_running(netdev)) {
5548                 status = be_open(netdev);
5549                 if (status)
5550                         goto err;
5551         }
5552
5553         schedule_delayed_work(&adapter->func_recovery_work,
5554                               msecs_to_jiffies(1000));
5555         netif_device_attach(netdev);
5556         return;
5557 err:
5558         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5559 }
5560
5561 static const struct pci_error_handlers be_eeh_handlers = {
5562         .error_detected = be_eeh_err_detected,
5563         .slot_reset = be_eeh_reset,
5564         .resume = be_eeh_resume,
5565 };
5566
5567 static struct pci_driver be_driver = {
5568         .name = DRV_NAME,
5569         .id_table = be_dev_ids,
5570         .probe = be_probe,
5571         .remove = be_remove,
5572         .suspend = be_suspend,
5573         .resume = be_resume,
5574         .shutdown = be_shutdown,
5575         .err_handler = &be_eeh_handlers
5576 };
5577
5578 static int __init be_init_module(void)
5579 {
5580         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5581             rx_frag_size != 2048) {
5582                 printk(KERN_WARNING DRV_NAME
5583                         " : Module param rx_frag_size must be 2048/4096/8192."
5584                         " Using 2048\n");
5585                 rx_frag_size = 2048;
5586         }
5587
5588         return pci_register_driver(&be_driver);
5589 }
5590 module_init(be_init_module);
5591
5592 static void __exit be_exit_module(void)
5593 {
5594         pci_unregister_driver(&be_driver);
5595 }
5596 module_exit(be_exit_module);