drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 /* num_vfs module param is obsolete.
  34  * Use sysfs method to enable/disable VFs.
  35  */
  36 static unsigned int num_vfs;
  37 module_param(num_vfs, uint, S_IRUGO);
  38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  39
  40 static ushort rx_frag_size = 2048;
  41 module_param(rx_frag_size, ushort, S_IRUGO);
  42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  43
  44 static const struct pci_device_id be_dev_ids[] = {
  45         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  47         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  48         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  51         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  52         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  53         { 0 }
  54 };
  55 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  56 /* UE Status Low CSR */
  57 static const char * const ue_status_low_desc[] = {
  58         "CEV",
  59         "CTX",
  60         "DBUF",
  61         "ERX",
  62         "Host",
  63         "MPU",
  64         "NDMA",
  65         "PTC ",
  66         "RDMA ",
  67         "RXF ",
  68         "RXIPS ",
  69         "RXULP0 ",
  70         "RXULP1 ",
  71         "RXULP2 ",
  72         "TIM ",
  73         "TPOST ",
  74         "TPRE ",
  75         "TXIPS ",
  76         "TXULP0 ",
  77         "TXULP1 ",
  78         "UC ",
  79         "WDMA ",
  80         "TXULP2 ",
  81         "HOST1 ",
  82         "P0_OB_LINK ",
  83         "P1_OB_LINK ",
  84         "HOST_GPIO ",
  85         "MBOX ",
  86         "ERX2 ",
  87         "SPARE ",
  88         "JTAG ",
  89         "MPU_INTPEND "
  90 };
  91
  92 /* UE Status High CSR */
  93 static const char * const ue_status_hi_desc[] = {
  94         "LPCMEMHOST",
  95         "MGMT_MAC",
  96         "PCS0ONLINE",
  97         "MPU_IRAM",
  98         "PCS1ONLINE",
  99         "PCTL0",
 100         "PCTL1",
 101         "PMEM",
 102         "RR",
 103         "TXPB",
 104         "RXPP",
 105         "XAUI",
 106         "TXP",
 107         "ARM",
 108         "IPC",
 109         "HOST2",
 110         "HOST3",
 111         "HOST4",
 112         "HOST5",
 113         "HOST6",
 114         "HOST7",
 115         "ECRC",
 116         "Poison TLP",
 117         "NETC",
 118         "PERIPH",
 119         "LLTXULP",
 120         "D2P",
 121         "RCON",
 122         "LDMA",
 123         "LLTXP",
 124         "LLTXPB",
 125         "Unknown"
 126 };
 127
 128 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 129 {
 130         struct be_dma_mem *mem = &q->dma_mem;
 131
 132         if (mem->va) {
 133                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 134                                   mem->dma);
 135                 mem->va = NULL;
 136         }
 137 }
 138
 139 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 140                           u16 len, u16 entry_size)
 141 {
 142         struct be_dma_mem *mem = &q->dma_mem;
 143
 144         memset(q, 0, sizeof(*q));
 145         q->len = len;
 146         q->entry_size = entry_size;
 147         mem->size = len * entry_size;
 148         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 149                                       GFP_KERNEL);
 150         if (!mem->va)
 151                 return -ENOMEM;
 152         return 0;
 153 }
 154
 155 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 156 {
 157         u32 reg, enabled;
 158
 159         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 160                               &reg);
 161         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162
 163         if (!enabled && enable)
 164                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else if (enabled && !enable)
 166                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 167         else
 168                 return;
 169
 170         pci_write_config_dword(adapter->pdev,
 171                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 172 }
 173
 174 static void be_intr_set(struct be_adapter *adapter, bool enable)
 175 {
 176         int status = 0;
 177
 178         /* On lancer interrupts can't be controlled via this register */
 179         if (lancer_chip(adapter))
 180                 return;
 181
 182         if (adapter->eeh_error)
 183                 return;
 184
 185         status = be_cmd_intr_set(adapter, enable);
 186         if (status)
 187                 be_reg_intr_set(adapter, enable);
 188 }
 189
 190 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 191 {
 192         u32 val = 0;
 193
 194         val |= qid & DB_RQ_RING_ID_MASK;
 195         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 196
 197         wmb();
 198         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 199 }
 200
 201 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 202                           u16 posted)
 203 {
 204         u32 val = 0;
 205
 206         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 207         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 208
 209         wmb();
 210         iowrite32(val, adapter->db + txo->db_offset);
 211 }
 212
 213 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 214                          bool arm, bool clear_int, u16 num_popped,
 215                          u32 eq_delay_mult_enc)
 216 {
 217         u32 val = 0;
 218
 219         val |= qid & DB_EQ_RING_ID_MASK;
 220         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 221
 222         if (adapter->eeh_error)
 223                 return;
 224
 225         if (arm)
 226                 val |= 1 << DB_EQ_REARM_SHIFT;
 227         if (clear_int)
 228                 val |= 1 << DB_EQ_CLR_SHIFT;
 229         val |= 1 << DB_EQ_EVNT_SHIFT;
 230         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 231         val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT;
 232         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 233 }
 234
 235 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 236 {
 237         u32 val = 0;
 238
 239         val |= qid & DB_CQ_RING_ID_MASK;
 240         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 241                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 242
 243         if (adapter->eeh_error)
 244                 return;
 245
 246         if (arm)
 247                 val |= 1 << DB_CQ_REARM_SHIFT;
 248         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 249         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 250 }
 251
 252 static int be_mac_addr_set(struct net_device *netdev, void *p)
 253 {
 254         struct be_adapter *adapter = netdev_priv(netdev);
 255         struct device *dev = &adapter->pdev->dev;
 256         struct sockaddr *addr = p;
 257         int status;
 258         u8 mac[ETH_ALEN];
 259         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 260
 261         if (!is_valid_ether_addr(addr->sa_data))
 262                 return -EADDRNOTAVAIL;
 263
 264         /* Proceed further only if, User provided MAC is different
 265          * from active MAC
 266          */
 267         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 268                 return 0;
 269
 270         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 271          * privilege or if PF did not provision the new MAC address.
 272          * On BE3, this cmd will always fail if the VF doesn't have the
 273          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 274          * the MAC for the VF.
 275          */
 276         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 277                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 278         if (!status) {
 279                 curr_pmac_id = adapter->pmac_id[0];
 280
 281                 /* Delete the old programmed MAC. This call may fail if the
 282                  * old MAC was already deleted by the PF driver.
 283                  */
 284                 if (adapter->pmac_id[0] != old_pmac_id)
 285                         be_cmd_pmac_del(adapter, adapter->if_handle,
 286                                         old_pmac_id, 0);
 287         }
 288
 289         /* Decide if the new MAC is successfully activated only after
 290          * querying the FW
 291          */
 292         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 293                                        adapter->if_handle, true, 0);
 294         if (status)
 295                 goto err;
 296
 297         /* The MAC change did not happen, either due to lack of privilege
 298          * or PF didn't pre-provision.
 299          */
 300         if (!ether_addr_equal(addr->sa_data, mac)) {
 301                 status = -EPERM;
 302                 goto err;
 303         }
 304
 305         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 306         dev_info(dev, "MAC address changed to %pM\n", mac);
 307         return 0;
 308 err:
 309         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 310         return status;
 311 }
 312
 313 /* BE2 supports only v0 cmd */
 314 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 315 {
 316         if (BE2_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else if (BE3_chip(adapter)) {
 321                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         } else {
 325                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 326
 327                 return &cmd->hw_stats;
 328         }
 329 }
 330
 331 /* BE2 supports only v0 cmd */
 332 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 333 {
 334         if (BE2_chip(adapter)) {
 335                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else if (BE3_chip(adapter)) {
 339                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         } else {
 343                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 344
 345                 return &hw_stats->erx;
 346         }
 347 }
 348
 349 static void populate_be_v0_stats(struct be_adapter *adapter)
 350 {
 351         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 352         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 353         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 354         struct be_port_rxf_stats_v0 *port_stats =
 355                                         &rxf_stats->port[adapter->port_num];
 356         struct be_drv_stats *drvs = &adapter->drv_stats;
 357
 358         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 359         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 360         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 361         drvs->rx_control_frames = port_stats->rx_control_frames;
 362         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 363         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 364         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 365         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 366         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 367         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 368         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 369         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 370         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 371         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 372         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 373         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 374         drvs->rx_dropped_header_too_small =
 375                 port_stats->rx_dropped_header_too_small;
 376         drvs->rx_address_filtered =
 377                                         port_stats->rx_address_filtered +
 378                                         port_stats->rx_vlan_filtered;
 379         drvs->rx_alignment_symbol_errors =
 380                 port_stats->rx_alignment_symbol_errors;
 381
 382         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 383         drvs->tx_controlframes = port_stats->tx_controlframes;
 384
 385         if (adapter->port_num)
 386                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 387         else
 388                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 389         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 390         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 391         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 392         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 393         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 394         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 395         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 396 }
 397
 398 static void populate_be_v1_stats(struct be_adapter *adapter)
 399 {
 400         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 401         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 402         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 403         struct be_port_rxf_stats_v1 *port_stats =
 404                                         &rxf_stats->port[adapter->port_num];
 405         struct be_drv_stats *drvs = &adapter->drv_stats;
 406
 407         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 408         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 409         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 410         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 411         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 412         drvs->rx_control_frames = port_stats->rx_control_frames;
 413         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 414         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 415         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 416         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 417         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 418         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 419         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 420         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 421         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 422         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 423         drvs->rx_dropped_header_too_small =
 424                 port_stats->rx_dropped_header_too_small;
 425         drvs->rx_input_fifo_overflow_drop =
 426                 port_stats->rx_input_fifo_overflow_drop;
 427         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 428         drvs->rx_alignment_symbol_errors =
 429                 port_stats->rx_alignment_symbol_errors;
 430         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 431         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 432         drvs->tx_controlframes = port_stats->tx_controlframes;
 433         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 434         drvs->jabber_events = port_stats->jabber_events;
 435         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 436         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 437         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 438         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 439         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 440         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 441         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 442 }
 443
 444 static void populate_be_v2_stats(struct be_adapter *adapter)
 445 {
 446         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 447         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 448         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 449         struct be_port_rxf_stats_v2 *port_stats =
 450                                         &rxf_stats->port[adapter->port_num];
 451         struct be_drv_stats *drvs = &adapter->drv_stats;
 452
 453         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 454         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 455         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 456         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 457         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 458         drvs->rx_control_frames = port_stats->rx_control_frames;
 459         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 460         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 461         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 462         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 463         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 464         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 465         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 466         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 467         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 468         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 469         drvs->rx_dropped_header_too_small =
 470                 port_stats->rx_dropped_header_too_small;
 471         drvs->rx_input_fifo_overflow_drop =
 472                 port_stats->rx_input_fifo_overflow_drop;
 473         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 474         drvs->rx_alignment_symbol_errors =
 475                 port_stats->rx_alignment_symbol_errors;
 476         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 477         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 478         drvs->tx_controlframes = port_stats->tx_controlframes;
 479         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 480         drvs->jabber_events = port_stats->jabber_events;
 481         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 482         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 483         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 484         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 485         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 486         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 487         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 488         if (be_roce_supported(adapter)) {
 489                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 490                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 491                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 492                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 493                 drvs->roce_drops_payload_len =
 494                         port_stats->roce_drops_payload_len;
 495         }
 496 }
 497
 498 static void populate_lancer_stats(struct be_adapter *adapter)
 499 {
 500         struct be_drv_stats *drvs = &adapter->drv_stats;
 501         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 502
 503         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 504         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 505         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 506         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 507         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 508         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 509         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 510         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 511         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 512         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 513         drvs->rx_dropped_tcp_length =
 514                                 pport_stats->rx_dropped_invalid_tcp_length;
 515         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 516         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 517         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 518         drvs->rx_dropped_header_too_small =
 519                                 pport_stats->rx_dropped_header_too_small;
 520         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 521         drvs->rx_address_filtered =
 522                                         pport_stats->rx_address_filtered +
 523                                         pport_stats->rx_vlan_filtered;
 524         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 525         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 526         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 527         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 528         drvs->jabber_events = pport_stats->rx_jabbers;
 529         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 530         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 531         drvs->rx_drops_too_many_frags =
 532                                 pport_stats->rx_drops_too_many_frags_lo;
 533 }
 534
 535 static void accumulate_16bit_val(u32 *acc, u16 val)
 536 {
 537 #define lo(x)                   (x & 0xFFFF)
 538 #define hi(x)                   (x & 0xFFFF0000)
 539         bool wrapped = val < lo(*acc);
 540         u32 newacc = hi(*acc) + val;
 541
 542         if (wrapped)
 543                 newacc += 65536;
 544         ACCESS_ONCE(*acc) = newacc;
 545 }
 546
 547 static void populate_erx_stats(struct be_adapter *adapter,
 548                                struct be_rx_obj *rxo, u32 erx_stat)
 549 {
 550         if (!BEx_chip(adapter))
 551                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 552         else
 553                 /* below erx HW counter can actually wrap around after
 554                  * 65535. Driver accumulates a 32-bit value
 555                  */
 556                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 557                                      (u16)erx_stat);
 558 }
 559
 560 void be_parse_stats(struct be_adapter *adapter)
 561 {
 562         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 563         struct be_rx_obj *rxo;
 564         int i;
 565         u32 erx_stat;
 566
 567         if (lancer_chip(adapter)) {
 568                 populate_lancer_stats(adapter);
 569         } else {
 570                 if (BE2_chip(adapter))
 571                         populate_be_v0_stats(adapter);
 572                 else if (BE3_chip(adapter))
 573                         /* for BE3 */
 574                         populate_be_v1_stats(adapter);
 575                 else
 576                         populate_be_v2_stats(adapter);
 577
 578                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 579                 for_all_rx_queues(adapter, rxo, i) {
 580                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 581                         populate_erx_stats(adapter, rxo, erx_stat);
 582                 }
 583         }
 584 }
 585
 586 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 587                                                 struct rtnl_link_stats64 *stats)
 588 {
 589         struct be_adapter *adapter = netdev_priv(netdev);
 590         struct be_drv_stats *drvs = &adapter->drv_stats;
 591         struct be_rx_obj *rxo;
 592         struct be_tx_obj *txo;
 593         u64 pkts, bytes;
 594         unsigned int start;
 595         int i;
 596
 597         for_all_rx_queues(adapter, rxo, i) {
 598                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 599
 600                 do {
 601                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 602                         pkts = rx_stats(rxo)->rx_pkts;
 603                         bytes = rx_stats(rxo)->rx_bytes;
 604                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 605                 stats->rx_packets += pkts;
 606                 stats->rx_bytes += bytes;
 607                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 608                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 609                                         rx_stats(rxo)->rx_drops_no_frags;
 610         }
 611
 612         for_all_tx_queues(adapter, txo, i) {
 613                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 614
 615                 do {
 616                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 617                         pkts = tx_stats(txo)->tx_pkts;
 618                         bytes = tx_stats(txo)->tx_bytes;
 619                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 620                 stats->tx_packets += pkts;
 621                 stats->tx_bytes += bytes;
 622         }
 623
 624         /* bad pkts received */
 625         stats->rx_errors = drvs->rx_crc_errors +
 626                 drvs->rx_alignment_symbol_errors +
 627                 drvs->rx_in_range_errors +
 628                 drvs->rx_out_range_errors +
 629                 drvs->rx_frame_too_long +
 630                 drvs->rx_dropped_too_small +
 631                 drvs->rx_dropped_too_short +
 632                 drvs->rx_dropped_header_too_small +
 633                 drvs->rx_dropped_tcp_length +
 634                 drvs->rx_dropped_runt;
 635
 636         /* detailed rx errors */
 637         stats->rx_length_errors = drvs->rx_in_range_errors +
 638                 drvs->rx_out_range_errors +
 639                 drvs->rx_frame_too_long;
 640
 641         stats->rx_crc_errors = drvs->rx_crc_errors;
 642
 643         /* frame alignment errors */
 644         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 645
 646         /* receiver fifo overrun */
 647         /* drops_no_pbuf is no per i/f, it's per BE card */
 648         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 649                                 drvs->rx_input_fifo_overflow_drop +
 650                                 drvs->rx_drops_no_pbuf;
 651         return stats;
 652 }
 653
 654 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 655 {
 656         struct net_device *netdev = adapter->netdev;
 657
 658         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 659                 netif_carrier_off(netdev);
 660                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 661         }
 662
 663         if (link_status)
 664                 netif_carrier_on(netdev);
 665         else
 666                 netif_carrier_off(netdev);
 667
 668         netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down");
 669 }
 670
 671 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 672 {
 673         struct be_tx_stats *stats = tx_stats(txo);
 674
 675         u64_stats_update_begin(&stats->sync);
 676         stats->tx_reqs++;
 677         stats->tx_bytes += skb->len;
 678         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 679         u64_stats_update_end(&stats->sync);
 680 }
 681
 682 /* Returns number of WRBs needed for the skb */
 683 static u32 skb_wrb_cnt(struct sk_buff *skb)
 684 {
 685         /* +1 for the header wrb */
 686         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 687 }
 688
 689 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 690 {
 691         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 692         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 693         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 694         wrb->rsvd0 = 0;
 695 }
 696
 697 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 698  * to avoid the swap and shift/mask operations in wrb_fill().
 699  */
 700 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 701 {
 702         wrb->frag_pa_hi = 0;
 703         wrb->frag_pa_lo = 0;
 704         wrb->frag_len = 0;
 705         wrb->rsvd0 = 0;
 706 }
 707
 708 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 709                                      struct sk_buff *skb)
 710 {
 711         u8 vlan_prio;
 712         u16 vlan_tag;
 713
 714         vlan_tag = skb_vlan_tag_get(skb);
 715         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 716         /* If vlan priority provided by OS is NOT in available bmap */
 717         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 718                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 719                                 adapter->recommended_prio;
 720
 721         return vlan_tag;
 722 }
 723
 724 /* Used only for IP tunnel packets */
 725 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 726 {
 727         return (inner_ip_hdr(skb)->version == 4) ?
 728                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 729 }
 730
 731 static u16 skb_ip_proto(struct sk_buff *skb)
 732 {
 733         return (ip_hdr(skb)->version == 4) ?
 734                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 735 }
 736
 737 static inline bool be_is_txq_full(struct be_tx_obj *txo)
 738 {
 739         return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;
 740 }
 741
 742 static inline bool be_can_txq_wake(struct be_tx_obj *txo)
 743 {
 744         return atomic_read(&txo->q.used) < txo->q.len / 2;
 745 }
 746
 747 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)
 748 {
 749         return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;
 750 }
 751
 752 static void be_get_wrb_params_from_skb(struct be_adapter *adapter,
 753                                        struct sk_buff *skb,
 754                                        struct be_wrb_params *wrb_params)
 755 {
 756         u16 proto;
 757
 758         if (skb_is_gso(skb)) {
 759                 BE_WRB_F_SET(wrb_params->features, LSO, 1);
 760                 wrb_params->lso_mss = skb_shinfo(skb)->gso_size;
 761                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 762                         BE_WRB_F_SET(wrb_params->features, LSO6, 1);
 763         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 764                 if (skb->encapsulation) {
 765                         BE_WRB_F_SET(wrb_params->features, IPCS, 1);
 766                         proto = skb_inner_ip_proto(skb);
 767                 } else {
 768                         proto = skb_ip_proto(skb);
 769                 }
 770                 if (proto == IPPROTO_TCP)
 771                         BE_WRB_F_SET(wrb_params->features, TCPCS, 1);
 772                 else if (proto == IPPROTO_UDP)
 773                         BE_WRB_F_SET(wrb_params->features, UDPCS, 1);
 774         }
 775
 776         if (skb_vlan_tag_present(skb)) {
 777                 BE_WRB_F_SET(wrb_params->features, VLAN, 1);
 778                 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 779         }
 780
 781         BE_WRB_F_SET(wrb_params->features, CRC, 1);
 782 }
 783
 784 static void wrb_fill_hdr(struct be_adapter *adapter,
 785                          struct be_eth_hdr_wrb *hdr,
 786                          struct be_wrb_params *wrb_params,
 787                          struct sk_buff *skb)
 788 {
 789         memset(hdr, 0, sizeof(*hdr));
 790
 791         SET_TX_WRB_HDR_BITS(crc, hdr,
 792                             BE_WRB_F_GET(wrb_params->features, CRC));
 793         SET_TX_WRB_HDR_BITS(ipcs, hdr,
 794                             BE_WRB_F_GET(wrb_params->features, IPCS));
 795         SET_TX_WRB_HDR_BITS(tcpcs, hdr,
 796                             BE_WRB_F_GET(wrb_params->features, TCPCS));
 797         SET_TX_WRB_HDR_BITS(udpcs, hdr,
 798                             BE_WRB_F_GET(wrb_params->features, UDPCS));
 799
 800         SET_TX_WRB_HDR_BITS(lso, hdr,
 801                             BE_WRB_F_GET(wrb_params->features, LSO));
 802         SET_TX_WRB_HDR_BITS(lso6, hdr,
 803                             BE_WRB_F_GET(wrb_params->features, LSO6));
 804         SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss);
 805
 806         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this
 807          * hack is not needed, the evt bit is set while ringing DB.
 808          */
 809         SET_TX_WRB_HDR_BITS(event, hdr,
 810                             BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW));
 811         SET_TX_WRB_HDR_BITS(vlan, hdr,
 812                             BE_WRB_F_GET(wrb_params->features, VLAN));
 813         SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag);
 814
 815         SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb));
 816         SET_TX_WRB_HDR_BITS(len, hdr, skb->len);
 817 }
 818
 819 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 820                           bool unmap_single)
 821 {
 822         dma_addr_t dma;
 823         u32 frag_len = le32_to_cpu(wrb->frag_len);
 824
 825
 826         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 827                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 828         if (frag_len) {
 829                 if (unmap_single)
 830                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 831                 else
 832                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 833         }
 834 }
 835
 836 /* Grab a WRB header for xmit */
 837 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo)
 838 {
 839         u16 head = txo->q.head;
 840
 841         queue_head_inc(&txo->q);
 842         return head;
 843 }
 844
 845 /* Set up the WRB header for xmit */
 846 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter,
 847                                 struct be_tx_obj *txo,
 848                                 struct be_wrb_params *wrb_params,
 849                                 struct sk_buff *skb, u16 head)
 850 {
 851         u32 num_frags = skb_wrb_cnt(skb);
 852         struct be_queue_info *txq = &txo->q;
 853         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head);
 854
 855         wrb_fill_hdr(adapter, hdr, wrb_params, skb);
 856         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 857
 858         BUG_ON(txo->sent_skb_list[head]);
 859         txo->sent_skb_list[head] = skb;
 860         txo->last_req_hdr = head;
 861         atomic_add(num_frags, &txq->used);
 862         txo->last_req_wrb_cnt = num_frags;
 863         txo->pend_wrb_cnt += num_frags;
 864 }
 865
 866 /* Setup a WRB fragment (buffer descriptor) for xmit */
 867 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,
 868                                  int len)
 869 {
 870         struct be_eth_wrb *wrb;
 871         struct be_queue_info *txq = &txo->q;
 872
 873         wrb = queue_head_node(txq);
 874         wrb_fill(wrb, busaddr, len);
 875         queue_head_inc(txq);
 876 }
 877
 878 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine
 879  * was invoked. The producer index is restored to the previous packet and the
 880  * WRBs of the current packet are unmapped. Invoked to handle tx setup errors.
 881  */
 882 static void be_xmit_restore(struct be_adapter *adapter,
 883                             struct be_tx_obj *txo, u16 head, bool map_single,
 884                             u32 copied)
 885 {
 886         struct device *dev;
 887         struct be_eth_wrb *wrb;
 888         struct be_queue_info *txq = &txo->q;
 889
 890         dev = &adapter->pdev->dev;
 891         txq->head = head;
 892
 893         /* skip the first wrb (hdr); it's not mapped */
 894         queue_head_inc(txq);
 895         while (copied) {
 896                 wrb = queue_head_node(txq);
 897                 unmap_tx_frag(dev, wrb, map_single);
 898                 map_single = false;
 899                 copied -= le32_to_cpu(wrb->frag_len);
 900                 queue_head_inc(txq);
 901         }
 902
 903         txq->head = head;
 904 }
 905
 906 /* Enqueue the given packet for transmit. This routine allocates WRBs for the
 907  * packet, dma maps the packet buffers and sets up the WRBs. Returns the number
 908  * of WRBs used up by the packet.
 909  */
 910 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 911                            struct sk_buff *skb,
 912                            struct be_wrb_params *wrb_params)
 913 {
 914         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 915         struct device *dev = &adapter->pdev->dev;
 916         struct be_queue_info *txq = &txo->q;
 917         bool map_single = false;
 918         u16 head = txq->head;
 919         dma_addr_t busaddr;
 920         int len;
 921
 922         head = be_tx_get_wrb_hdr(txo);
 923
 924         if (skb->len > skb->data_len) {
 925                 len = skb_headlen(skb);
 926
 927                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 928                 if (dma_mapping_error(dev, busaddr))
 929                         goto dma_err;
 930                 map_single = true;
 931                 be_tx_setup_wrb_frag(txo, busaddr, len);
 932                 copied += len;
 933         }
 934
 935         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 936                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 937                 len = skb_frag_size(frag);
 938
 939                 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
 940                 if (dma_mapping_error(dev, busaddr))
 941                         goto dma_err;
 942                 be_tx_setup_wrb_frag(txo, busaddr, len);
 943                 copied += len;
 944         }
 945
 946         be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);
 947
 948         be_tx_stats_update(txo, skb);
 949         return wrb_cnt;
 950
 951 dma_err:
 952         adapter->drv_stats.dma_map_errors++;
 953         be_xmit_restore(adapter, txo, head, map_single, copied);
 954         return 0;
 955 }
 956
 957 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 958 {
 959         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 960 }
 961
 962 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 963                                              struct sk_buff *skb,
 964                                              struct be_wrb_params
 965                                              *wrb_params)
 966 {
 967         u16 vlan_tag = 0;
 968
 969         skb = skb_share_check(skb, GFP_ATOMIC);
 970         if (unlikely(!skb))
 971                 return skb;
 972
 973         if (skb_vlan_tag_present(skb))
 974                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 975
 976         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 977                 if (!vlan_tag)
 978                         vlan_tag = adapter->pvid;
 979                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 980                  * skip VLAN insertion
 981                  */
 982                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
 983         }
 984
 985         if (vlan_tag) {
 986                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 987                                                 vlan_tag);
 988                 if (unlikely(!skb))
 989                         return skb;
 990                 skb->vlan_tci = 0;
 991         }
 992
 993         /* Insert the outer VLAN, if any */
 994         if (adapter->qnq_vid) {
 995                 vlan_tag = adapter->qnq_vid;
 996                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 997                                                 vlan_tag);
 998                 if (unlikely(!skb))
 999                         return skb;
1000                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1001         }
1002
1003         return skb;
1004 }
1005
1006 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
1007 {
1008         struct ethhdr *eh = (struct ethhdr *)skb->data;
1009         u16 offset = ETH_HLEN;
1010
1011         if (eh->h_proto == htons(ETH_P_IPV6)) {
1012                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
1013
1014                 offset += sizeof(struct ipv6hdr);
1015                 if (ip6h->nexthdr != NEXTHDR_TCP &&
1016                     ip6h->nexthdr != NEXTHDR_UDP) {
1017                         struct ipv6_opt_hdr *ehdr =
1018                                 (struct ipv6_opt_hdr *)(skb->data + offset);
1019
1020                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
1021                         if (ehdr->hdrlen == 0xff)
1022                                 return true;
1023                 }
1024         }
1025         return false;
1026 }
1027
1028 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
1029 {
1030         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
1031 }
1032
1033 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
1034 {
1035         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
1036 }
1037
1038 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
1039                                                   struct sk_buff *skb,
1040                                                   struct be_wrb_params
1041                                                   *wrb_params)
1042 {
1043         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1044         unsigned int eth_hdr_len;
1045         struct iphdr *ip;
1046
1047         /* For padded packets, BE HW modifies tot_len field in IP header
1048          * incorrecly when VLAN tag is inserted by HW.
1049          * For padded packets, Lancer computes incorrect checksum.
1050          */
1051         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
1052                                                 VLAN_ETH_HLEN : ETH_HLEN;
1053         if (skb->len <= 60 &&
1054             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
1055             is_ipv4_pkt(skb)) {
1056                 ip = (struct iphdr *)ip_hdr(skb);
1057                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
1058         }
1059
1060         /* If vlan tag is already inlined in the packet, skip HW VLAN
1061          * tagging in pvid-tagging mode
1062          */
1063         if (be_pvid_tagging_enabled(adapter) &&
1064             veh->h_vlan_proto == htons(ETH_P_8021Q))
1065                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1066
1067         /* HW has a bug wherein it will calculate CSUM for VLAN
1068          * pkts even though it is disabled.
1069          * Manually insert VLAN in pkt.
1070          */
1071         if (skb->ip_summed != CHECKSUM_PARTIAL &&
1072             skb_vlan_tag_present(skb)) {
1073                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1074                 if (unlikely(!skb))
1075                         goto err;
1076         }
1077
1078         /* HW may lockup when VLAN HW tagging is requested on
1079          * certain ipv6 packets. Drop such pkts if the HW workaround to
1080          * skip HW tagging is not enabled by FW.
1081          */
1082         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
1083                      (adapter->pvid || adapter->qnq_vid) &&
1084                      !qnq_async_evt_rcvd(adapter)))
1085                 goto tx_drop;
1086
1087         /* Manual VLAN tag insertion to prevent:
1088          * ASIC lockup when the ASIC inserts VLAN tag into
1089          * certain ipv6 packets. Insert VLAN tags in driver,
1090          * and set event, completion, vlan bits accordingly
1091          * in the Tx WRB.
1092          */
1093         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1094             be_vlan_tag_tx_chk(adapter, skb)) {
1095                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1096                 if (unlikely(!skb))
1097                         goto err;
1098         }
1099
1100         return skb;
1101 tx_drop:
1102         dev_kfree_skb_any(skb);
1103 err:
1104         return NULL;
1105 }
1106
1107 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1108                                            struct sk_buff *skb,
1109                                            struct be_wrb_params *wrb_params)
1110 {
1111         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1112          * less may cause a transmit stall on that port. So the work-around is
1113          * to pad short packets (<= 32 bytes) to a 36-byte length.
1114          */
1115         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1116                 if (skb_put_padto(skb, 36))
1117                         return NULL;
1118         }
1119
1120         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1121                 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params);
1122                 if (!skb)
1123                         return NULL;
1124         }
1125
1126         return skb;
1127 }
1128
1129 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1130 {
1131         struct be_queue_info *txq = &txo->q;
1132         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1133
1134         /* Mark the last request eventable if it hasn't been marked already */
1135         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1136                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1137
1138         /* compose a dummy wrb if there are odd set of wrbs to notify */
1139         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1140                 wrb_fill_dummy(queue_head_node(txq));
1141                 queue_head_inc(txq);
1142                 atomic_inc(&txq->used);
1143                 txo->pend_wrb_cnt++;
1144                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1145                                            TX_HDR_WRB_NUM_SHIFT);
1146                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1147                                           TX_HDR_WRB_NUM_SHIFT);
1148         }
1149         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1150         txo->pend_wrb_cnt = 0;
1151 }
1152
1153 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1154 {
1155         struct be_adapter *adapter = netdev_priv(netdev);
1156         u16 q_idx = skb_get_queue_mapping(skb);
1157         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1158         struct be_wrb_params wrb_params = { 0 };
1159         bool flush = !skb->xmit_more;
1160         u16 wrb_cnt;
1161
1162         skb = be_xmit_workarounds(adapter, skb, &wrb_params);
1163         if (unlikely(!skb))
1164                 goto drop;
1165
1166         be_get_wrb_params_from_skb(adapter, skb, &wrb_params);
1167
1168         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1169         if (unlikely(!wrb_cnt)) {
1170                 dev_kfree_skb_any(skb);
1171                 goto drop;
1172         }
1173
1174         if (be_is_txq_full(txo)) {
1175                 netif_stop_subqueue(netdev, q_idx);
1176                 tx_stats(txo)->tx_stops++;
1177         }
1178
1179         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1180                 be_xmit_flush(adapter, txo);
1181
1182         return NETDEV_TX_OK;
1183 drop:
1184         tx_stats(txo)->tx_drv_drops++;
1185         /* Flush the already enqueued tx requests */
1186         if (flush && txo->pend_wrb_cnt)
1187                 be_xmit_flush(adapter, txo);
1188
1189         return NETDEV_TX_OK;
1190 }
1191
1192 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1193 {
1194         struct be_adapter *adapter = netdev_priv(netdev);
1195         struct device *dev = &adapter->pdev->dev;
1196
1197         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1198                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1199                          BE_MIN_MTU, BE_MAX_MTU);
1200                 return -EINVAL;
1201         }
1202
1203         dev_info(dev, "MTU changed from %d to %d bytes\n",
1204                  netdev->mtu, new_mtu);
1205         netdev->mtu = new_mtu;
1206         return 0;
1207 }
1208
1209 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1210 {
1211         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1212                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1213 }
1214
1215 static int be_set_vlan_promisc(struct be_adapter *adapter)
1216 {
1217         struct device *dev = &adapter->pdev->dev;
1218         int status;
1219
1220         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1221                 return 0;
1222
1223         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1224         if (!status) {
1225                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1226                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1227         } else {
1228                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1229         }
1230         return status;
1231 }
1232
1233 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1234 {
1235         struct device *dev = &adapter->pdev->dev;
1236         int status;
1237
1238         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1239         if (!status) {
1240                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1241                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1242         }
1243         return status;
1244 }
1245
1246 /*
1247  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1248  * If the user configures more, place BE in vlan promiscuous mode.
1249  */
1250 static int be_vid_config(struct be_adapter *adapter)
1251 {
1252         struct device *dev = &adapter->pdev->dev;
1253         u16 vids[BE_NUM_VLANS_SUPPORTED];
1254         u16 num = 0, i = 0;
1255         int status = 0;
1256
1257         /* No need to further configure vids if in promiscuous mode */
1258         if (be_in_all_promisc(adapter))
1259                 return 0;
1260
1261         if (adapter->vlans_added > be_max_vlans(adapter))
1262                 return be_set_vlan_promisc(adapter);
1263
1264         /* Construct VLAN Table to give to HW */
1265         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1266                 vids[num++] = cpu_to_le16(i);
1267
1268         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1269         if (status) {
1270                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1271                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1272                 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS ||
1273                     addl_status(status) ==
1274                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1275                         return be_set_vlan_promisc(adapter);
1276         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1277                 status = be_clear_vlan_promisc(adapter);
1278         }
1279         return status;
1280 }
1281
1282 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1283 {
1284         struct be_adapter *adapter = netdev_priv(netdev);
1285         int status = 0;
1286
1287         /* Packets with VID 0 are always received by Lancer by default */
1288         if (lancer_chip(adapter) && vid == 0)
1289                 return status;
1290
1291         if (test_bit(vid, adapter->vids))
1292                 return status;
1293
1294         set_bit(vid, adapter->vids);
1295         adapter->vlans_added++;
1296
1297         status = be_vid_config(adapter);
1298         if (status) {
1299                 adapter->vlans_added--;
1300                 clear_bit(vid, adapter->vids);
1301         }
1302
1303         return status;
1304 }
1305
1306 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1307 {
1308         struct be_adapter *adapter = netdev_priv(netdev);
1309
1310         /* Packets with VID 0 are always received by Lancer by default */
1311         if (lancer_chip(adapter) && vid == 0)
1312                 return 0;
1313
1314         clear_bit(vid, adapter->vids);
1315         adapter->vlans_added--;
1316
1317         return be_vid_config(adapter);
1318 }
1319
1320 static void be_clear_all_promisc(struct be_adapter *adapter)
1321 {
1322         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1323         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1324 }
1325
1326 static void be_set_all_promisc(struct be_adapter *adapter)
1327 {
1328         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1329         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1330 }
1331
1332 static void be_set_mc_promisc(struct be_adapter *adapter)
1333 {
1334         int status;
1335
1336         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1337                 return;
1338
1339         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1340         if (!status)
1341                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1342 }
1343
1344 static void be_set_mc_list(struct be_adapter *adapter)
1345 {
1346         int status;
1347
1348         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1349         if (!status)
1350                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1351         else
1352                 be_set_mc_promisc(adapter);
1353 }
1354
1355 static void be_set_uc_list(struct be_adapter *adapter)
1356 {
1357         struct netdev_hw_addr *ha;
1358         int i = 1; /* First slot is claimed by the Primary MAC */
1359
1360         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1361                 be_cmd_pmac_del(adapter, adapter->if_handle,
1362                                 adapter->pmac_id[i], 0);
1363
1364         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1365                 be_set_all_promisc(adapter);
1366                 return;
1367         }
1368
1369         netdev_for_each_uc_addr(ha, adapter->netdev) {
1370                 adapter->uc_macs++; /* First slot is for Primary MAC */
1371                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1372                                 &adapter->pmac_id[adapter->uc_macs], 0);
1373         }
1374 }
1375
1376 static void be_clear_uc_list(struct be_adapter *adapter)
1377 {
1378         int i;
1379
1380         for (i = 1; i < (adapter->uc_macs + 1); i++)
1381                 be_cmd_pmac_del(adapter, adapter->if_handle,
1382                                 adapter->pmac_id[i], 0);
1383         adapter->uc_macs = 0;
1384 }
1385
1386 static void be_set_rx_mode(struct net_device *netdev)
1387 {
1388         struct be_adapter *adapter = netdev_priv(netdev);
1389
1390         if (netdev->flags & IFF_PROMISC) {
1391                 be_set_all_promisc(adapter);
1392                 return;
1393         }
1394
1395         /* Interface was previously in promiscuous mode; disable it */
1396         if (be_in_all_promisc(adapter)) {
1397                 be_clear_all_promisc(adapter);
1398                 if (adapter->vlans_added)
1399                         be_vid_config(adapter);
1400         }
1401
1402         /* Enable multicast promisc if num configured exceeds what we support */
1403         if (netdev->flags & IFF_ALLMULTI ||
1404             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1405                 be_set_mc_promisc(adapter);
1406                 return;
1407         }
1408
1409         if (netdev_uc_count(netdev) != adapter->uc_macs)
1410                 be_set_uc_list(adapter);
1411
1412         be_set_mc_list(adapter);
1413 }
1414
1415 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1416 {
1417         struct be_adapter *adapter = netdev_priv(netdev);
1418         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1419         int status;
1420
1421         if (!sriov_enabled(adapter))
1422                 return -EPERM;
1423
1424         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1425                 return -EINVAL;
1426
1427         /* Proceed further only if user provided MAC is different
1428          * from active MAC
1429          */
1430         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1431                 return 0;
1432
1433         if (BEx_chip(adapter)) {
1434                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1435                                 vf + 1);
1436
1437                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1438                                          &vf_cfg->pmac_id, vf + 1);
1439         } else {
1440                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1441                                         vf + 1);
1442         }
1443
1444         if (status) {
1445                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1446                         mac, vf, status);
1447                 return be_cmd_status(status);
1448         }
1449
1450         ether_addr_copy(vf_cfg->mac_addr, mac);
1451
1452         return 0;
1453 }
1454
1455 static int be_get_vf_config(struct net_device *netdev, int vf,
1456                             struct ifla_vf_info *vi)
1457 {
1458         struct be_adapter *adapter = netdev_priv(netdev);
1459         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1460
1461         if (!sriov_enabled(adapter))
1462                 return -EPERM;
1463
1464         if (vf >= adapter->num_vfs)
1465                 return -EINVAL;
1466
1467         vi->vf = vf;
1468         vi->max_tx_rate = vf_cfg->tx_rate;
1469         vi->min_tx_rate = 0;
1470         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1471         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1472         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1473         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1474         vi->spoofchk = adapter->vf_cfg[vf].spoofchk;
1475
1476         return 0;
1477 }
1478
1479 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1480 {
1481         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1482         u16 vids[BE_NUM_VLANS_SUPPORTED];
1483         int vf_if_id = vf_cfg->if_handle;
1484         int status;
1485
1486         /* Enable Transparent VLAN Tagging */
1487         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0);
1488         if (status)
1489                 return status;
1490
1491         /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1492         vids[0] = 0;
1493         status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1494         if (!status)
1495                 dev_info(&adapter->pdev->dev,
1496                          "Cleared guest VLANs on VF%d", vf);
1497
1498         /* After TVT is enabled, disallow VFs to program VLAN filters */
1499         if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1500                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1501                                                   ~BE_PRIV_FILTMGMT, vf + 1);
1502                 if (!status)
1503                         vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1504         }
1505         return 0;
1506 }
1507
1508 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1509 {
1510         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1511         struct device *dev = &adapter->pdev->dev;
1512         int status;
1513
1514         /* Reset Transparent VLAN Tagging. */
1515         status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1516                                        vf_cfg->if_handle, 0, 0);
1517         if (status)
1518                 return status;
1519
1520         /* Allow VFs to program VLAN filtering */
1521         if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1522                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1523                                                   BE_PRIV_FILTMGMT, vf + 1);
1524                 if (!status) {
1525                         vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1526                         dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1527                 }
1528         }
1529
1530         dev_info(dev,
1531                  "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1532         return 0;
1533 }
1534
1535 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1536 {
1537         struct be_adapter *adapter = netdev_priv(netdev);
1538         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1539         int status;
1540
1541         if (!sriov_enabled(adapter))
1542                 return -EPERM;
1543
1544         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1545                 return -EINVAL;
1546
1547         if (vlan || qos) {
1548                 vlan |= qos << VLAN_PRIO_SHIFT;
1549                 status = be_set_vf_tvt(adapter, vf, vlan);
1550         } else {
1551                 status = be_clear_vf_tvt(adapter, vf);
1552         }
1553
1554         if (status) {
1555                 dev_err(&adapter->pdev->dev,
1556                         "VLAN %d config on VF %d failed : %#x\n", vlan, vf,
1557                         status);
1558                 return be_cmd_status(status);
1559         }
1560
1561         vf_cfg->vlan_tag = vlan;
1562         return 0;
1563 }
1564
1565 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1566                              int min_tx_rate, int max_tx_rate)
1567 {
1568         struct be_adapter *adapter = netdev_priv(netdev);
1569         struct device *dev = &adapter->pdev->dev;
1570         int percent_rate, status = 0;
1571         u16 link_speed = 0;
1572         u8 link_status;
1573
1574         if (!sriov_enabled(adapter))
1575                 return -EPERM;
1576
1577         if (vf >= adapter->num_vfs)
1578                 return -EINVAL;
1579
1580         if (min_tx_rate)
1581                 return -EINVAL;
1582
1583         if (!max_tx_rate)
1584                 goto config_qos;
1585
1586         status = be_cmd_link_status_query(adapter, &link_speed,
1587                                           &link_status, 0);
1588         if (status)
1589                 goto err;
1590
1591         if (!link_status) {
1592                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1593                 status = -ENETDOWN;
1594                 goto err;
1595         }
1596
1597         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1598                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1599                         link_speed);
1600                 status = -EINVAL;
1601                 goto err;
1602         }
1603
1604         /* On Skyhawk the QOS setting must be done only as a % value */
1605         percent_rate = link_speed / 100;
1606         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1607                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1608                         percent_rate);
1609                 status = -EINVAL;
1610                 goto err;
1611         }
1612
1613 config_qos:
1614         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1615         if (status)
1616                 goto err;
1617
1618         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1619         return 0;
1620
1621 err:
1622         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1623                 max_tx_rate, vf);
1624         return be_cmd_status(status);
1625 }
1626
1627 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1628                                 int link_state)
1629 {
1630         struct be_adapter *adapter = netdev_priv(netdev);
1631         int status;
1632
1633         if (!sriov_enabled(adapter))
1634                 return -EPERM;
1635
1636         if (vf >= adapter->num_vfs)
1637                 return -EINVAL;
1638
1639         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1640         if (status) {
1641                 dev_err(&adapter->pdev->dev,
1642                         "Link state change on VF %d failed: %#x\n", vf, status);
1643                 return be_cmd_status(status);
1644         }
1645
1646         adapter->vf_cfg[vf].plink_tracking = link_state;
1647
1648         return 0;
1649 }
1650
1651 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
1652 {
1653         struct be_adapter *adapter = netdev_priv(netdev);
1654         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1655         u8 spoofchk;
1656         int status;
1657
1658         if (!sriov_enabled(adapter))
1659                 return -EPERM;
1660
1661         if (vf >= adapter->num_vfs)
1662                 return -EINVAL;
1663
1664         if (BEx_chip(adapter))
1665                 return -EOPNOTSUPP;
1666
1667         if (enable == vf_cfg->spoofchk)
1668                 return 0;
1669
1670         spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK;
1671
1672         status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle,
1673                                        0, spoofchk);
1674         if (status) {
1675                 dev_err(&adapter->pdev->dev,
1676                         "Spoofchk change on VF %d failed: %#x\n", vf, status);
1677                 return be_cmd_status(status);
1678         }
1679
1680         vf_cfg->spoofchk = enable;
1681         return 0;
1682 }
1683
1684 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1685                           ulong now)
1686 {
1687         aic->rx_pkts_prev = rx_pkts;
1688         aic->tx_reqs_prev = tx_pkts;
1689         aic->jiffies = now;
1690 }
1691
1692 static int be_get_new_eqd(struct be_eq_obj *eqo)
1693 {
1694         struct be_adapter *adapter = eqo->adapter;
1695         int eqd, start;
1696         struct be_aic_obj *aic;
1697         struct be_rx_obj *rxo;
1698         struct be_tx_obj *txo;
1699         u64 rx_pkts = 0, tx_pkts = 0;
1700         ulong now;
1701         u32 pps, delta;
1702         int i;
1703
1704         aic = &adapter->aic_obj[eqo->idx];
1705         if (!aic->enable) {
1706                 if (aic->jiffies)
1707                         aic->jiffies = 0;
1708                 eqd = aic->et_eqd;
1709                 return eqd;
1710         }
1711
1712         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
1713                 do {
1714                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1715                         rx_pkts += rxo->stats.rx_pkts;
1716                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1717         }
1718
1719         for_all_tx_queues_on_eq(adapter, eqo, txo, i) {
1720                 do {
1721                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1722                         tx_pkts += txo->stats.tx_reqs;
1723                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1724         }
1725
1726         /* Skip, if wrapped around or first calculation */
1727         now = jiffies;
1728         if (!aic->jiffies || time_before(now, aic->jiffies) ||
1729             rx_pkts < aic->rx_pkts_prev ||
1730             tx_pkts < aic->tx_reqs_prev) {
1731                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1732                 return aic->prev_eqd;
1733         }
1734
1735         delta = jiffies_to_msecs(now - aic->jiffies);
1736         if (delta == 0)
1737                 return aic->prev_eqd;
1738
1739         pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1740                 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1741         eqd = (pps / 15000) << 2;
1742
1743         if (eqd < 8)
1744                 eqd = 0;
1745         eqd = min_t(u32, eqd, aic->max_eqd);
1746         eqd = max_t(u32, eqd, aic->min_eqd);
1747
1748         be_aic_update(aic, rx_pkts, tx_pkts, now);
1749
1750         return eqd;
1751 }
1752
1753 /* For Skyhawk-R only */
1754 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo)
1755 {
1756         struct be_adapter *adapter = eqo->adapter;
1757         struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx];
1758         ulong now = jiffies;
1759         int eqd;
1760         u32 mult_enc;
1761
1762         if (!aic->enable)
1763                 return 0;
1764
1765         if (time_before_eq(now, aic->jiffies) ||
1766             jiffies_to_msecs(now - aic->jiffies) < 1)
1767                 eqd = aic->prev_eqd;
1768         else
1769                 eqd = be_get_new_eqd(eqo);
1770
1771         if (eqd > 100)
1772                 mult_enc = R2I_DLY_ENC_1;
1773         else if (eqd > 60)
1774                 mult_enc = R2I_DLY_ENC_2;
1775         else if (eqd > 20)
1776                 mult_enc = R2I_DLY_ENC_3;
1777         else
1778                 mult_enc = R2I_DLY_ENC_0;
1779
1780         aic->prev_eqd = eqd;
1781
1782         return mult_enc;
1783 }
1784
1785 void be_eqd_update(struct be_adapter *adapter, bool force_update)
1786 {
1787         struct be_set_eqd set_eqd[MAX_EVT_QS];
1788         struct be_aic_obj *aic;
1789         struct be_eq_obj *eqo;
1790         int i, num = 0, eqd;
1791
1792         for_all_evt_queues(adapter, eqo, i) {
1793                 aic = &adapter->aic_obj[eqo->idx];
1794                 eqd = be_get_new_eqd(eqo);
1795                 if (force_update || eqd != aic->prev_eqd) {
1796                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1797                         set_eqd[num].eq_id = eqo->q.id;
1798                         aic->prev_eqd = eqd;
1799                         num++;
1800                 }
1801         }
1802
1803         if (num)
1804                 be_cmd_modify_eqd(adapter, set_eqd, num);
1805 }
1806
1807 static void be_rx_stats_update(struct be_rx_obj *rxo,
1808                                struct be_rx_compl_info *rxcp)
1809 {
1810         struct be_rx_stats *stats = rx_stats(rxo);
1811
1812         u64_stats_update_begin(&stats->sync);
1813         stats->rx_compl++;
1814         stats->rx_bytes += rxcp->pkt_size;
1815         stats->rx_pkts++;
1816         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1817                 stats->rx_mcast_pkts++;
1818         if (rxcp->err)
1819                 stats->rx_compl_err++;
1820         u64_stats_update_end(&stats->sync);
1821 }
1822
1823 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1824 {
1825         /* L4 checksum is not reliable for non TCP/UDP packets.
1826          * Also ignore ipcksm for ipv6 pkts
1827          */
1828         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1829                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1830 }
1831
1832 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1833 {
1834         struct be_adapter *adapter = rxo->adapter;
1835         struct be_rx_page_info *rx_page_info;
1836         struct be_queue_info *rxq = &rxo->q;
1837         u16 frag_idx = rxq->tail;
1838
1839         rx_page_info = &rxo->page_info_tbl[frag_idx];
1840         BUG_ON(!rx_page_info->page);
1841
1842         if (rx_page_info->last_frag) {
1843                 dma_unmap_page(&adapter->pdev->dev,
1844                                dma_unmap_addr(rx_page_info, bus),
1845                                adapter->big_page_size, DMA_FROM_DEVICE);
1846                 rx_page_info->last_frag = false;
1847         } else {
1848                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1849                                         dma_unmap_addr(rx_page_info, bus),
1850                                         rx_frag_size, DMA_FROM_DEVICE);
1851         }
1852
1853         queue_tail_inc(rxq);
1854         atomic_dec(&rxq->used);
1855         return rx_page_info;
1856 }
1857
1858 /* Throwaway the data in the Rx completion */
1859 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1860                                 struct be_rx_compl_info *rxcp)
1861 {
1862         struct be_rx_page_info *page_info;
1863         u16 i, num_rcvd = rxcp->num_rcvd;
1864
1865         for (i = 0; i < num_rcvd; i++) {
1866                 page_info = get_rx_page_info(rxo);
1867                 put_page(page_info->page);
1868                 memset(page_info, 0, sizeof(*page_info));
1869         }
1870 }
1871
1872 /*
1873  * skb_fill_rx_data forms a complete skb for an ether frame
1874  * indicated by rxcp.
1875  */
1876 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1877                              struct be_rx_compl_info *rxcp)
1878 {
1879         struct be_rx_page_info *page_info;
1880         u16 i, j;
1881         u16 hdr_len, curr_frag_len, remaining;
1882         u8 *start;
1883
1884         page_info = get_rx_page_info(rxo);
1885         start = page_address(page_info->page) + page_info->page_offset;
1886         prefetch(start);
1887
1888         /* Copy data in the first descriptor of this completion */
1889         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1890
1891         skb->len = curr_frag_len;
1892         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1893                 memcpy(skb->data, start, curr_frag_len);
1894                 /* Complete packet has now been moved to data */
1895                 put_page(page_info->page);
1896                 skb->data_len = 0;
1897                 skb->tail += curr_frag_len;
1898         } else {
1899                 hdr_len = ETH_HLEN;
1900                 memcpy(skb->data, start, hdr_len);
1901                 skb_shinfo(skb)->nr_frags = 1;
1902                 skb_frag_set_page(skb, 0, page_info->page);
1903                 skb_shinfo(skb)->frags[0].page_offset =
1904                                         page_info->page_offset + hdr_len;
1905                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1906                                   curr_frag_len - hdr_len);
1907                 skb->data_len = curr_frag_len - hdr_len;
1908                 skb->truesize += rx_frag_size;
1909                 skb->tail += hdr_len;
1910         }
1911         page_info->page = NULL;
1912
1913         if (rxcp->pkt_size <= rx_frag_size) {
1914                 BUG_ON(rxcp->num_rcvd != 1);
1915                 return;
1916         }
1917
1918         /* More frags present for this completion */
1919         remaining = rxcp->pkt_size - curr_frag_len;
1920         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1921                 page_info = get_rx_page_info(rxo);
1922                 curr_frag_len = min(remaining, rx_frag_size);
1923
1924                 /* Coalesce all frags from the same physical page in one slot */
1925                 if (page_info->page_offset == 0) {
1926                         /* Fresh page */
1927                         j++;
1928                         skb_frag_set_page(skb, j, page_info->page);
1929                         skb_shinfo(skb)->frags[j].page_offset =
1930                                                         page_info->page_offset;
1931                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1932                         skb_shinfo(skb)->nr_frags++;
1933                 } else {
1934                         put_page(page_info->page);
1935                 }
1936
1937                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1938                 skb->len += curr_frag_len;
1939                 skb->data_len += curr_frag_len;
1940                 skb->truesize += rx_frag_size;
1941                 remaining -= curr_frag_len;
1942                 page_info->page = NULL;
1943         }
1944         BUG_ON(j > MAX_SKB_FRAGS);
1945 }
1946
1947 /* Process the RX completion indicated by rxcp when GRO is disabled */
1948 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1949                                 struct be_rx_compl_info *rxcp)
1950 {
1951         struct be_adapter *adapter = rxo->adapter;
1952         struct net_device *netdev = adapter->netdev;
1953         struct sk_buff *skb;
1954
1955         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1956         if (unlikely(!skb)) {
1957                 rx_stats(rxo)->rx_drops_no_skbs++;
1958                 be_rx_compl_discard(rxo, rxcp);
1959                 return;
1960         }
1961
1962         skb_fill_rx_data(rxo, skb, rxcp);
1963
1964         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1965                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1966         else
1967                 skb_checksum_none_assert(skb);
1968
1969         skb->protocol = eth_type_trans(skb, netdev);
1970         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1971         if (netdev->features & NETIF_F_RXHASH)
1972                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1973
1974         skb->csum_level = rxcp->tunneled;
1975         skb_mark_napi_id(skb, napi);
1976
1977         if (rxcp->vlanf)
1978                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1979
1980         netif_receive_skb(skb);
1981 }
1982
1983 /* Process the RX completion indicated by rxcp when GRO is enabled */
1984 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1985                                     struct napi_struct *napi,
1986                                     struct be_rx_compl_info *rxcp)
1987 {
1988         struct be_adapter *adapter = rxo->adapter;
1989         struct be_rx_page_info *page_info;
1990         struct sk_buff *skb = NULL;
1991         u16 remaining, curr_frag_len;
1992         u16 i, j;
1993
1994         skb = napi_get_frags(napi);
1995         if (!skb) {
1996                 be_rx_compl_discard(rxo, rxcp);
1997                 return;
1998         }
1999
2000         remaining = rxcp->pkt_size;
2001         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
2002                 page_info = get_rx_page_info(rxo);
2003
2004                 curr_frag_len = min(remaining, rx_frag_size);
2005
2006                 /* Coalesce all frags from the same physical page in one slot */
2007                 if (i == 0 || page_info->page_offset == 0) {
2008                         /* First frag or Fresh page */
2009                         j++;
2010                         skb_frag_set_page(skb, j, page_info->page);
2011                         skb_shinfo(skb)->frags[j].page_offset =
2012                                                         page_info->page_offset;
2013                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
2014                 } else {
2015                         put_page(page_info->page);
2016                 }
2017                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
2018                 skb->truesize += rx_frag_size;
2019                 remaining -= curr_frag_len;
2020                 memset(page_info, 0, sizeof(*page_info));
2021         }
2022         BUG_ON(j > MAX_SKB_FRAGS);
2023
2024         skb_shinfo(skb)->nr_frags = j + 1;
2025         skb->len = rxcp->pkt_size;
2026         skb->data_len = rxcp->pkt_size;
2027         skb->ip_summed = CHECKSUM_UNNECESSARY;
2028         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
2029         if (adapter->netdev->features & NETIF_F_RXHASH)
2030                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
2031
2032         skb->csum_level = rxcp->tunneled;
2033         skb_mark_napi_id(skb, napi);
2034
2035         if (rxcp->vlanf)
2036                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
2037
2038         napi_gro_frags(napi);
2039 }
2040
2041 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
2042                                  struct be_rx_compl_info *rxcp)
2043 {
2044         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
2045         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
2046         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
2047         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
2048         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
2049         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
2050         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
2051         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
2052         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
2053         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
2054         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
2055         if (rxcp->vlanf) {
2056                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
2057                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
2058         }
2059         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
2060         rxcp->tunneled =
2061                 GET_RX_COMPL_V1_BITS(tunneled, compl);
2062 }
2063
2064 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
2065                                  struct be_rx_compl_info *rxcp)
2066 {
2067         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
2068         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
2069         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
2070         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
2071         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
2072         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
2073         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
2074         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
2075         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
2076         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
2077         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
2078         if (rxcp->vlanf) {
2079                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
2080                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
2081         }
2082         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
2083         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
2084 }
2085
2086 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
2087 {
2088         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
2089         struct be_rx_compl_info *rxcp = &rxo->rxcp;
2090         struct be_adapter *adapter = rxo->adapter;
2091
2092         /* For checking the valid bit it is Ok to use either definition as the
2093          * valid bit is at the same position in both v0 and v1 Rx compl */
2094         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
2095                 return NULL;
2096
2097         rmb();
2098         be_dws_le_to_cpu(compl, sizeof(*compl));
2099
2100         if (adapter->be3_native)
2101                 be_parse_rx_compl_v1(compl, rxcp);
2102         else
2103                 be_parse_rx_compl_v0(compl, rxcp);
2104
2105         if (rxcp->ip_frag)
2106                 rxcp->l4_csum = 0;
2107
2108         if (rxcp->vlanf) {
2109                 /* In QNQ modes, if qnq bit is not set, then the packet was
2110                  * tagged only with the transparent outer vlan-tag and must
2111                  * not be treated as a vlan packet by host
2112                  */
2113                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
2114                         rxcp->vlanf = 0;
2115
2116                 if (!lancer_chip(adapter))
2117                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
2118
2119                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
2120                     !test_bit(rxcp->vlan_tag, adapter->vids))
2121                         rxcp->vlanf = 0;
2122         }
2123
2124         /* As the compl has been parsed, reset it; we wont touch it again */
2125         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
2126
2127         queue_tail_inc(&rxo->cq);
2128         return rxcp;
2129 }
2130
2131 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
2132 {
2133         u32 order = get_order(size);
2134
2135         if (order > 0)
2136                 gfp |= __GFP_COMP;
2137         return  alloc_pages(gfp, order);
2138 }
2139
2140 /*
2141  * Allocate a page, split it to fragments of size rx_frag_size and post as
2142  * receive buffers to BE
2143  */
2144 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
2145 {
2146         struct be_adapter *adapter = rxo->adapter;
2147         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
2148         struct be_queue_info *rxq = &rxo->q;
2149         struct page *pagep = NULL;
2150         struct device *dev = &adapter->pdev->dev;
2151         struct be_eth_rx_d *rxd;
2152         u64 page_dmaaddr = 0, frag_dmaaddr;
2153         u32 posted, page_offset = 0, notify = 0;
2154
2155         page_info = &rxo->page_info_tbl[rxq->head];
2156         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
2157                 if (!pagep) {
2158                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
2159                         if (unlikely(!pagep)) {
2160                                 rx_stats(rxo)->rx_post_fail++;
2161                                 break;
2162                         }
2163                         page_dmaaddr = dma_map_page(dev, pagep, 0,
2164                                                     adapter->big_page_size,
2165                                                     DMA_FROM_DEVICE);
2166                         if (dma_mapping_error(dev, page_dmaaddr)) {
2167                                 put_page(pagep);
2168                                 pagep = NULL;
2169                                 adapter->drv_stats.dma_map_errors++;
2170                                 break;
2171                         }
2172                         page_offset = 0;
2173                 } else {
2174                         get_page(pagep);
2175                         page_offset += rx_frag_size;
2176                 }
2177                 page_info->page_offset = page_offset;
2178                 page_info->page = pagep;
2179
2180                 rxd = queue_head_node(rxq);
2181                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2182                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2183                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2184
2185                 /* Any space left in the current big page for another frag? */
2186                 if ((page_offset + rx_frag_size + rx_frag_size) >
2187                                         adapter->big_page_size) {
2188                         pagep = NULL;
2189                         page_info->last_frag = true;
2190                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2191                 } else {
2192                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2193                 }
2194
2195                 prev_page_info = page_info;
2196                 queue_head_inc(rxq);
2197                 page_info = &rxo->page_info_tbl[rxq->head];
2198         }
2199
2200         /* Mark the last frag of a page when we break out of the above loop
2201          * with no more slots available in the RXQ
2202          */
2203         if (pagep) {
2204                 prev_page_info->last_frag = true;
2205                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2206         }
2207
2208         if (posted) {
2209                 atomic_add(posted, &rxq->used);
2210                 if (rxo->rx_post_starved)
2211                         rxo->rx_post_starved = false;
2212                 do {
2213                         notify = min(MAX_NUM_POST_ERX_DB, posted);
2214                         be_rxq_notify(adapter, rxq->id, notify);
2215                         posted -= notify;
2216                 } while (posted);
2217         } else if (atomic_read(&rxq->used) == 0) {
2218                 /* Let be_worker replenish when memory is available */
2219                 rxo->rx_post_starved = true;
2220         }
2221 }
2222
2223 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)
2224 {
2225         struct be_queue_info *tx_cq = &txo->cq;
2226         struct be_tx_compl_info *txcp = &txo->txcp;
2227         struct be_eth_tx_compl *compl = queue_tail_node(tx_cq);
2228
2229         if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2230                 return NULL;
2231
2232         /* Ensure load ordering of valid bit dword and other dwords below */
2233         rmb();
2234         be_dws_le_to_cpu(compl, sizeof(*compl));
2235
2236         txcp->status = GET_TX_COMPL_BITS(status, compl);
2237         txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl);
2238
2239         compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2240         queue_tail_inc(tx_cq);
2241         return txcp;
2242 }
2243
2244 static u16 be_tx_compl_process(struct be_adapter *adapter,
2245                                struct be_tx_obj *txo, u16 last_index)
2246 {
2247         struct sk_buff **sent_skbs = txo->sent_skb_list;
2248         struct be_queue_info *txq = &txo->q;
2249         u16 frag_index, num_wrbs = 0;
2250         struct sk_buff *skb = NULL;
2251         bool unmap_skb_hdr = false;
2252         struct be_eth_wrb *wrb;
2253
2254         do {
2255                 if (sent_skbs[txq->tail]) {
2256                         /* Free skb from prev req */
2257                         if (skb)
2258                                 dev_consume_skb_any(skb);
2259                         skb = sent_skbs[txq->tail];
2260                         sent_skbs[txq->tail] = NULL;
2261                         queue_tail_inc(txq);  /* skip hdr wrb */
2262                         num_wrbs++;
2263                         unmap_skb_hdr = true;
2264                 }
2265                 wrb = queue_tail_node(txq);
2266                 frag_index = txq->tail;
2267                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2268                               (unmap_skb_hdr && skb_headlen(skb)));
2269                 unmap_skb_hdr = false;
2270                 queue_tail_inc(txq);
2271                 num_wrbs++;
2272         } while (frag_index != last_index);
2273         dev_consume_skb_any(skb);
2274
2275         return num_wrbs;
2276 }
2277
2278 /* Return the number of events in the event queue */
2279 static inline int events_get(struct be_eq_obj *eqo)
2280 {
2281         struct be_eq_entry *eqe;
2282         int num = 0;
2283
2284         do {
2285                 eqe = queue_tail_node(&eqo->q);
2286                 if (eqe->evt == 0)
2287                         break;
2288
2289                 rmb();
2290                 eqe->evt = 0;
2291                 num++;
2292                 queue_tail_inc(&eqo->q);
2293         } while (true);
2294
2295         return num;
2296 }
2297
2298 /* Leaves the EQ is disarmed state */
2299 static void be_eq_clean(struct be_eq_obj *eqo)
2300 {
2301         int num = events_get(eqo);
2302
2303         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0);
2304 }
2305
2306 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2307 {
2308         struct be_rx_page_info *page_info;
2309         struct be_queue_info *rxq = &rxo->q;
2310         struct be_queue_info *rx_cq = &rxo->cq;
2311         struct be_rx_compl_info *rxcp;
2312         struct be_adapter *adapter = rxo->adapter;
2313         int flush_wait = 0;
2314
2315         /* Consume pending rx completions.
2316          * Wait for the flush completion (identified by zero num_rcvd)
2317          * to arrive. Notify CQ even when there are no more CQ entries
2318          * for HW to flush partially coalesced CQ entries.
2319          * In Lancer, there is no need to wait for flush compl.
2320          */
2321         for (;;) {
2322                 rxcp = be_rx_compl_get(rxo);
2323                 if (!rxcp) {
2324                         if (lancer_chip(adapter))
2325                                 break;
2326
2327                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2328                                 dev_warn(&adapter->pdev->dev,
2329                                          "did not receive flush compl\n");
2330                                 break;
2331                         }
2332                         be_cq_notify(adapter, rx_cq->id, true, 0);
2333                         mdelay(1);
2334                 } else {
2335                         be_rx_compl_discard(rxo, rxcp);
2336                         be_cq_notify(adapter, rx_cq->id, false, 1);
2337                         if (rxcp->num_rcvd == 0)
2338                                 break;
2339                 }
2340         }
2341
2342         /* After cleanup, leave the CQ in unarmed state */
2343         be_cq_notify(adapter, rx_cq->id, false, 0);
2344
2345         /* Then free posted rx buffers that were not used */
2346         while (atomic_read(&rxq->used) > 0) {
2347                 page_info = get_rx_page_info(rxo);
2348                 put_page(page_info->page);
2349                 memset(page_info, 0, sizeof(*page_info));
2350         }
2351         BUG_ON(atomic_read(&rxq->used));
2352         rxq->tail = 0;
2353         rxq->head = 0;
2354 }
2355
2356 static void be_tx_compl_clean(struct be_adapter *adapter)
2357 {
2358         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2359         struct device *dev = &adapter->pdev->dev;
2360         struct be_tx_compl_info *txcp;
2361         struct be_queue_info *txq;
2362         struct be_tx_obj *txo;
2363         int i, pending_txqs;
2364
2365         /* Stop polling for compls when HW has been silent for 10ms */
2366         do {
2367                 pending_txqs = adapter->num_tx_qs;
2368
2369                 for_all_tx_queues(adapter, txo, i) {
2370                         cmpl = 0;
2371                         num_wrbs = 0;
2372                         txq = &txo->q;
2373                         while ((txcp = be_tx_compl_get(txo))) {
2374                                 num_wrbs +=
2375                                         be_tx_compl_process(adapter, txo,
2376                                                             txcp->end_index);
2377                                 cmpl++;
2378                         }
2379                         if (cmpl) {
2380                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2381                                 atomic_sub(num_wrbs, &txq->used);
2382                                 timeo = 0;
2383                         }
2384                         if (!be_is_tx_compl_pending(txo))
2385                                 pending_txqs--;
2386                 }
2387
2388                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2389                         break;
2390
2391                 mdelay(1);
2392         } while (true);
2393
2394         /* Free enqueued TX that was never notified to HW */
2395         for_all_tx_queues(adapter, txo, i) {
2396                 txq = &txo->q;
2397
2398                 if (atomic_read(&txq->used)) {
2399                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2400                                  i, atomic_read(&txq->used));
2401                         notified_idx = txq->tail;
2402                         end_idx = txq->tail;
2403                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2404                                   txq->len);
2405                         /* Use the tx-compl process logic to handle requests
2406                          * that were not sent to the HW.
2407                          */
2408                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2409                         atomic_sub(num_wrbs, &txq->used);
2410                         BUG_ON(atomic_read(&txq->used));
2411                         txo->pend_wrb_cnt = 0;
2412                         /* Since hw was never notified of these requests,
2413                          * reset TXQ indices
2414                          */
2415                         txq->head = notified_idx;
2416                         txq->tail = notified_idx;
2417                 }
2418         }
2419 }
2420
2421 static void be_evt_queues_destroy(struct be_adapter *adapter)
2422 {
2423         struct be_eq_obj *eqo;
2424         int i;
2425
2426         for_all_evt_queues(adapter, eqo, i) {
2427                 if (eqo->q.created) {
2428                         be_eq_clean(eqo);
2429                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2430                         napi_hash_del(&eqo->napi);
2431                         netif_napi_del(&eqo->napi);
2432                 }
2433                 free_cpumask_var(eqo->affinity_mask);
2434                 be_queue_free(adapter, &eqo->q);
2435         }
2436 }
2437
2438 static int be_evt_queues_create(struct be_adapter *adapter)
2439 {
2440         struct be_queue_info *eq;
2441         struct be_eq_obj *eqo;
2442         struct be_aic_obj *aic;
2443         int i, rc;
2444
2445         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2446                                     adapter->cfg_num_qs);
2447
2448         for_all_evt_queues(adapter, eqo, i) {
2449                 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL))
2450                         return -ENOMEM;
2451                 cpumask_set_cpu_local_first(i, dev_to_node(&adapter->pdev->dev),
2452                                             eqo->affinity_mask);
2453
2454                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2455                                BE_NAPI_WEIGHT);
2456                 napi_hash_add(&eqo->napi);
2457                 aic = &adapter->aic_obj[i];
2458                 eqo->adapter = adapter;
2459                 eqo->idx = i;
2460                 aic->max_eqd = BE_MAX_EQD;
2461                 aic->enable = true;
2462
2463                 eq = &eqo->q;
2464                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2465                                     sizeof(struct be_eq_entry));
2466                 if (rc)
2467                         return rc;
2468
2469                 rc = be_cmd_eq_create(adapter, eqo);
2470                 if (rc)
2471                         return rc;
2472         }
2473         return 0;
2474 }
2475
2476 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2477 {
2478         struct be_queue_info *q;
2479
2480         q = &adapter->mcc_obj.q;
2481         if (q->created)
2482                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2483         be_queue_free(adapter, q);
2484
2485         q = &adapter->mcc_obj.cq;
2486         if (q->created)
2487                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2488         be_queue_free(adapter, q);
2489 }
2490
2491 /* Must be called only after TX qs are created as MCC shares TX EQ */
2492 static int be_mcc_queues_create(struct be_adapter *adapter)
2493 {
2494         struct be_queue_info *q, *cq;
2495
2496         cq = &adapter->mcc_obj.cq;
2497         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2498                            sizeof(struct be_mcc_compl)))
2499                 goto err;
2500
2501         /* Use the default EQ for MCC completions */
2502         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2503                 goto mcc_cq_free;
2504
2505         q = &adapter->mcc_obj.q;
2506         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2507                 goto mcc_cq_destroy;
2508
2509         if (be_cmd_mccq_create(adapter, q, cq))
2510                 goto mcc_q_free;
2511
2512         return 0;
2513
2514 mcc_q_free:
2515         be_queue_free(adapter, q);
2516 mcc_cq_destroy:
2517         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2518 mcc_cq_free:
2519         be_queue_free(adapter, cq);
2520 err:
2521         return -1;
2522 }
2523
2524 static void be_tx_queues_destroy(struct be_adapter *adapter)
2525 {
2526         struct be_queue_info *q;
2527         struct be_tx_obj *txo;
2528         u8 i;
2529
2530         for_all_tx_queues(adapter, txo, i) {
2531                 q = &txo->q;
2532                 if (q->created)
2533                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2534                 be_queue_free(adapter, q);
2535
2536                 q = &txo->cq;
2537                 if (q->created)
2538                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2539                 be_queue_free(adapter, q);
2540         }
2541 }
2542
2543 static int be_tx_qs_create(struct be_adapter *adapter)
2544 {
2545         struct be_queue_info *cq;
2546         struct be_tx_obj *txo;
2547         struct be_eq_obj *eqo;
2548         int status, i;
2549
2550         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2551
2552         for_all_tx_queues(adapter, txo, i) {
2553                 cq = &txo->cq;
2554                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2555                                         sizeof(struct be_eth_tx_compl));
2556                 if (status)
2557                         return status;
2558
2559                 u64_stats_init(&txo->stats.sync);
2560                 u64_stats_init(&txo->stats.sync_compl);
2561
2562                 /* If num_evt_qs is less than num_tx_qs, then more than
2563                  * one txq share an eq
2564                  */
2565                 eqo = &adapter->eq_obj[i % adapter->num_evt_qs];
2566                 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3);
2567                 if (status)
2568                         return status;
2569
2570                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2571                                         sizeof(struct be_eth_wrb));
2572                 if (status)
2573                         return status;
2574
2575                 status = be_cmd_txq_create(adapter, txo);
2576                 if (status)
2577                         return status;
2578
2579                 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask,
2580                                     eqo->idx);
2581         }
2582
2583         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2584                  adapter->num_tx_qs);
2585         return 0;
2586 }
2587
2588 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2589 {
2590         struct be_queue_info *q;
2591         struct be_rx_obj *rxo;
2592         int i;
2593
2594         for_all_rx_queues(adapter, rxo, i) {
2595                 q = &rxo->cq;
2596                 if (q->created)
2597                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2598                 be_queue_free(adapter, q);
2599         }
2600 }
2601
2602 static int be_rx_cqs_create(struct be_adapter *adapter)
2603 {
2604         struct be_queue_info *eq, *cq;
2605         struct be_rx_obj *rxo;
2606         int rc, i;
2607
2608         /* We can create as many RSS rings as there are EQs. */
2609         adapter->num_rss_qs = adapter->num_evt_qs;
2610
2611         /* We'll use RSS only if atleast 2 RSS rings are supported. */
2612         if (adapter->num_rss_qs <= 1)
2613                 adapter->num_rss_qs = 0;
2614
2615         adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq;
2616
2617         /* When the interface is not capable of RSS rings (and there is no
2618          * need to create a default RXQ) we'll still need one RXQ
2619          */
2620         if (adapter->num_rx_qs == 0)
2621                 adapter->num_rx_qs = 1;
2622
2623         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2624         for_all_rx_queues(adapter, rxo, i) {
2625                 rxo->adapter = adapter;
2626                 cq = &rxo->cq;
2627                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2628                                     sizeof(struct be_eth_rx_compl));
2629                 if (rc)
2630                         return rc;
2631
2632                 u64_stats_init(&rxo->stats.sync);
2633                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2634                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2635                 if (rc)
2636                         return rc;
2637         }
2638
2639         dev_info(&adapter->pdev->dev,
2640                  "created %d RX queue(s)\n", adapter->num_rx_qs);
2641         return 0;
2642 }
2643
2644 static irqreturn_t be_intx(int irq, void *dev)
2645 {
2646         struct be_eq_obj *eqo = dev;
2647         struct be_adapter *adapter = eqo->adapter;
2648         int num_evts = 0;
2649
2650         /* IRQ is not expected when NAPI is scheduled as the EQ
2651          * will not be armed.
2652          * But, this can happen on Lancer INTx where it takes
2653          * a while to de-assert INTx or in BE2 where occasionaly
2654          * an interrupt may be raised even when EQ is unarmed.
2655          * If NAPI is already scheduled, then counting & notifying
2656          * events will orphan them.
2657          */
2658         if (napi_schedule_prep(&eqo->napi)) {
2659                 num_evts = events_get(eqo);
2660                 __napi_schedule(&eqo->napi);
2661                 if (num_evts)
2662                         eqo->spurious_intr = 0;
2663         }
2664         be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0);
2665
2666         /* Return IRQ_HANDLED only for the the first spurious intr
2667          * after a valid intr to stop the kernel from branding
2668          * this irq as a bad one!
2669          */
2670         if (num_evts || eqo->spurious_intr++ == 0)
2671                 return IRQ_HANDLED;
2672         else
2673                 return IRQ_NONE;
2674 }
2675
2676 static irqreturn_t be_msix(int irq, void *dev)
2677 {
2678         struct be_eq_obj *eqo = dev;
2679
2680         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
2681         napi_schedule(&eqo->napi);
2682         return IRQ_HANDLED;
2683 }
2684
2685 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2686 {
2687         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2688 }
2689
2690 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2691                          int budget, int polling)
2692 {
2693         struct be_adapter *adapter = rxo->adapter;
2694         struct be_queue_info *rx_cq = &rxo->cq;
2695         struct be_rx_compl_info *rxcp;
2696         u32 work_done;
2697         u32 frags_consumed = 0;
2698
2699         for (work_done = 0; work_done < budget; work_done++) {
2700                 rxcp = be_rx_compl_get(rxo);
2701                 if (!rxcp)
2702                         break;
2703
2704                 /* Is it a flush compl that has no data */
2705                 if (unlikely(rxcp->num_rcvd == 0))
2706                         goto loop_continue;
2707
2708                 /* Discard compl with partial DMA Lancer B0 */
2709                 if (unlikely(!rxcp->pkt_size)) {
2710                         be_rx_compl_discard(rxo, rxcp);
2711                         goto loop_continue;
2712                 }
2713
2714                 /* On BE drop pkts that arrive due to imperfect filtering in
2715                  * promiscuous mode on some skews
2716                  */
2717                 if (unlikely(rxcp->port != adapter->port_num &&
2718                              !lancer_chip(adapter))) {
2719                         be_rx_compl_discard(rxo, rxcp);
2720                         goto loop_continue;
2721                 }
2722
2723                 /* Don't do gro when we're busy_polling */
2724                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2725                         be_rx_compl_process_gro(rxo, napi, rxcp);
2726                 else
2727                         be_rx_compl_process(rxo, napi, rxcp);
2728
2729 loop_continue:
2730                 frags_consumed += rxcp->num_rcvd;
2731                 be_rx_stats_update(rxo, rxcp);
2732         }
2733
2734         if (work_done) {
2735                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2736
2737                 /* When an rx-obj gets into post_starved state, just
2738                  * let be_worker do the posting.
2739                  */
2740                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2741                     !rxo->rx_post_starved)
2742                         be_post_rx_frags(rxo, GFP_ATOMIC,
2743                                          max_t(u32, MAX_RX_POST,
2744                                                frags_consumed));
2745         }
2746
2747         return work_done;
2748 }
2749
2750 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
2751 {
2752         switch (status) {
2753         case BE_TX_COMP_HDR_PARSE_ERR:
2754                 tx_stats(txo)->tx_hdr_parse_err++;
2755                 break;
2756         case BE_TX_COMP_NDMA_ERR:
2757                 tx_stats(txo)->tx_dma_err++;
2758                 break;
2759         case BE_TX_COMP_ACL_ERR:
2760                 tx_stats(txo)->tx_spoof_check_err++;
2761                 break;
2762         }
2763 }
2764
2765 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
2766 {
2767         switch (status) {
2768         case LANCER_TX_COMP_LSO_ERR:
2769                 tx_stats(txo)->tx_tso_err++;
2770                 break;
2771         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2772         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2773                 tx_stats(txo)->tx_spoof_check_err++;
2774                 break;
2775         case LANCER_TX_COMP_QINQ_ERR:
2776                 tx_stats(txo)->tx_qinq_err++;
2777                 break;
2778         case LANCER_TX_COMP_PARITY_ERR:
2779                 tx_stats(txo)->tx_internal_parity_err++;
2780                 break;
2781         case LANCER_TX_COMP_DMA_ERR:
2782                 tx_stats(txo)->tx_dma_err++;
2783                 break;
2784         }
2785 }
2786
2787 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2788                           int idx)
2789 {
2790         int num_wrbs = 0, work_done = 0;
2791         struct be_tx_compl_info *txcp;
2792
2793         while ((txcp = be_tx_compl_get(txo))) {
2794                 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);
2795                 work_done++;
2796
2797                 if (txcp->status) {
2798                         if (lancer_chip(adapter))
2799                                 lancer_update_tx_err(txo, txcp->status);
2800                         else
2801                                 be_update_tx_err(txo, txcp->status);
2802                 }
2803         }
2804
2805         if (work_done) {
2806                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2807                 atomic_sub(num_wrbs, &txo->q.used);
2808
2809                 /* As Tx wrbs have been freed up, wake up netdev queue
2810                  * if it was stopped due to lack of tx wrbs.  */
2811                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2812                     be_can_txq_wake(txo)) {
2813                         netif_wake_subqueue(adapter->netdev, idx);
2814                 }
2815
2816                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2817                 tx_stats(txo)->tx_compl += work_done;
2818                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2819         }
2820 }
2821
2822 #ifdef CONFIG_NET_RX_BUSY_POLL
2823 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2824 {
2825         bool status = true;
2826
2827         spin_lock(&eqo->lock); /* BH is already disabled */
2828         if (eqo->state & BE_EQ_LOCKED) {
2829                 WARN_ON(eqo->state & BE_EQ_NAPI);
2830                 eqo->state |= BE_EQ_NAPI_YIELD;
2831                 status = false;
2832         } else {
2833                 eqo->state = BE_EQ_NAPI;
2834         }
2835         spin_unlock(&eqo->lock);
2836         return status;
2837 }
2838
2839 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2840 {
2841         spin_lock(&eqo->lock); /* BH is already disabled */
2842
2843         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2844         eqo->state = BE_EQ_IDLE;
2845
2846         spin_unlock(&eqo->lock);
2847 }
2848
2849 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2850 {
2851         bool status = true;
2852
2853         spin_lock_bh(&eqo->lock);
2854         if (eqo->state & BE_EQ_LOCKED) {
2855                 eqo->state |= BE_EQ_POLL_YIELD;
2856                 status = false;
2857         } else {
2858                 eqo->state |= BE_EQ_POLL;
2859         }
2860         spin_unlock_bh(&eqo->lock);
2861         return status;
2862 }
2863
2864 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2865 {
2866         spin_lock_bh(&eqo->lock);
2867
2868         WARN_ON(eqo->state & (BE_EQ_NAPI));
2869         eqo->state = BE_EQ_IDLE;
2870
2871         spin_unlock_bh(&eqo->lock);
2872 }
2873
2874 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2875 {
2876         spin_lock_init(&eqo->lock);
2877         eqo->state = BE_EQ_IDLE;
2878 }
2879
2880 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2881 {
2882         local_bh_disable();
2883
2884         /* It's enough to just acquire napi lock on the eqo to stop
2885          * be_busy_poll() from processing any queueus.
2886          */
2887         while (!be_lock_napi(eqo))
2888                 mdelay(1);
2889
2890         local_bh_enable();
2891 }
2892
2893 #else /* CONFIG_NET_RX_BUSY_POLL */
2894
2895 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2896 {
2897         return true;
2898 }
2899
2900 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2901 {
2902 }
2903
2904 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2905 {
2906         return false;
2907 }
2908
2909 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2910 {
2911 }
2912
2913 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2914 {
2915 }
2916
2917 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2918 {
2919 }
2920 #endif /* CONFIG_NET_RX_BUSY_POLL */
2921
2922 int be_poll(struct napi_struct *napi, int budget)
2923 {
2924         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2925         struct be_adapter *adapter = eqo->adapter;
2926         int max_work = 0, work, i, num_evts;
2927         struct be_rx_obj *rxo;
2928         struct be_tx_obj *txo;
2929         u32 mult_enc = 0;
2930
2931         num_evts = events_get(eqo);
2932
2933         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2934                 be_process_tx(adapter, txo, i);
2935
2936         if (be_lock_napi(eqo)) {
2937                 /* This loop will iterate twice for EQ0 in which
2938                  * completions of the last RXQ (default one) are also processed
2939                  * For other EQs the loop iterates only once
2940                  */
2941                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2942                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2943                         max_work = max(work, max_work);
2944                 }
2945                 be_unlock_napi(eqo);
2946         } else {
2947                 max_work = budget;
2948         }
2949
2950         if (is_mcc_eqo(eqo))
2951                 be_process_mcc(adapter);
2952
2953         if (max_work < budget) {
2954                 napi_complete(napi);
2955
2956                 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt
2957                  * delay via a delay multiplier encoding value
2958                  */
2959                 if (skyhawk_chip(adapter))
2960                         mult_enc = be_get_eq_delay_mult_enc(eqo);
2961
2962                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts,
2963                              mult_enc);
2964         } else {
2965                 /* As we'll continue in polling mode, count and clear events */
2966                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0);
2967         }
2968         return max_work;
2969 }
2970
2971 #ifdef CONFIG_NET_RX_BUSY_POLL
2972 static int be_busy_poll(struct napi_struct *napi)
2973 {
2974         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2975         struct be_adapter *adapter = eqo->adapter;
2976         struct be_rx_obj *rxo;
2977         int i, work = 0;
2978
2979         if (!be_lock_busy_poll(eqo))
2980                 return LL_FLUSH_BUSY;
2981
2982         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2983                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2984                 if (work)
2985                         break;
2986         }
2987
2988         be_unlock_busy_poll(eqo);
2989         return work;
2990 }
2991 #endif
2992
2993 void be_detect_error(struct be_adapter *adapter)
2994 {
2995         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2996         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2997         u32 i;
2998         bool error_detected = false;
2999         struct device *dev = &adapter->pdev->dev;
3000         struct net_device *netdev = adapter->netdev;
3001
3002         if (be_hw_error(adapter))
3003                 return;
3004
3005         if (lancer_chip(adapter)) {
3006                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
3007                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
3008                         sliport_err1 = ioread32(adapter->db +
3009                                                 SLIPORT_ERROR1_OFFSET);
3010                         sliport_err2 = ioread32(adapter->db +
3011                                                 SLIPORT_ERROR2_OFFSET);
3012                         adapter->hw_error = true;
3013                         error_detected = true;
3014                         /* Do not log error messages if its a FW reset */
3015                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
3016                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
3017                                 dev_info(dev, "Firmware update in progress\n");
3018                         } else {
3019                                 dev_err(dev, "Error detected in the card\n");
3020                                 dev_err(dev, "ERR: sliport status 0x%x\n",
3021                                         sliport_status);
3022                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
3023                                         sliport_err1);
3024                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
3025                                         sliport_err2);
3026                         }
3027                 }
3028         } else {
3029                 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW);
3030                 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH);
3031                 ue_lo_mask = ioread32(adapter->pcicfg +
3032                                       PCICFG_UE_STATUS_LOW_MASK);
3033                 ue_hi_mask = ioread32(adapter->pcicfg +
3034                                       PCICFG_UE_STATUS_HI_MASK);
3035
3036                 ue_lo = (ue_lo & ~ue_lo_mask);
3037                 ue_hi = (ue_hi & ~ue_hi_mask);
3038
3039                 /* On certain platforms BE hardware can indicate spurious UEs.
3040                  * Allow HW to stop working completely in case of a real UE.
3041                  * Hence not setting the hw_error for UE detection.
3042                  */
3043
3044                 if (ue_lo || ue_hi) {
3045                         error_detected = true;
3046                         dev_err(dev,
3047                                 "Unrecoverable Error detected in the adapter");
3048                         dev_err(dev, "Please reboot server to recover");
3049                         if (skyhawk_chip(adapter))
3050                                 adapter->hw_error = true;
3051                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
3052                                 if (ue_lo & 1)
3053                                         dev_err(dev, "UE: %s bit set\n",
3054                                                 ue_status_low_desc[i]);
3055                         }
3056                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
3057                                 if (ue_hi & 1)
3058                                         dev_err(dev, "UE: %s bit set\n",
3059                                                 ue_status_hi_desc[i]);
3060                         }
3061                 }
3062         }
3063         if (error_detected)
3064                 netif_carrier_off(netdev);
3065 }
3066
3067 static void be_msix_disable(struct be_adapter *adapter)
3068 {
3069         if (msix_enabled(adapter)) {
3070                 pci_disable_msix(adapter->pdev);
3071                 adapter->num_msix_vec = 0;
3072                 adapter->num_msix_roce_vec = 0;
3073         }
3074 }
3075
3076 static int be_msix_enable(struct be_adapter *adapter)
3077 {
3078         int i, num_vec;
3079         struct device *dev = &adapter->pdev->dev;
3080
3081         /* If RoCE is supported, program the max number of NIC vectors that
3082          * may be configured via set-channels, along with vectors needed for
3083          * RoCe. Else, just program the number we'll use initially.
3084          */
3085         if (be_roce_supported(adapter))
3086                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
3087                                 2 * num_online_cpus());
3088         else
3089                 num_vec = adapter->cfg_num_qs;
3090
3091         for (i = 0; i < num_vec; i++)
3092                 adapter->msix_entries[i].entry = i;
3093
3094         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
3095                                         MIN_MSIX_VECTORS, num_vec);
3096         if (num_vec < 0)
3097                 goto fail;
3098
3099         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
3100                 adapter->num_msix_roce_vec = num_vec / 2;
3101                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
3102                          adapter->num_msix_roce_vec);
3103         }
3104
3105         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
3106
3107         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
3108                  adapter->num_msix_vec);
3109         return 0;
3110
3111 fail:
3112         dev_warn(dev, "MSIx enable failed\n");
3113
3114         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
3115         if (!be_physfn(adapter))
3116                 return num_vec;
3117         return 0;
3118 }
3119
3120 static inline int be_msix_vec_get(struct be_adapter *adapter,
3121                                   struct be_eq_obj *eqo)
3122 {
3123         return adapter->msix_entries[eqo->msix_idx].vector;
3124 }
3125
3126 static int be_msix_register(struct be_adapter *adapter)
3127 {
3128         struct net_device *netdev = adapter->netdev;
3129         struct be_eq_obj *eqo;
3130         int status, i, vec;
3131
3132         for_all_evt_queues(adapter, eqo, i) {
3133                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
3134                 vec = be_msix_vec_get(adapter, eqo);
3135                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
3136                 if (status)
3137                         goto err_msix;
3138
3139                 irq_set_affinity_hint(vec, eqo->affinity_mask);
3140         }
3141
3142         return 0;
3143 err_msix:
3144         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
3145                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
3146         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
3147                  status);
3148         be_msix_disable(adapter);
3149         return status;
3150 }
3151
3152 static int be_irq_register(struct be_adapter *adapter)
3153 {
3154         struct net_device *netdev = adapter->netdev;
3155         int status;
3156
3157         if (msix_enabled(adapter)) {
3158                 status = be_msix_register(adapter);
3159                 if (status == 0)
3160                         goto done;
3161                 /* INTx is not supported for VF */
3162                 if (!be_physfn(adapter))
3163                         return status;
3164         }
3165
3166         /* INTx: only the first EQ is used */
3167         netdev->irq = adapter->pdev->irq;
3168         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
3169                              &adapter->eq_obj[0]);
3170         if (status) {
3171                 dev_err(&adapter->pdev->dev,
3172                         "INTx request IRQ failed - err %d\n", status);
3173                 return status;
3174         }
3175 done:
3176         adapter->isr_registered = true;
3177         return 0;
3178 }
3179
3180 static void be_irq_unregister(struct be_adapter *adapter)
3181 {
3182         struct net_device *netdev = adapter->netdev;
3183         struct be_eq_obj *eqo;
3184         int i, vec;
3185
3186         if (!adapter->isr_registered)
3187                 return;
3188
3189         /* INTx */
3190         if (!msix_enabled(adapter)) {
3191                 free_irq(netdev->irq, &adapter->eq_obj[0]);
3192                 goto done;
3193         }
3194
3195         /* MSIx */
3196         for_all_evt_queues(adapter, eqo, i) {
3197                 vec = be_msix_vec_get(adapter, eqo);
3198                 irq_set_affinity_hint(vec, NULL);
3199                 free_irq(vec, eqo);
3200         }
3201
3202 done:
3203         adapter->isr_registered = false;
3204 }
3205
3206 static void be_rx_qs_destroy(struct be_adapter *adapter)
3207 {
3208         struct be_queue_info *q;
3209         struct be_rx_obj *rxo;
3210         int i;
3211
3212         for_all_rx_queues(adapter, rxo, i) {
3213                 q = &rxo->q;
3214                 if (q->created) {
3215                         be_cmd_rxq_destroy(adapter, q);
3216                         be_rx_cq_clean(rxo);
3217                 }
3218                 be_queue_free(adapter, q);
3219         }
3220 }
3221
3222 static int be_close(struct net_device *netdev)
3223 {
3224         struct be_adapter *adapter = netdev_priv(netdev);
3225         struct be_eq_obj *eqo;
3226         int i;
3227
3228         /* This protection is needed as be_close() may be called even when the
3229          * adapter is in cleared state (after eeh perm failure)
3230          */
3231         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3232                 return 0;
3233
3234         be_roce_dev_close(adapter);
3235
3236         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3237                 for_all_evt_queues(adapter, eqo, i) {
3238                         napi_disable(&eqo->napi);
3239                         be_disable_busy_poll(eqo);
3240                 }
3241                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3242         }
3243
3244         be_async_mcc_disable(adapter);
3245
3246         /* Wait for all pending tx completions to arrive so that
3247          * all tx skbs are freed.
3248          */
3249         netif_tx_disable(netdev);
3250         be_tx_compl_clean(adapter);
3251
3252         be_rx_qs_destroy(adapter);
3253         be_clear_uc_list(adapter);
3254
3255         for_all_evt_queues(adapter, eqo, i) {
3256                 if (msix_enabled(adapter))
3257                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3258                 else
3259                         synchronize_irq(netdev->irq);
3260                 be_eq_clean(eqo);
3261         }
3262
3263         be_irq_unregister(adapter);
3264
3265         return 0;
3266 }
3267
3268 static int be_rx_qs_create(struct be_adapter *adapter)
3269 {
3270         struct rss_info *rss = &adapter->rss_info;
3271         u8 rss_key[RSS_HASH_KEY_LEN];
3272         struct be_rx_obj *rxo;
3273         int rc, i, j;
3274
3275         for_all_rx_queues(adapter, rxo, i) {
3276                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3277                                     sizeof(struct be_eth_rx_d));
3278                 if (rc)
3279                         return rc;
3280         }
3281
3282         if (adapter->need_def_rxq || !adapter->num_rss_qs) {
3283                 rxo = default_rxo(adapter);
3284                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3285                                        rx_frag_size, adapter->if_handle,
3286                                        false, &rxo->rss_id);
3287                 if (rc)
3288                         return rc;
3289         }
3290
3291         for_all_rss_queues(adapter, rxo, i) {
3292                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3293                                        rx_frag_size, adapter->if_handle,
3294                                        true, &rxo->rss_id);
3295                 if (rc)
3296                         return rc;
3297         }
3298
3299         if (be_multi_rxq(adapter)) {
3300                 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) {
3301                         for_all_rss_queues(adapter, rxo, i) {
3302                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3303                                         break;
3304                                 rss->rsstable[j + i] = rxo->rss_id;
3305                                 rss->rss_queue[j + i] = i;
3306                         }
3307                 }
3308                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3309                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3310
3311                 if (!BEx_chip(adapter))
3312                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3313                                 RSS_ENABLE_UDP_IPV6;
3314         } else {
3315                 /* Disable RSS, if only default RX Q is created */
3316                 rss->rss_flags = RSS_ENABLE_NONE;
3317         }
3318
3319         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3320         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3321                                128, rss_key);
3322         if (rc) {
3323                 rss->rss_flags = RSS_ENABLE_NONE;
3324                 return rc;
3325         }
3326
3327         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3328
3329         /* Post 1 less than RXQ-len to avoid head being equal to tail,
3330          * which is a queue empty condition
3331          */
3332         for_all_rx_queues(adapter, rxo, i)
3333                 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1);
3334
3335         return 0;
3336 }
3337
3338 static int be_open(struct net_device *netdev)
3339 {
3340         struct be_adapter *adapter = netdev_priv(netdev);
3341         struct be_eq_obj *eqo;
3342         struct be_rx_obj *rxo;
3343         struct be_tx_obj *txo;
3344         u8 link_status;
3345         int status, i;
3346
3347         status = be_rx_qs_create(adapter);
3348         if (status)
3349                 goto err;
3350
3351         status = be_irq_register(adapter);
3352         if (status)
3353                 goto err;
3354
3355         for_all_rx_queues(adapter, rxo, i)
3356                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3357
3358         for_all_tx_queues(adapter, txo, i)
3359                 be_cq_notify(adapter, txo->cq.id, true, 0);
3360
3361         be_async_mcc_enable(adapter);
3362
3363         for_all_evt_queues(adapter, eqo, i) {
3364                 napi_enable(&eqo->napi);
3365                 be_enable_busy_poll(eqo);
3366                 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0);
3367         }
3368         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3369
3370         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3371         if (!status)
3372                 be_link_status_update(adapter, link_status);
3373
3374         netif_tx_start_all_queues(netdev);
3375         be_roce_dev_open(adapter);
3376
3377 #ifdef CONFIG_BE2NET_VXLAN
3378         if (skyhawk_chip(adapter))
3379                 vxlan_get_rx_port(netdev);
3380 #endif
3381
3382         return 0;
3383 err:
3384         be_close(adapter->netdev);
3385         return -EIO;
3386 }
3387
3388 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3389 {
3390         struct be_dma_mem cmd;
3391         int status = 0;
3392         u8 mac[ETH_ALEN];
3393
3394         eth_zero_addr(mac);
3395
3396         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3397         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3398                                      GFP_KERNEL);
3399         if (!cmd.va)
3400                 return -ENOMEM;
3401
3402         if (enable) {
3403                 status = pci_write_config_dword(adapter->pdev,
3404                                                 PCICFG_PM_CONTROL_OFFSET,
3405                                                 PCICFG_PM_CONTROL_MASK);
3406                 if (status) {
3407                         dev_err(&adapter->pdev->dev,
3408                                 "Could not enable Wake-on-lan\n");
3409                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3410                                           cmd.dma);
3411                         return status;
3412                 }
3413                 status = be_cmd_enable_magic_wol(adapter,
3414                                                  adapter->netdev->dev_addr,
3415                                                  &cmd);
3416                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3417                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3418         } else {
3419                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3420                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3421                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3422         }
3423
3424         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3425         return status;
3426 }
3427
3428 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3429 {
3430         u32 addr;
3431
3432         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3433
3434         mac[5] = (u8)(addr & 0xFF);
3435         mac[4] = (u8)((addr >> 8) & 0xFF);
3436         mac[3] = (u8)((addr >> 16) & 0xFF);
3437         /* Use the OUI from the current MAC address */
3438         memcpy(mac, adapter->netdev->dev_addr, 3);
3439 }
3440
3441 /*
3442  * Generate a seed MAC address from the PF MAC Address using jhash.
3443  * MAC Address for VFs are assigned incrementally starting from the seed.
3444  * These addresses are programmed in the ASIC by the PF and the VF driver
3445  * queries for the MAC address during its probe.
3446  */
3447 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3448 {
3449         u32 vf;
3450         int status = 0;
3451         u8 mac[ETH_ALEN];
3452         struct be_vf_cfg *vf_cfg;
3453
3454         be_vf_eth_addr_generate(adapter, mac);
3455
3456         for_all_vfs(adapter, vf_cfg, vf) {
3457                 if (BEx_chip(adapter))
3458                         status = be_cmd_pmac_add(adapter, mac,
3459                                                  vf_cfg->if_handle,
3460                                                  &vf_cfg->pmac_id, vf + 1);
3461                 else
3462                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3463                                                 vf + 1);
3464
3465                 if (status)
3466                         dev_err(&adapter->pdev->dev,
3467                                 "Mac address assignment failed for VF %d\n",
3468                                 vf);
3469                 else
3470                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3471
3472                 mac[5] += 1;
3473         }
3474         return status;
3475 }
3476
3477 static int be_vfs_mac_query(struct be_adapter *adapter)
3478 {
3479         int status, vf;
3480         u8 mac[ETH_ALEN];
3481         struct be_vf_cfg *vf_cfg;
3482
3483         for_all_vfs(adapter, vf_cfg, vf) {
3484                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3485                                                mac, vf_cfg->if_handle,
3486                                                false, vf+1);
3487                 if (status)
3488                         return status;
3489                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3490         }
3491         return 0;
3492 }
3493
3494 static void be_vf_clear(struct be_adapter *adapter)
3495 {
3496         struct be_vf_cfg *vf_cfg;
3497         u32 vf;
3498
3499         if (pci_vfs_assigned(adapter->pdev)) {
3500                 dev_warn(&adapter->pdev->dev,
3501                          "VFs are assigned to VMs: not disabling VFs\n");
3502                 goto done;
3503         }
3504
3505         pci_disable_sriov(adapter->pdev);
3506
3507         for_all_vfs(adapter, vf_cfg, vf) {
3508                 if (BEx_chip(adapter))
3509                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3510                                         vf_cfg->pmac_id, vf + 1);
3511                 else
3512                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3513                                        vf + 1);
3514
3515                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3516         }
3517 done:
3518         kfree(adapter->vf_cfg);
3519         adapter->num_vfs = 0;
3520         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3521 }
3522
3523 static void be_clear_queues(struct be_adapter *adapter)
3524 {
3525         be_mcc_queues_destroy(adapter);
3526         be_rx_cqs_destroy(adapter);
3527         be_tx_queues_destroy(adapter);
3528         be_evt_queues_destroy(adapter);
3529 }
3530
3531 static void be_cancel_worker(struct be_adapter *adapter)
3532 {
3533         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3534                 cancel_delayed_work_sync(&adapter->work);
3535                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3536         }
3537 }
3538
3539 static void be_cancel_err_detection(struct be_adapter *adapter)
3540 {
3541         if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) {
3542                 cancel_delayed_work_sync(&adapter->be_err_detection_work);
3543                 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED;
3544         }
3545 }
3546
3547 static void be_mac_clear(struct be_adapter *adapter)
3548 {
3549         if (adapter->pmac_id) {
3550                 be_cmd_pmac_del(adapter, adapter->if_handle,
3551                                 adapter->pmac_id[0], 0);
3552                 kfree(adapter->pmac_id);
3553                 adapter->pmac_id = NULL;
3554         }
3555 }
3556
3557 #ifdef CONFIG_BE2NET_VXLAN
3558 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3559 {
3560         struct net_device *netdev = adapter->netdev;
3561
3562         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3563                 be_cmd_manage_iface(adapter, adapter->if_handle,
3564                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3565
3566         if (adapter->vxlan_port)
3567                 be_cmd_set_vxlan_port(adapter, 0);
3568
3569         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3570         adapter->vxlan_port = 0;
3571
3572         netdev->hw_enc_features = 0;
3573         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3574         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3575 }
3576 #endif
3577
3578 static u16 be_calculate_vf_qs(struct be_adapter *adapter, u16 num_vfs)
3579 {
3580         struct be_resources res = adapter->pool_res;
3581         u16 num_vf_qs = 1;
3582
3583         /* Distribute the queue resources equally among the PF and it's VFs
3584          * Do not distribute queue resources in multi-channel configuration.
3585          */
3586         if (num_vfs && !be_is_mc(adapter)) {
3587                 /* If number of VFs requested is 8 less than max supported,
3588                  * assign 8 queue pairs to the PF and divide the remaining
3589                  * resources evenly among the VFs
3590                  */
3591                 if (num_vfs < (be_max_vfs(adapter) - 8))
3592                         num_vf_qs = (res.max_rss_qs - 8) / num_vfs;
3593                 else
3594                         num_vf_qs = res.max_rss_qs / num_vfs;
3595
3596                 /* Skyhawk-R chip supports only MAX_RSS_IFACES RSS capable
3597                  * interfaces per port. Provide RSS on VFs, only if number
3598                  * of VFs requested is less than MAX_RSS_IFACES limit.
3599                  */
3600                 if (num_vfs >= MAX_RSS_IFACES)
3601                         num_vf_qs = 1;
3602         }
3603         return num_vf_qs;
3604 }
3605
3606 static int be_clear(struct be_adapter *adapter)
3607 {
3608         struct pci_dev *pdev = adapter->pdev;
3609         u16 num_vf_qs;
3610
3611         be_cancel_worker(adapter);
3612
3613         if (sriov_enabled(adapter))
3614                 be_vf_clear(adapter);
3615
3616         /* Re-configure FW to distribute resources evenly across max-supported
3617          * number of VFs, only when VFs are not already enabled.
3618          */
3619         if (skyhawk_chip(adapter) && be_physfn(adapter) &&
3620             !pci_vfs_assigned(pdev)) {
3621                 num_vf_qs = be_calculate_vf_qs(adapter,
3622                                                pci_sriov_get_totalvfs(pdev));
3623                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3624                                         pci_sriov_get_totalvfs(pdev),
3625                                         num_vf_qs);
3626         }
3627
3628 #ifdef CONFIG_BE2NET_VXLAN
3629         be_disable_vxlan_offloads(adapter);
3630 #endif
3631         /* delete the primary mac along with the uc-mac list */
3632         be_mac_clear(adapter);
3633
3634         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3635
3636         be_clear_queues(adapter);
3637
3638         be_msix_disable(adapter);
3639         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3640         return 0;
3641 }
3642
3643 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3644                         u32 cap_flags, u32 vf)
3645 {
3646         u32 en_flags;
3647
3648         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3649                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3650                    BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS;
3651
3652         en_flags &= cap_flags;
3653
3654         return be_cmd_if_create(adapter, cap_flags, en_flags, if_handle, vf);
3655 }
3656
3657 static int be_vfs_if_create(struct be_adapter *adapter)
3658 {
3659         struct be_resources res = {0};
3660         struct be_vf_cfg *vf_cfg;
3661         u32 cap_flags, vf;
3662         int status;
3663
3664         /* If a FW profile exists, then cap_flags are updated */
3665         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3666                     BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3667
3668         for_all_vfs(adapter, vf_cfg, vf) {
3669                 if (!BE3_chip(adapter)) {
3670                         status = be_cmd_get_profile_config(adapter, &res,
3671                                                            RESOURCE_LIMITS,
3672                                                            vf + 1);
3673                         if (!status) {
3674                                 cap_flags = res.if_cap_flags;
3675                                 /* Prevent VFs from enabling VLAN promiscuous
3676                                  * mode
3677                                  */
3678                                 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
3679                         }
3680                 }
3681
3682                 status = be_if_create(adapter, &vf_cfg->if_handle,
3683                                       cap_flags, vf + 1);
3684                 if (status)
3685                         return status;
3686         }
3687
3688         return 0;
3689 }
3690
3691 static int be_vf_setup_init(struct be_adapter *adapter)
3692 {
3693         struct be_vf_cfg *vf_cfg;
3694         int vf;
3695
3696         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3697                                   GFP_KERNEL);
3698         if (!adapter->vf_cfg)
3699                 return -ENOMEM;
3700
3701         for_all_vfs(adapter, vf_cfg, vf) {
3702                 vf_cfg->if_handle = -1;
3703                 vf_cfg->pmac_id = -1;
3704         }
3705         return 0;
3706 }
3707
3708 static int be_vf_setup(struct be_adapter *adapter)
3709 {
3710         struct device *dev = &adapter->pdev->dev;
3711         struct be_vf_cfg *vf_cfg;
3712         int status, old_vfs, vf;
3713         bool spoofchk;
3714
3715         old_vfs = pci_num_vf(adapter->pdev);
3716
3717         status = be_vf_setup_init(adapter);
3718         if (status)
3719                 goto err;
3720
3721         if (old_vfs) {
3722                 for_all_vfs(adapter, vf_cfg, vf) {
3723                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3724                         if (status)
3725                                 goto err;
3726                 }
3727
3728                 status = be_vfs_mac_query(adapter);
3729                 if (status)
3730                         goto err;
3731         } else {
3732                 status = be_vfs_if_create(adapter);
3733                 if (status)
3734                         goto err;
3735
3736                 status = be_vf_eth_addr_config(adapter);
3737                 if (status)
3738                         goto err;
3739         }
3740
3741         for_all_vfs(adapter, vf_cfg, vf) {
3742                 /* Allow VFs to programs MAC/VLAN filters */
3743                 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
3744                                                   vf + 1);
3745                 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
3746                         status = be_cmd_set_fn_privileges(adapter,
3747                                                           vf_cfg->privileges |
3748                                                           BE_PRIV_FILTMGMT,
3749                                                           vf + 1);
3750                         if (!status) {
3751                                 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
3752                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3753                                          vf);
3754                         }
3755                 }
3756
3757                 /* Allow full available bandwidth */
3758                 if (!old_vfs)
3759                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3760
3761                 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1,
3762                                                vf_cfg->if_handle, NULL,
3763                                                &spoofchk);
3764                 if (!status)
3765                         vf_cfg->spoofchk = spoofchk;
3766
3767                 if (!old_vfs) {
3768                         be_cmd_enable_vf(adapter, vf + 1);
3769                         be_cmd_set_logical_link_config(adapter,
3770                                                        IFLA_VF_LINK_STATE_AUTO,
3771                                                        vf+1);
3772                 }
3773         }
3774
3775         if (!old_vfs) {
3776                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3777                 if (status) {
3778                         dev_err(dev, "SRIOV enable failed\n");
3779                         adapter->num_vfs = 0;
3780                         goto err;
3781                 }
3782         }
3783
3784         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3785         return 0;
3786 err:
3787         dev_err(dev, "VF setup failed\n");
3788         be_vf_clear(adapter);
3789         return status;
3790 }
3791
3792 /* Converting function_mode bits on BE3 to SH mc_type enums */
3793
3794 static u8 be_convert_mc_type(u32 function_mode)
3795 {
3796         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3797                 return vNIC1;
3798         else if (function_mode & QNQ_MODE)
3799                 return FLEX10;
3800         else if (function_mode & VNIC_MODE)
3801                 return vNIC2;
3802         else if (function_mode & UMC_ENABLED)
3803                 return UMC;
3804         else
3805                 return MC_NONE;
3806 }
3807
3808 /* On BE2/BE3 FW does not suggest the supported limits */
3809 static void BEx_get_resources(struct be_adapter *adapter,
3810                               struct be_resources *res)
3811 {
3812         bool use_sriov = adapter->num_vfs ? 1 : 0;
3813
3814         if (be_physfn(adapter))
3815                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3816         else
3817                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3818
3819         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3820
3821         if (be_is_mc(adapter)) {
3822                 /* Assuming that there are 4 channels per port,
3823                  * when multi-channel is enabled
3824                  */
3825                 if (be_is_qnq_mode(adapter))
3826                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3827                 else
3828                         /* In a non-qnq multichannel mode, the pvid
3829                          * takes up one vlan entry
3830                          */
3831                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3832         } else {
3833                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3834         }
3835
3836         res->max_mcast_mac = BE_MAX_MC;
3837
3838         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3839          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3840          *    *only* if it is RSS-capable.
3841          */
3842         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3843             !be_physfn(adapter) || (be_is_mc(adapter) &&
3844             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3845                 res->max_tx_qs = 1;
3846         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3847                 struct be_resources super_nic_res = {0};
3848
3849                 /* On a SuperNIC profile, the driver needs to use the
3850                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3851                  */
3852                 be_cmd_get_profile_config(adapter, &super_nic_res,
3853                                           RESOURCE_LIMITS, 0);
3854                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3855                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3856         } else {
3857                 res->max_tx_qs = BE3_MAX_TX_QS;
3858         }
3859
3860         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3861             !use_sriov && be_physfn(adapter))
3862                 res->max_rss_qs = (adapter->be3_native) ?
3863                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3864         res->max_rx_qs = res->max_rss_qs + 1;
3865
3866         if (be_physfn(adapter))
3867                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3868                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3869         else
3870                 res->max_evt_qs = 1;
3871
3872         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3873         res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS;
3874         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3875                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3876 }
3877
3878 static void be_setup_init(struct be_adapter *adapter)
3879 {
3880         adapter->vlan_prio_bmap = 0xff;
3881         adapter->phy.link_speed = -1;
3882         adapter->if_handle = -1;
3883         adapter->be3_native = false;
3884         adapter->if_flags = 0;
3885         if (be_physfn(adapter))
3886                 adapter->cmd_privileges = MAX_PRIVILEGES;
3887         else
3888                 adapter->cmd_privileges = MIN_PRIVILEGES;
3889 }
3890
3891 static int be_get_sriov_config(struct be_adapter *adapter)
3892 {
3893         struct be_resources res = {0};
3894         int max_vfs, old_vfs;
3895
3896         be_cmd_get_profile_config(adapter, &res, RESOURCE_LIMITS, 0);
3897
3898         /* Some old versions of BE3 FW don't report max_vfs value */
3899         if (BE3_chip(adapter) && !res.max_vfs) {
3900                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3901                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3902         }
3903
3904         adapter->pool_res = res;
3905
3906         /* If during previous unload of the driver, the VFs were not disabled,
3907          * then we cannot rely on the PF POOL limits for the TotalVFs value.
3908          * Instead use the TotalVFs value stored in the pci-dev struct.
3909          */
3910         old_vfs = pci_num_vf(adapter->pdev);
3911         if (old_vfs) {
3912                 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n",
3913                          old_vfs);
3914
3915                 adapter->pool_res.max_vfs =
3916                         pci_sriov_get_totalvfs(adapter->pdev);
3917                 adapter->num_vfs = old_vfs;
3918         }
3919
3920         return 0;
3921 }
3922
3923 static void be_alloc_sriov_res(struct be_adapter *adapter)
3924 {
3925         int old_vfs = pci_num_vf(adapter->pdev);
3926         u16 num_vf_qs;
3927         int status;
3928
3929         be_get_sriov_config(adapter);
3930
3931         if (!old_vfs)
3932                 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3933
3934         /* When the HW is in SRIOV capable configuration, the PF-pool
3935          * resources are given to PF during driver load, if there are no
3936          * old VFs. This facility is not available in BE3 FW.
3937          * Also, this is done by FW in Lancer chip.
3938          */
3939         if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
3940                 num_vf_qs = be_calculate_vf_qs(adapter, 0);
3941                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0,
3942                                                  num_vf_qs);
3943                 if (status)
3944                         dev_err(&adapter->pdev->dev,
3945                                 "Failed to optimize SRIOV resources\n");
3946         }
3947 }
3948
3949 static int be_get_resources(struct be_adapter *adapter)
3950 {
3951         struct device *dev = &adapter->pdev->dev;
3952         struct be_resources res = {0};
3953         int status;
3954
3955         if (BEx_chip(adapter)) {
3956                 BEx_get_resources(adapter, &res);
3957                 adapter->res = res;
3958         }
3959
3960         /* For Lancer, SH etc read per-function resource limits from FW.
3961          * GET_FUNC_CONFIG returns per function guaranteed limits.
3962          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3963          */
3964         if (!BEx_chip(adapter)) {
3965                 status = be_cmd_get_func_config(adapter, &res);
3966                 if (status)
3967                         return status;
3968
3969                 /* If a deafault RXQ must be created, we'll use up one RSSQ*/
3970                 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs &&
3971                     !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS))
3972                         res.max_rss_qs -= 1;
3973
3974                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3975                 if (be_roce_supported(adapter))
3976                         res.max_evt_qs /= 2;
3977                 adapter->res = res;
3978         }
3979
3980         /* If FW supports RSS default queue, then skip creating non-RSS
3981          * queue for non-IP traffic.
3982          */
3983         adapter->need_def_rxq = (be_if_cap_flags(adapter) &
3984                                  BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1;
3985
3986         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3987                  be_max_txqs(adapter), be_max_rxqs(adapter),
3988                  be_max_rss(adapter), be_max_eqs(adapter),
3989                  be_max_vfs(adapter));
3990         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3991                  be_max_uc(adapter), be_max_mc(adapter),
3992                  be_max_vlans(adapter));
3993
3994         /* Sanitize cfg_num_qs based on HW and platform limits */
3995         adapter->cfg_num_qs = min_t(u16, netif_get_num_default_rss_queues(),
3996                                     be_max_qs(adapter));
3997         return 0;
3998 }
3999
4000 static int be_get_config(struct be_adapter *adapter)
4001 {
4002         int status, level;
4003         u16 profile_id;
4004
4005         status = be_cmd_get_cntl_attributes(adapter);
4006         if (status)
4007                 return status;
4008
4009         status = be_cmd_query_fw_cfg(adapter);
4010         if (status)
4011                 return status;
4012
4013         if (BEx_chip(adapter)) {
4014                 level = be_cmd_get_fw_log_level(adapter);
4015                 adapter->msg_enable =
4016                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4017         }
4018
4019         be_cmd_get_acpi_wol_cap(adapter);
4020
4021         be_cmd_query_port_name(adapter);
4022
4023         if (be_physfn(adapter)) {
4024                 status = be_cmd_get_active_profile(adapter, &profile_id);
4025                 if (!status)
4026                         dev_info(&adapter->pdev->dev,
4027                                  "Using profile 0x%x\n", profile_id);
4028         }
4029
4030         status = be_get_resources(adapter);
4031         if (status)
4032                 return status;
4033
4034         adapter->pmac_id = kcalloc(be_max_uc(adapter),
4035                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
4036         if (!adapter->pmac_id)
4037                 return -ENOMEM;
4038
4039         return 0;
4040 }
4041
4042 static int be_mac_setup(struct be_adapter *adapter)
4043 {
4044         u8 mac[ETH_ALEN];
4045         int status;
4046
4047         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
4048                 status = be_cmd_get_perm_mac(adapter, mac);
4049                 if (status)
4050                         return status;
4051
4052                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
4053                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
4054         } else {
4055                 /* Maybe the HW was reset; dev_addr must be re-programmed */
4056                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
4057         }
4058
4059         /* For BE3-R VFs, the PF programs the initial MAC address */
4060         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
4061                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
4062                                 &adapter->pmac_id[0], 0);
4063         return 0;
4064 }
4065
4066 static void be_schedule_worker(struct be_adapter *adapter)
4067 {
4068         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4069         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
4070 }
4071
4072 static void be_schedule_err_detection(struct be_adapter *adapter)
4073 {
4074         schedule_delayed_work(&adapter->be_err_detection_work,
4075                               msecs_to_jiffies(1000));
4076         adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED;
4077 }
4078
4079 static int be_setup_queues(struct be_adapter *adapter)
4080 {
4081         struct net_device *netdev = adapter->netdev;
4082         int status;
4083
4084         status = be_evt_queues_create(adapter);
4085         if (status)
4086                 goto err;
4087
4088         status = be_tx_qs_create(adapter);
4089         if (status)
4090                 goto err;
4091
4092         status = be_rx_cqs_create(adapter);
4093         if (status)
4094                 goto err;
4095
4096         status = be_mcc_queues_create(adapter);
4097         if (status)
4098                 goto err;
4099
4100         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
4101         if (status)
4102                 goto err;
4103
4104         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
4105         if (status)
4106                 goto err;
4107
4108         return 0;
4109 err:
4110         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
4111         return status;
4112 }
4113
4114 int be_update_queues(struct be_adapter *adapter)
4115 {
4116         struct net_device *netdev = adapter->netdev;
4117         int status;
4118
4119         if (netif_running(netdev))
4120                 be_close(netdev);
4121
4122         be_cancel_worker(adapter);
4123
4124         /* If any vectors have been shared with RoCE we cannot re-program
4125          * the MSIx table.
4126          */
4127         if (!adapter->num_msix_roce_vec)
4128                 be_msix_disable(adapter);
4129
4130         be_clear_queues(adapter);
4131
4132         if (!msix_enabled(adapter)) {
4133                 status = be_msix_enable(adapter);
4134                 if (status)
4135                         return status;
4136         }
4137
4138         status = be_setup_queues(adapter);
4139         if (status)
4140                 return status;
4141
4142         be_schedule_worker(adapter);
4143
4144         if (netif_running(netdev))
4145                 status = be_open(netdev);
4146
4147         return status;
4148 }
4149
4150 static inline int fw_major_num(const char *fw_ver)
4151 {
4152         int fw_major = 0, i;
4153
4154         i = sscanf(fw_ver, "%d.", &fw_major);
4155         if (i != 1)
4156                 return 0;
4157
4158         return fw_major;
4159 }
4160
4161 /* If any VFs are already enabled don't FLR the PF */
4162 static bool be_reset_required(struct be_adapter *adapter)
4163 {
4164         return pci_num_vf(adapter->pdev) ? false : true;
4165 }
4166
4167 /* Wait for the FW to be ready and perform the required initialization */
4168 static int be_func_init(struct be_adapter *adapter)
4169 {
4170         int status;
4171
4172         status = be_fw_wait_ready(adapter);
4173         if (status)
4174                 return status;
4175
4176         if (be_reset_required(adapter)) {
4177                 status = be_cmd_reset_function(adapter);
4178                 if (status)
4179                         return status;
4180
4181                 /* Wait for interrupts to quiesce after an FLR */
4182                 msleep(100);
4183
4184                 /* We can clear all errors when function reset succeeds */
4185                 be_clear_all_error(adapter);
4186         }
4187
4188         /* Tell FW we're ready to fire cmds */
4189         status = be_cmd_fw_init(adapter);
4190         if (status)
4191                 return status;
4192
4193         /* Allow interrupts for other ULPs running on NIC function */
4194         be_intr_set(adapter, true);
4195
4196         return 0;
4197 }
4198
4199 static int be_setup(struct be_adapter *adapter)
4200 {
4201         struct device *dev = &adapter->pdev->dev;
4202         int status;
4203
4204         status = be_func_init(adapter);
4205         if (status)
4206                 return status;
4207
4208         be_setup_init(adapter);
4209
4210         if (!lancer_chip(adapter))
4211                 be_cmd_req_native_mode(adapter);
4212
4213         if (!BE2_chip(adapter) && be_physfn(adapter))
4214                 be_alloc_sriov_res(adapter);
4215
4216         status = be_get_config(adapter);
4217         if (status)
4218                 goto err;
4219
4220         status = be_msix_enable(adapter);
4221         if (status)
4222                 goto err;
4223
4224         status = be_if_create(adapter, &adapter->if_handle,
4225                               be_if_cap_flags(adapter), 0);
4226         if (status)
4227                 goto err;
4228
4229         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
4230         rtnl_lock();
4231         status = be_setup_queues(adapter);
4232         rtnl_unlock();
4233         if (status)
4234                 goto err;
4235
4236         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
4237
4238         status = be_mac_setup(adapter);
4239         if (status)
4240                 goto err;
4241
4242         be_cmd_get_fw_ver(adapter);
4243         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
4244
4245         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
4246                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
4247                         adapter->fw_ver);
4248                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
4249         }
4250
4251         if (adapter->vlans_added)
4252                 be_vid_config(adapter);
4253
4254         be_set_rx_mode(adapter->netdev);
4255
4256         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
4257                                          adapter->rx_fc);
4258         if (status)
4259                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
4260                                         &adapter->rx_fc);
4261
4262         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
4263                  adapter->tx_fc, adapter->rx_fc);
4264
4265         if (be_physfn(adapter))
4266                 be_cmd_set_logical_link_config(adapter,
4267                                                IFLA_VF_LINK_STATE_AUTO, 0);
4268
4269         if (adapter->num_vfs)
4270                 be_vf_setup(adapter);
4271
4272         status = be_cmd_get_phy_info(adapter);
4273         if (!status && be_pause_supported(adapter))
4274                 adapter->phy.fc_autoneg = 1;
4275
4276         be_schedule_worker(adapter);
4277         adapter->flags |= BE_FLAGS_SETUP_DONE;
4278         return 0;
4279 err:
4280         be_clear(adapter);
4281         return status;
4282 }
4283
4284 #ifdef CONFIG_NET_POLL_CONTROLLER
4285 static void be_netpoll(struct net_device *netdev)
4286 {
4287         struct be_adapter *adapter = netdev_priv(netdev);
4288         struct be_eq_obj *eqo;
4289         int i;
4290
4291         for_all_evt_queues(adapter, eqo, i) {
4292                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
4293                 napi_schedule(&eqo->napi);
4294         }
4295 }
4296 #endif
4297
4298 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
4299
4300 static bool phy_flashing_required(struct be_adapter *adapter)
4301 {
4302         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
4303                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
4304 }
4305
4306 static bool is_comp_in_ufi(struct be_adapter *adapter,
4307                            struct flash_section_info *fsec, int type)
4308 {
4309         int i = 0, img_type = 0;
4310         struct flash_section_info_g2 *fsec_g2 = NULL;
4311
4312         if (BE2_chip(adapter))
4313                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
4314
4315         for (i = 0; i < MAX_FLASH_COMP; i++) {
4316                 if (fsec_g2)
4317                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
4318                 else
4319                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
4320
4321                 if (img_type == type)
4322                         return true;
4323         }
4324         return false;
4325
4326 }
4327
4328 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
4329                                                 int header_size,
4330                                                 const struct firmware *fw)
4331 {
4332         struct flash_section_info *fsec = NULL;
4333         const u8 *p = fw->data;
4334
4335         p += header_size;
4336         while (p < (fw->data + fw->size)) {
4337                 fsec = (struct flash_section_info *)p;
4338                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
4339                         return fsec;
4340                 p += 32;
4341         }
4342         return NULL;
4343 }
4344
4345 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
4346                               u32 img_offset, u32 img_size, int hdr_size,
4347                               u16 img_optype, bool *crc_match)
4348 {
4349         u32 crc_offset;
4350         int status;
4351         u8 crc[4];
4352
4353         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
4354                                       img_size - 4);
4355         if (status)
4356                 return status;
4357
4358         crc_offset = hdr_size + img_offset + img_size - 4;
4359
4360         /* Skip flashing, if crc of flashed region matches */
4361         if (!memcmp(crc, p + crc_offset, 4))
4362                 *crc_match = true;
4363         else
4364                 *crc_match = false;
4365
4366         return status;
4367 }
4368
4369 static int be_flash(struct be_adapter *adapter, const u8 *img,
4370                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4371                     u32 img_offset)
4372 {
4373         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4374         struct be_cmd_write_flashrom *req = flash_cmd->va;
4375         int status;
4376
4377         while (total_bytes) {
4378                 num_bytes = min_t(u32, 32*1024, total_bytes);
4379
4380                 total_bytes -= num_bytes;
4381
4382                 if (!total_bytes) {
4383                         if (optype == OPTYPE_PHY_FW)
4384                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4385                         else
4386                                 flash_op = FLASHROM_OPER_FLASH;
4387                 } else {
4388                         if (optype == OPTYPE_PHY_FW)
4389                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4390                         else
4391                                 flash_op = FLASHROM_OPER_SAVE;
4392                 }
4393
4394                 memcpy(req->data_buf, img, num_bytes);
4395                 img += num_bytes;
4396                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4397                                                flash_op, img_offset +
4398                                                bytes_sent, num_bytes);
4399                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4400                     optype == OPTYPE_PHY_FW)
4401                         break;
4402                 else if (status)
4403                         return status;
4404
4405                 bytes_sent += num_bytes;
4406         }
4407         return 0;
4408 }
4409
4410 /* For BE2, BE3 and BE3-R */
4411 static int be_flash_BEx(struct be_adapter *adapter,
4412                         const struct firmware *fw,
4413                         struct be_dma_mem *flash_cmd, int num_of_images)
4414 {
4415         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4416         struct device *dev = &adapter->pdev->dev;
4417         struct flash_section_info *fsec = NULL;
4418         int status, i, filehdr_size, num_comp;
4419         const struct flash_comp *pflashcomp;
4420         bool crc_match;
4421         const u8 *p;
4422
4423         struct flash_comp gen3_flash_types[] = {
4424                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4425                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4426                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4427                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4428                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4429                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4430                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4431                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4432                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4433                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4434                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4435                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4436                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4437                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4438                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4439                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4440                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4441                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4442                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4443                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4444         };
4445
4446         struct flash_comp gen2_flash_types[] = {
4447                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4448                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4449                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4450                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4451                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4452                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4453                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4454                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4455                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4456                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4457                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4458                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4459                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4460                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4461                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4462                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4463         };
4464
4465         if (BE3_chip(adapter)) {
4466                 pflashcomp = gen3_flash_types;
4467                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4468                 num_comp = ARRAY_SIZE(gen3_flash_types);
4469         } else {
4470                 pflashcomp = gen2_flash_types;
4471                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4472                 num_comp = ARRAY_SIZE(gen2_flash_types);
4473                 img_hdrs_size = 0;
4474         }
4475
4476         /* Get flash section info*/
4477         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4478         if (!fsec) {
4479                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4480                 return -1;
4481         }
4482         for (i = 0; i < num_comp; i++) {
4483                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4484                         continue;
4485
4486                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4487                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4488                         continue;
4489
4490                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4491                     !phy_flashing_required(adapter))
4492                                 continue;
4493
4494                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4495                         status = be_check_flash_crc(adapter, fw->data,
4496                                                     pflashcomp[i].offset,
4497                                                     pflashcomp[i].size,
4498                                                     filehdr_size +
4499                                                     img_hdrs_size,
4500                                                     OPTYPE_REDBOOT, &crc_match);
4501                         if (status) {
4502                                 dev_err(dev,
4503                                         "Could not get CRC for 0x%x region\n",
4504                                         pflashcomp[i].optype);
4505                                 continue;
4506                         }
4507
4508                         if (crc_match)
4509                                 continue;
4510                 }
4511
4512                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4513                         img_hdrs_size;
4514                 if (p + pflashcomp[i].size > fw->data + fw->size)
4515                         return -1;
4516
4517                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4518                                   pflashcomp[i].size, 0);
4519                 if (status) {
4520                         dev_err(dev, "Flashing section type 0x%x failed\n",
4521                                 pflashcomp[i].img_type);
4522                         return status;
4523                 }
4524         }
4525         return 0;
4526 }
4527
4528 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4529 {
4530         u32 img_type = le32_to_cpu(fsec_entry.type);
4531         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4532
4533         if (img_optype != 0xFFFF)
4534                 return img_optype;
4535
4536         switch (img_type) {
4537         case IMAGE_FIRMWARE_iSCSI:
4538                 img_optype = OPTYPE_ISCSI_ACTIVE;
4539                 break;
4540         case IMAGE_BOOT_CODE:
4541                 img_optype = OPTYPE_REDBOOT;
4542                 break;
4543         case IMAGE_OPTION_ROM_ISCSI:
4544                 img_optype = OPTYPE_BIOS;
4545                 break;
4546         case IMAGE_OPTION_ROM_PXE:
4547                 img_optype = OPTYPE_PXE_BIOS;
4548                 break;
4549         case IMAGE_OPTION_ROM_FCoE:
4550                 img_optype = OPTYPE_FCOE_BIOS;
4551                 break;
4552         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4553                 img_optype = OPTYPE_ISCSI_BACKUP;
4554                 break;
4555         case IMAGE_NCSI:
4556                 img_optype = OPTYPE_NCSI_FW;
4557                 break;
4558         case IMAGE_FLASHISM_JUMPVECTOR:
4559                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4560                 break;
4561         case IMAGE_FIRMWARE_PHY:
4562                 img_optype = OPTYPE_SH_PHY_FW;
4563                 break;
4564         case IMAGE_REDBOOT_DIR:
4565                 img_optype = OPTYPE_REDBOOT_DIR;
4566                 break;
4567         case IMAGE_REDBOOT_CONFIG:
4568                 img_optype = OPTYPE_REDBOOT_CONFIG;
4569                 break;
4570         case IMAGE_UFI_DIR:
4571                 img_optype = OPTYPE_UFI_DIR;
4572                 break;
4573         default:
4574                 break;
4575         }
4576
4577         return img_optype;
4578 }
4579
4580 static int be_flash_skyhawk(struct be_adapter *adapter,
4581                             const struct firmware *fw,
4582                             struct be_dma_mem *flash_cmd, int num_of_images)
4583 {
4584         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4585         bool crc_match, old_fw_img, flash_offset_support = true;
4586         struct device *dev = &adapter->pdev->dev;
4587         struct flash_section_info *fsec = NULL;
4588         u32 img_offset, img_size, img_type;
4589         u16 img_optype, flash_optype;
4590         int status, i, filehdr_size;
4591         const u8 *p;
4592
4593         filehdr_size = sizeof(struct flash_file_hdr_g3);
4594         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4595         if (!fsec) {
4596                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4597                 return -EINVAL;
4598         }
4599
4600 retry_flash:
4601         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4602                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4603                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4604                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4605                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4606                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4607
4608                 if (img_optype == 0xFFFF)
4609                         continue;
4610
4611                 if (flash_offset_support)
4612                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4613                 else
4614                         flash_optype = img_optype;
4615
4616                 /* Don't bother verifying CRC if an old FW image is being
4617                  * flashed
4618                  */
4619                 if (old_fw_img)
4620                         goto flash;
4621
4622                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4623                                             img_size, filehdr_size +
4624                                             img_hdrs_size, flash_optype,
4625                                             &crc_match);
4626                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4627                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4628                         /* The current FW image on the card does not support
4629                          * OFFSET based flashing. Retry using older mechanism
4630                          * of OPTYPE based flashing
4631                          */
4632                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4633                                 flash_offset_support = false;
4634                                 goto retry_flash;
4635                         }
4636
4637                         /* The current FW image on the card does not recognize
4638                          * the new FLASH op_type. The FW download is partially
4639                          * complete. Reboot the server now to enable FW image
4640                          * to recognize the new FLASH op_type. To complete the
4641                          * remaining process, download the same FW again after
4642                          * the reboot.
4643                          */
4644                         dev_err(dev, "Flash incomplete. Reset the server\n");
4645                         dev_err(dev, "Download FW image again after reset\n");
4646                         return -EAGAIN;
4647                 } else if (status) {
4648                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4649                                 img_optype);
4650                         return -EFAULT;
4651                 }
4652
4653                 if (crc_match)
4654                         continue;
4655
4656 flash:
4657                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4658                 if (p + img_size > fw->data + fw->size)
4659                         return -1;
4660
4661                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4662                                   img_offset);
4663
4664                 /* The current FW image on the card does not support OFFSET
4665                  * based flashing. Retry using older mechanism of OPTYPE based
4666                  * flashing
4667                  */
4668                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4669                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4670                         flash_offset_support = false;
4671                         goto retry_flash;
4672                 }
4673
4674                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4675                  * UFI_DIR region
4676                  */
4677                 if (old_fw_img &&
4678                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4679                      (img_optype == OPTYPE_UFI_DIR &&
4680                       base_status(status) == MCC_STATUS_FAILED))) {
4681                         continue;
4682                 } else if (status) {
4683                         dev_err(dev, "Flashing section type 0x%x failed\n",
4684                                 img_type);
4685                         return -EFAULT;
4686                 }
4687         }
4688         return 0;
4689 }
4690
4691 static int lancer_fw_download(struct be_adapter *adapter,
4692                               const struct firmware *fw)
4693 {
4694 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4695 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4696         struct device *dev = &adapter->pdev->dev;
4697         struct be_dma_mem flash_cmd;
4698         const u8 *data_ptr = NULL;
4699         u8 *dest_image_ptr = NULL;
4700         size_t image_size = 0;
4701         u32 chunk_size = 0;
4702         u32 data_written = 0;
4703         u32 offset = 0;
4704         int status = 0;
4705         u8 add_status = 0;
4706         u8 change_status;
4707
4708         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4709                 dev_err(dev, "FW image size should be multiple of 4\n");
4710                 return -EINVAL;
4711         }
4712
4713         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4714                                 + LANCER_FW_DOWNLOAD_CHUNK;
4715         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4716                                           &flash_cmd.dma, GFP_KERNEL);
4717         if (!flash_cmd.va)
4718                 return -ENOMEM;
4719
4720         dest_image_ptr = flash_cmd.va +
4721                                 sizeof(struct lancer_cmd_req_write_object);
4722         image_size = fw->size;
4723         data_ptr = fw->data;
4724
4725         while (image_size) {
4726                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4727
4728                 /* Copy the image chunk content. */
4729                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4730
4731                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4732                                                  chunk_size, offset,
4733                                                  LANCER_FW_DOWNLOAD_LOCATION,
4734                                                  &data_written, &change_status,
4735                                                  &add_status);
4736                 if (status)
4737                         break;
4738
4739                 offset += data_written;
4740                 data_ptr += data_written;
4741                 image_size -= data_written;
4742         }
4743
4744         if (!status) {
4745                 /* Commit the FW written */
4746                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4747                                                  0, offset,
4748                                                  LANCER_FW_DOWNLOAD_LOCATION,
4749                                                  &data_written, &change_status,
4750                                                  &add_status);
4751         }
4752
4753         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4754         if (status) {
4755                 dev_err(dev, "Firmware load error\n");
4756                 return be_cmd_status(status);
4757         }
4758
4759         dev_info(dev, "Firmware flashed successfully\n");
4760
4761         if (change_status == LANCER_FW_RESET_NEEDED) {
4762                 dev_info(dev, "Resetting adapter to activate new FW\n");
4763                 status = lancer_physdev_ctrl(adapter,
4764                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4765                 if (status) {
4766                         dev_err(dev, "Adapter busy, could not reset FW\n");
4767                         dev_err(dev, "Reboot server to activate new FW\n");
4768                 }
4769         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4770                 dev_info(dev, "Reboot server to activate new FW\n");
4771         }
4772
4773         return 0;
4774 }
4775
4776 #define BE2_UFI         2
4777 #define BE3_UFI         3
4778 #define BE3R_UFI        10
4779 #define SH_UFI          4
4780 #define SH_P2_UFI       11
4781
4782 static int be_get_ufi_type(struct be_adapter *adapter,
4783                            struct flash_file_hdr_g3 *fhdr)
4784 {
4785         if (!fhdr) {
4786                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4787                 return -1;
4788         }
4789
4790         /* First letter of the build version is used to identify
4791          * which chip this image file is meant for.
4792          */
4793         switch (fhdr->build[0]) {
4794         case BLD_STR_UFI_TYPE_SH:
4795                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4796                                                                 SH_UFI;
4797         case BLD_STR_UFI_TYPE_BE3:
4798                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4799                                                                 BE3_UFI;
4800         case BLD_STR_UFI_TYPE_BE2:
4801                 return BE2_UFI;
4802         default:
4803                 return -1;
4804         }
4805 }
4806
4807 /* Check if the flash image file is compatible with the adapter that
4808  * is being flashed.
4809  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4810  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4811  */
4812 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4813                                        struct flash_file_hdr_g3 *fhdr)
4814 {
4815         int ufi_type = be_get_ufi_type(adapter, fhdr);
4816
4817         switch (ufi_type) {
4818         case SH_P2_UFI:
4819                 return skyhawk_chip(adapter);
4820         case SH_UFI:
4821                 return (skyhawk_chip(adapter) &&
4822                         adapter->asic_rev < ASIC_REV_P2);
4823         case BE3R_UFI:
4824                 return BE3_chip(adapter);
4825         case BE3_UFI:
4826                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4827         case BE2_UFI:
4828                 return BE2_chip(adapter);
4829         default:
4830                 return false;
4831         }
4832 }
4833
4834 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4835 {
4836         struct device *dev = &adapter->pdev->dev;
4837         struct flash_file_hdr_g3 *fhdr3;
4838         struct image_hdr *img_hdr_ptr;
4839         int status = 0, i, num_imgs;
4840         struct be_dma_mem flash_cmd;
4841
4842         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4843         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4844                 dev_err(dev, "Flash image is not compatible with adapter\n");
4845                 return -EINVAL;
4846         }
4847
4848         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4849         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4850                                           GFP_KERNEL);
4851         if (!flash_cmd.va)
4852                 return -ENOMEM;
4853
4854         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4855         for (i = 0; i < num_imgs; i++) {
4856                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4857                                 (sizeof(struct flash_file_hdr_g3) +
4858                                  i * sizeof(struct image_hdr)));
4859                 if (!BE2_chip(adapter) &&
4860                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4861                         continue;
4862
4863                 if (skyhawk_chip(adapter))
4864                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4865                                                   num_imgs);
4866                 else
4867                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4868                                               num_imgs);
4869         }
4870
4871         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4872         if (!status)
4873                 dev_info(dev, "Firmware flashed successfully\n");
4874
4875         return status;
4876 }
4877
4878 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4879 {
4880         const struct firmware *fw;
4881         int status;
4882
4883         if (!netif_running(adapter->netdev)) {
4884                 dev_err(&adapter->pdev->dev,
4885                         "Firmware load not allowed (interface is down)\n");
4886                 return -ENETDOWN;
4887         }
4888
4889         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4890         if (status)
4891                 goto fw_exit;
4892
4893         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4894
4895         if (lancer_chip(adapter))
4896                 status = lancer_fw_download(adapter, fw);
4897         else
4898                 status = be_fw_download(adapter, fw);
4899
4900         if (!status)
4901                 be_cmd_get_fw_ver(adapter);
4902
4903 fw_exit:
4904         release_firmware(fw);
4905         return status;
4906 }
4907
4908 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4909                                  u16 flags)
4910 {
4911         struct be_adapter *adapter = netdev_priv(dev);
4912         struct nlattr *attr, *br_spec;
4913         int rem;
4914         int status = 0;
4915         u16 mode = 0;
4916
4917         if (!sriov_enabled(adapter))
4918                 return -EOPNOTSUPP;
4919
4920         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4921         if (!br_spec)
4922                 return -EINVAL;
4923
4924         nla_for_each_nested(attr, br_spec, rem) {
4925                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4926                         continue;
4927
4928                 if (nla_len(attr) < sizeof(mode))
4929                         return -EINVAL;
4930
4931                 mode = nla_get_u16(attr);
4932                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4933                         return -EINVAL;
4934
4935                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4936                                                adapter->if_handle,
4937                                                mode == BRIDGE_MODE_VEPA ?
4938                                                PORT_FWD_TYPE_VEPA :
4939                                                PORT_FWD_TYPE_VEB, 0);
4940                 if (status)
4941                         goto err;
4942
4943                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4944                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4945
4946                 return status;
4947         }
4948 err:
4949         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4950                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4951
4952         return status;
4953 }
4954
4955 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4956                                  struct net_device *dev, u32 filter_mask,
4957                                  int nlflags)
4958 {
4959         struct be_adapter *adapter = netdev_priv(dev);
4960         int status = 0;
4961         u8 hsw_mode;
4962
4963         if (!sriov_enabled(adapter))
4964                 return 0;
4965
4966         /* BE and Lancer chips support VEB mode only */
4967         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4968                 hsw_mode = PORT_FWD_TYPE_VEB;
4969         } else {
4970                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4971                                                adapter->if_handle, &hsw_mode,
4972                                                NULL);
4973                 if (status)
4974                         return 0;
4975         }
4976
4977         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4978                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4979                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4980                                        0, 0, nlflags);
4981 }
4982
4983 #ifdef CONFIG_BE2NET_VXLAN
4984 /* VxLAN offload Notes:
4985  *
4986  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4987  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4988  * is expected to work across all types of IP tunnels once exported. Skyhawk
4989  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4990  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4991  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4992  * those other tunnels are unexported on the fly through ndo_features_check().
4993  *
4994  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4995  * adds more than one port, disable offloads and don't re-enable them again
4996  * until after all the tunnels are removed.
4997  */
4998 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4999                               __be16 port)
5000 {
5001         struct be_adapter *adapter = netdev_priv(netdev);
5002         struct device *dev = &adapter->pdev->dev;
5003         int status;
5004
5005         if (lancer_chip(adapter) || BEx_chip(adapter))
5006                 return;
5007
5008         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
5009                 dev_info(dev,
5010                          "Only one UDP port supported for VxLAN offloads\n");
5011                 dev_info(dev, "Disabling VxLAN offloads\n");
5012                 adapter->vxlan_port_count++;
5013                 goto err;
5014         }
5015
5016         if (adapter->vxlan_port_count++ >= 1)
5017                 return;
5018
5019         status = be_cmd_manage_iface(adapter, adapter->if_handle,
5020                                      OP_CONVERT_NORMAL_TO_TUNNEL);
5021         if (status) {
5022                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
5023                 goto err;
5024         }
5025
5026         status = be_cmd_set_vxlan_port(adapter, port);
5027         if (status) {
5028                 dev_warn(dev, "Failed to add VxLAN port\n");
5029                 goto err;
5030         }
5031         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
5032         adapter->vxlan_port = port;
5033
5034         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
5035                                    NETIF_F_TSO | NETIF_F_TSO6 |
5036                                    NETIF_F_GSO_UDP_TUNNEL;
5037         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
5038         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
5039
5040         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
5041                  be16_to_cpu(port));
5042         return;
5043 err:
5044         be_disable_vxlan_offloads(adapter);
5045 }
5046
5047 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
5048                               __be16 port)
5049 {
5050         struct be_adapter *adapter = netdev_priv(netdev);
5051
5052         if (lancer_chip(adapter) || BEx_chip(adapter))
5053                 return;
5054
5055         if (adapter->vxlan_port != port)
5056                 goto done;
5057
5058         be_disable_vxlan_offloads(adapter);
5059
5060         dev_info(&adapter->pdev->dev,
5061                  "Disabled VxLAN offloads for UDP port %d\n",
5062                  be16_to_cpu(port));
5063 done:
5064         adapter->vxlan_port_count--;
5065 }
5066
5067 static netdev_features_t be_features_check(struct sk_buff *skb,
5068                                            struct net_device *dev,
5069                                            netdev_features_t features)
5070 {
5071         struct be_adapter *adapter = netdev_priv(dev);
5072         u8 l4_hdr = 0;
5073
5074         /* The code below restricts offload features for some tunneled packets.
5075          * Offload features for normal (non tunnel) packets are unchanged.
5076          */
5077         if (!skb->encapsulation ||
5078             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
5079                 return features;
5080
5081         /* It's an encapsulated packet and VxLAN offloads are enabled. We
5082          * should disable tunnel offload features if it's not a VxLAN packet,
5083          * as tunnel offloads have been enabled only for VxLAN. This is done to
5084          * allow other tunneled traffic like GRE work fine while VxLAN
5085          * offloads are configured in Skyhawk-R.
5086          */
5087         switch (vlan_get_protocol(skb)) {
5088         case htons(ETH_P_IP):
5089                 l4_hdr = ip_hdr(skb)->protocol;
5090                 break;
5091         case htons(ETH_P_IPV6):
5092                 l4_hdr = ipv6_hdr(skb)->nexthdr;
5093                 break;
5094         default:
5095                 return features;
5096         }
5097
5098         if (l4_hdr != IPPROTO_UDP ||
5099             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
5100             skb->inner_protocol != htons(ETH_P_TEB) ||
5101             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
5102             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
5103                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
5104
5105         return features;
5106 }
5107 #endif
5108
5109 static const struct net_device_ops be_netdev_ops = {
5110         .ndo_open               = be_open,
5111         .ndo_stop               = be_close,
5112         .ndo_start_xmit         = be_xmit,
5113         .ndo_set_rx_mode        = be_set_rx_mode,
5114         .ndo_set_mac_address    = be_mac_addr_set,
5115         .ndo_change_mtu         = be_change_mtu,
5116         .ndo_get_stats64        = be_get_stats64,
5117         .ndo_validate_addr      = eth_validate_addr,
5118         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
5119         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
5120         .ndo_set_vf_mac         = be_set_vf_mac,
5121         .ndo_set_vf_vlan        = be_set_vf_vlan,
5122         .ndo_set_vf_rate        = be_set_vf_tx_rate,
5123         .ndo_get_vf_config      = be_get_vf_config,
5124         .ndo_set_vf_link_state  = be_set_vf_link_state,
5125         .ndo_set_vf_spoofchk    = be_set_vf_spoofchk,
5126 #ifdef CONFIG_NET_POLL_CONTROLLER
5127         .ndo_poll_controller    = be_netpoll,
5128 #endif
5129         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
5130         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
5131 #ifdef CONFIG_NET_RX_BUSY_POLL
5132         .ndo_busy_poll          = be_busy_poll,
5133 #endif
5134 #ifdef CONFIG_BE2NET_VXLAN
5135         .ndo_add_vxlan_port     = be_add_vxlan_port,
5136         .ndo_del_vxlan_port     = be_del_vxlan_port,
5137         .ndo_features_check     = be_features_check,
5138 #endif
5139 };
5140
5141 static void be_netdev_init(struct net_device *netdev)
5142 {
5143         struct be_adapter *adapter = netdev_priv(netdev);
5144
5145         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5146                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
5147                 NETIF_F_HW_VLAN_CTAG_TX;
5148         if (be_multi_rxq(adapter))
5149                 netdev->hw_features |= NETIF_F_RXHASH;
5150
5151         netdev->features |= netdev->hw_features |
5152                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
5153
5154         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5155                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
5156
5157         netdev->priv_flags |= IFF_UNICAST_FLT;
5158
5159         netdev->flags |= IFF_MULTICAST;
5160
5161         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
5162
5163         netdev->netdev_ops = &be_netdev_ops;
5164
5165         netdev->ethtool_ops = &be_ethtool_ops;
5166 }
5167
5168 static void be_cleanup(struct be_adapter *adapter)
5169 {
5170         struct net_device *netdev = adapter->netdev;
5171
5172         rtnl_lock();
5173         netif_device_detach(netdev);
5174         if (netif_running(netdev))
5175                 be_close(netdev);
5176         rtnl_unlock();
5177
5178         be_clear(adapter);
5179 }
5180
5181 static int be_resume(struct be_adapter *adapter)
5182 {
5183         struct net_device *netdev = adapter->netdev;
5184         int status;
5185
5186         status = be_setup(adapter);
5187         if (status)
5188                 return status;
5189
5190         if (netif_running(netdev)) {
5191                 status = be_open(netdev);
5192                 if (status)
5193                         return status;
5194         }
5195
5196         netif_device_attach(netdev);
5197
5198         return 0;
5199 }
5200
5201 static int be_err_recover(struct be_adapter *adapter)
5202 {
5203         struct device *dev = &adapter->pdev->dev;
5204         int status;
5205
5206         status = be_resume(adapter);
5207         if (status)
5208                 goto err;
5209
5210         dev_info(dev, "Adapter recovery successful\n");
5211         return 0;
5212 err:
5213         if (be_physfn(adapter))
5214                 dev_err(dev, "Adapter recovery failed\n");
5215         else
5216                 dev_err(dev, "Re-trying adapter recovery\n");
5217
5218         return status;
5219 }
5220
5221 static void be_err_detection_task(struct work_struct *work)
5222 {
5223         struct be_adapter *adapter =
5224                                 container_of(work, struct be_adapter,
5225                                              be_err_detection_work.work);
5226         int status = 0;
5227
5228         be_detect_error(adapter);
5229
5230         if (adapter->hw_error) {
5231                 be_cleanup(adapter);
5232
5233                 /* As of now error recovery support is in Lancer only */
5234                 if (lancer_chip(adapter))
5235                         status = be_err_recover(adapter);
5236         }
5237
5238         /* Always attempt recovery on VFs */
5239         if (!status || be_virtfn(adapter))
5240                 be_schedule_err_detection(adapter);
5241 }
5242
5243 static void be_log_sfp_info(struct be_adapter *adapter)
5244 {
5245         int status;
5246
5247         status = be_cmd_query_sfp_info(adapter);
5248         if (!status) {
5249                 dev_err(&adapter->pdev->dev,
5250                         "Unqualified SFP+ detected on %c from %s part no: %s",
5251                         adapter->port_name, adapter->phy.vendor_name,
5252                         adapter->phy.vendor_pn);
5253         }
5254         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5255 }
5256
5257 static void be_worker(struct work_struct *work)
5258 {
5259         struct be_adapter *adapter =
5260                 container_of(work, struct be_adapter, work.work);
5261         struct be_rx_obj *rxo;
5262         int i;
5263
5264         /* when interrupts are not yet enabled, just reap any pending
5265          * mcc completions
5266          */
5267         if (!netif_running(adapter->netdev)) {
5268                 local_bh_disable();
5269                 be_process_mcc(adapter);
5270                 local_bh_enable();
5271                 goto reschedule;
5272         }
5273
5274         if (!adapter->stats_cmd_sent) {
5275                 if (lancer_chip(adapter))
5276                         lancer_cmd_get_pport_stats(adapter,
5277                                                    &adapter->stats_cmd);
5278                 else
5279                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5280         }
5281
5282         if (be_physfn(adapter) &&
5283             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5284                 be_cmd_get_die_temperature(adapter);
5285
5286         for_all_rx_queues(adapter, rxo, i) {
5287                 /* Replenish RX-queues starved due to memory
5288                  * allocation failures.
5289                  */
5290                 if (rxo->rx_post_starved)
5291                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5292         }
5293
5294         /* EQ-delay update for Skyhawk is done while notifying EQ */
5295         if (!skyhawk_chip(adapter))
5296                 be_eqd_update(adapter, false);
5297
5298         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5299                 be_log_sfp_info(adapter);
5300
5301 reschedule:
5302         adapter->work_counter++;
5303         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5304 }
5305
5306 static void be_unmap_pci_bars(struct be_adapter *adapter)
5307 {
5308         if (adapter->csr)
5309                 pci_iounmap(adapter->pdev, adapter->csr);
5310         if (adapter->db)
5311                 pci_iounmap(adapter->pdev, adapter->db);
5312 }
5313
5314 static int db_bar(struct be_adapter *adapter)
5315 {
5316         if (lancer_chip(adapter) || !be_physfn(adapter))
5317                 return 0;
5318         else
5319                 return 4;
5320 }
5321
5322 static int be_roce_map_pci_bars(struct be_adapter *adapter)
5323 {
5324         if (skyhawk_chip(adapter)) {
5325                 adapter->roce_db.size = 4096;
5326                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
5327                                                               db_bar(adapter));
5328                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
5329                                                                db_bar(adapter));
5330         }
5331         return 0;
5332 }
5333
5334 static int be_map_pci_bars(struct be_adapter *adapter)
5335 {
5336         struct pci_dev *pdev = adapter->pdev;
5337         u8 __iomem *addr;
5338         u32 sli_intf;
5339
5340         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
5341         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
5342                                 SLI_INTF_FAMILY_SHIFT;
5343         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
5344
5345         if (BEx_chip(adapter) && be_physfn(adapter)) {
5346                 adapter->csr = pci_iomap(pdev, 2, 0);
5347                 if (!adapter->csr)
5348                         return -ENOMEM;
5349         }
5350
5351         addr = pci_iomap(pdev, db_bar(adapter), 0);
5352         if (!addr)
5353                 goto pci_map_err;
5354         adapter->db = addr;
5355
5356         if (skyhawk_chip(adapter) || BEx_chip(adapter)) {
5357                 if (be_physfn(adapter)) {
5358                         /* PCICFG is the 2nd BAR in BE2 */
5359                         addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0);
5360                         if (!addr)
5361                                 goto pci_map_err;
5362                         adapter->pcicfg = addr;
5363                 } else {
5364                         adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
5365                 }
5366         }
5367
5368         be_roce_map_pci_bars(adapter);
5369         return 0;
5370
5371 pci_map_err:
5372         dev_err(&pdev->dev, "Error in mapping PCI BARs\n");
5373         be_unmap_pci_bars(adapter);
5374         return -ENOMEM;
5375 }
5376
5377 static void be_drv_cleanup(struct be_adapter *adapter)
5378 {
5379         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
5380         struct device *dev = &adapter->pdev->dev;
5381
5382         if (mem->va)
5383                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5384
5385         mem = &adapter->rx_filter;
5386         if (mem->va)
5387                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5388
5389         mem = &adapter->stats_cmd;
5390         if (mem->va)
5391                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5392 }
5393
5394 /* Allocate and initialize various fields in be_adapter struct */
5395 static int be_drv_init(struct be_adapter *adapter)
5396 {
5397         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
5398         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
5399         struct be_dma_mem *rx_filter = &adapter->rx_filter;
5400         struct be_dma_mem *stats_cmd = &adapter->stats_cmd;
5401         struct device *dev = &adapter->pdev->dev;
5402         int status = 0;
5403
5404         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
5405         mbox_mem_alloc->va = dma_alloc_coherent(dev, mbox_mem_alloc->size,
5406                                                 &mbox_mem_alloc->dma,
5407                                                 GFP_KERNEL);
5408         if (!mbox_mem_alloc->va)
5409                 return -ENOMEM;
5410
5411         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
5412         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
5413         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
5414         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
5415
5416         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
5417         rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size,
5418                                             &rx_filter->dma, GFP_KERNEL);
5419         if (!rx_filter->va) {
5420                 status = -ENOMEM;
5421                 goto free_mbox;
5422         }
5423
5424         if (lancer_chip(adapter))
5425                 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5426         else if (BE2_chip(adapter))
5427                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5428         else if (BE3_chip(adapter))
5429                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5430         else
5431                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5432         stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size,
5433                                             &stats_cmd->dma, GFP_KERNEL);
5434         if (!stats_cmd->va) {
5435                 status = -ENOMEM;
5436                 goto free_rx_filter;
5437         }
5438
5439         mutex_init(&adapter->mbox_lock);
5440         spin_lock_init(&adapter->mcc_lock);
5441         spin_lock_init(&adapter->mcc_cq_lock);
5442         init_completion(&adapter->et_cmd_compl);
5443
5444         pci_save_state(adapter->pdev);
5445
5446         INIT_DELAYED_WORK(&adapter->work, be_worker);
5447         INIT_DELAYED_WORK(&adapter->be_err_detection_work,
5448                           be_err_detection_task);
5449
5450         adapter->rx_fc = true;
5451         adapter->tx_fc = true;
5452
5453         /* Must be a power of 2 or else MODULO will BUG_ON */
5454         adapter->be_get_temp_freq = 64;
5455
5456         return 0;
5457
5458 free_rx_filter:
5459         dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma);
5460 free_mbox:
5461         dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va,
5462                           mbox_mem_alloc->dma);
5463         return status;
5464 }
5465
5466 static void be_remove(struct pci_dev *pdev)
5467 {
5468         struct be_adapter *adapter = pci_get_drvdata(pdev);
5469
5470         if (!adapter)
5471                 return;
5472
5473         be_roce_dev_remove(adapter);
5474         be_intr_set(adapter, false);
5475
5476         be_cancel_err_detection(adapter);
5477
5478         unregister_netdev(adapter->netdev);
5479
5480         be_clear(adapter);
5481
5482         /* tell fw we're done with firing cmds */
5483         be_cmd_fw_clean(adapter);
5484
5485         be_unmap_pci_bars(adapter);
5486         be_drv_cleanup(adapter);
5487
5488         pci_disable_pcie_error_reporting(pdev);
5489
5490         pci_release_regions(pdev);
5491         pci_disable_device(pdev);
5492
5493         free_netdev(adapter->netdev);
5494 }
5495
5496 static char *mc_name(struct be_adapter *adapter)
5497 {
5498         char *str = ""; /* default */
5499
5500         switch (adapter->mc_type) {
5501         case UMC:
5502                 str = "UMC";
5503                 break;
5504         case FLEX10:
5505                 str = "FLEX10";
5506                 break;
5507         case vNIC1:
5508                 str = "vNIC-1";
5509                 break;
5510         case nPAR:
5511                 str = "nPAR";
5512                 break;
5513         case UFP:
5514                 str = "UFP";
5515                 break;
5516         case vNIC2:
5517                 str = "vNIC-2";
5518                 break;
5519         default:
5520                 str = "";
5521         }
5522
5523         return str;
5524 }
5525
5526 static inline char *func_name(struct be_adapter *adapter)
5527 {
5528         return be_physfn(adapter) ? "PF" : "VF";
5529 }
5530
5531 static inline char *nic_name(struct pci_dev *pdev)
5532 {
5533         switch (pdev->device) {
5534         case OC_DEVICE_ID1:
5535                 return OC_NAME;
5536         case OC_DEVICE_ID2:
5537                 return OC_NAME_BE;
5538         case OC_DEVICE_ID3:
5539         case OC_DEVICE_ID4:
5540                 return OC_NAME_LANCER;
5541         case BE_DEVICE_ID2:
5542                 return BE3_NAME;
5543         case OC_DEVICE_ID5:
5544         case OC_DEVICE_ID6:
5545                 return OC_NAME_SH;
5546         default:
5547                 return BE_NAME;
5548         }
5549 }
5550
5551 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5552 {
5553         struct be_adapter *adapter;
5554         struct net_device *netdev;
5555         int status = 0;
5556
5557         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5558
5559         status = pci_enable_device(pdev);
5560         if (status)
5561                 goto do_none;
5562
5563         status = pci_request_regions(pdev, DRV_NAME);
5564         if (status)
5565                 goto disable_dev;
5566         pci_set_master(pdev);
5567
5568         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5569         if (!netdev) {
5570                 status = -ENOMEM;
5571                 goto rel_reg;
5572         }
5573         adapter = netdev_priv(netdev);
5574         adapter->pdev = pdev;
5575         pci_set_drvdata(pdev, adapter);
5576         adapter->netdev = netdev;
5577         SET_NETDEV_DEV(netdev, &pdev->dev);
5578
5579         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5580         if (!status) {
5581                 netdev->features |= NETIF_F_HIGHDMA;
5582         } else {
5583                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5584                 if (status) {
5585                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5586                         goto free_netdev;
5587                 }
5588         }
5589
5590         status = pci_enable_pcie_error_reporting(pdev);
5591         if (!status)
5592                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5593
5594         status = be_map_pci_bars(adapter);
5595         if (status)
5596                 goto free_netdev;
5597
5598         status = be_drv_init(adapter);
5599         if (status)
5600                 goto unmap_bars;
5601
5602         status = be_setup(adapter);
5603         if (status)
5604                 goto drv_cleanup;
5605
5606         be_netdev_init(netdev);
5607         status = register_netdev(netdev);
5608         if (status != 0)
5609                 goto unsetup;
5610
5611         be_roce_dev_add(adapter);
5612
5613         be_schedule_err_detection(adapter);
5614
5615         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5616                  func_name(adapter), mc_name(adapter), adapter->port_name);
5617
5618         return 0;
5619
5620 unsetup:
5621         be_clear(adapter);
5622 drv_cleanup:
5623         be_drv_cleanup(adapter);
5624 unmap_bars:
5625         be_unmap_pci_bars(adapter);
5626 free_netdev:
5627         free_netdev(netdev);
5628 rel_reg:
5629         pci_release_regions(pdev);
5630 disable_dev:
5631         pci_disable_device(pdev);
5632 do_none:
5633         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5634         return status;
5635 }
5636
5637 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5638 {
5639         struct be_adapter *adapter = pci_get_drvdata(pdev);
5640
5641         if (adapter->wol_en)
5642                 be_setup_wol(adapter, true);
5643
5644         be_intr_set(adapter, false);
5645         be_cancel_err_detection(adapter);
5646
5647         be_cleanup(adapter);
5648
5649         pci_save_state(pdev);
5650         pci_disable_device(pdev);
5651         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5652         return 0;
5653 }
5654
5655 static int be_pci_resume(struct pci_dev *pdev)
5656 {
5657         struct be_adapter *adapter = pci_get_drvdata(pdev);
5658         int status = 0;
5659
5660         status = pci_enable_device(pdev);
5661         if (status)
5662                 return status;
5663
5664         pci_set_power_state(pdev, PCI_D0);
5665         pci_restore_state(pdev);
5666
5667         status = be_resume(adapter);
5668         if (status)
5669                 return status;
5670
5671         be_schedule_err_detection(adapter);
5672
5673         if (adapter->wol_en)
5674                 be_setup_wol(adapter, false);
5675
5676         return 0;
5677 }
5678
5679 /*
5680  * An FLR will stop BE from DMAing any data.
5681  */
5682 static void be_shutdown(struct pci_dev *pdev)
5683 {
5684         struct be_adapter *adapter = pci_get_drvdata(pdev);
5685
5686         if (!adapter)
5687                 return;
5688
5689         be_roce_dev_shutdown(adapter);
5690         cancel_delayed_work_sync(&adapter->work);
5691         be_cancel_err_detection(adapter);
5692
5693         netif_device_detach(adapter->netdev);
5694
5695         be_cmd_reset_function(adapter);
5696
5697         pci_disable_device(pdev);
5698 }
5699
5700 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5701                                             pci_channel_state_t state)
5702 {
5703         struct be_adapter *adapter = pci_get_drvdata(pdev);
5704
5705         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5706
5707         if (!adapter->eeh_error) {
5708                 adapter->eeh_error = true;
5709
5710                 be_cancel_err_detection(adapter);
5711
5712                 be_cleanup(adapter);
5713         }
5714
5715         if (state == pci_channel_io_perm_failure)
5716                 return PCI_ERS_RESULT_DISCONNECT;
5717
5718         pci_disable_device(pdev);
5719
5720         /* The error could cause the FW to trigger a flash debug dump.
5721          * Resetting the card while flash dump is in progress
5722          * can cause it not to recover; wait for it to finish.
5723          * Wait only for first function as it is needed only once per
5724          * adapter.
5725          */
5726         if (pdev->devfn == 0)
5727                 ssleep(30);
5728
5729         return PCI_ERS_RESULT_NEED_RESET;
5730 }
5731
5732 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5733 {
5734         struct be_adapter *adapter = pci_get_drvdata(pdev);
5735         int status;
5736
5737         dev_info(&adapter->pdev->dev, "EEH reset\n");
5738
5739         status = pci_enable_device(pdev);
5740         if (status)
5741                 return PCI_ERS_RESULT_DISCONNECT;
5742
5743         pci_set_master(pdev);
5744         pci_set_power_state(pdev, PCI_D0);
5745         pci_restore_state(pdev);
5746
5747         /* Check if card is ok and fw is ready */
5748         dev_info(&adapter->pdev->dev,
5749                  "Waiting for FW to be ready after EEH reset\n");
5750         status = be_fw_wait_ready(adapter);
5751         if (status)
5752                 return PCI_ERS_RESULT_DISCONNECT;
5753
5754         pci_cleanup_aer_uncorrect_error_status(pdev);
5755         be_clear_all_error(adapter);
5756         return PCI_ERS_RESULT_RECOVERED;
5757 }
5758
5759 static void be_eeh_resume(struct pci_dev *pdev)
5760 {
5761         int status = 0;
5762         struct be_adapter *adapter = pci_get_drvdata(pdev);
5763
5764         dev_info(&adapter->pdev->dev, "EEH resume\n");
5765
5766         pci_save_state(pdev);
5767
5768         status = be_resume(adapter);
5769         if (status)
5770                 goto err;
5771
5772         be_schedule_err_detection(adapter);
5773         return;
5774 err:
5775         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5776 }
5777
5778 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
5779 {
5780         struct be_adapter *adapter = pci_get_drvdata(pdev);
5781         u16 num_vf_qs;
5782         int status;
5783
5784         if (!num_vfs)
5785                 be_vf_clear(adapter);
5786
5787         adapter->num_vfs = num_vfs;
5788
5789         if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) {
5790                 dev_warn(&pdev->dev,
5791                          "Cannot disable VFs while they are assigned\n");
5792                 return -EBUSY;
5793         }
5794
5795         /* When the HW is in SRIOV capable configuration, the PF-pool resources
5796          * are equally distributed across the max-number of VFs. The user may
5797          * request only a subset of the max-vfs to be enabled.
5798          * Based on num_vfs, redistribute the resources across num_vfs so that
5799          * each VF will have access to more number of resources.
5800          * This facility is not available in BE3 FW.
5801          * Also, this is done by FW in Lancer chip.
5802          */
5803         if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) {
5804                 num_vf_qs = be_calculate_vf_qs(adapter, adapter->num_vfs);
5805                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res,
5806                                                  adapter->num_vfs, num_vf_qs);
5807                 if (status)
5808                         dev_err(&pdev->dev,
5809                                 "Failed to optimize SR-IOV resources\n");
5810         }
5811
5812         status = be_get_resources(adapter);
5813         if (status)
5814                 return be_cmd_status(status);
5815
5816         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
5817         rtnl_lock();
5818         status = be_update_queues(adapter);
5819         rtnl_unlock();
5820         if (status)
5821                 return be_cmd_status(status);
5822
5823         if (adapter->num_vfs)
5824                 status = be_vf_setup(adapter);
5825
5826         if (!status)
5827                 return adapter->num_vfs;
5828
5829         return 0;
5830 }
5831
5832 static const struct pci_error_handlers be_eeh_handlers = {
5833         .error_detected = be_eeh_err_detected,
5834         .slot_reset = be_eeh_reset,
5835         .resume = be_eeh_resume,
5836 };
5837
5838 static struct pci_driver be_driver = {
5839         .name = DRV_NAME,
5840         .id_table = be_dev_ids,
5841         .probe = be_probe,
5842         .remove = be_remove,
5843         .suspend = be_suspend,
5844         .resume = be_pci_resume,
5845         .shutdown = be_shutdown,
5846         .sriov_configure = be_pci_sriov_configure,
5847         .err_handler = &be_eeh_handlers
5848 };
5849
5850 static int __init be_init_module(void)
5851 {
5852         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5853             rx_frag_size != 2048) {
5854                 printk(KERN_WARNING DRV_NAME
5855                         " : Module param rx_frag_size must be 2048/4096/8192."
5856                         " Using 2048\n");
5857                 rx_frag_size = 2048;
5858         }
5859
5860         if (num_vfs > 0) {
5861                 pr_info(DRV_NAME " : Module param num_vfs is obsolete.");
5862                 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n");
5863         }
5864
5865         return pci_register_driver(&be_driver);
5866 }
5867 module_init(be_init_module);
5868
5869 static void __exit be_exit_module(void)
5870 {
5871         pci_unregister_driver(&be_driver);
5872 }
5873 module_exit(be_exit_module);