drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo,
 666                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 667                                bool stopped)
 668 {
 669         struct be_tx_stats *stats = tx_stats(txo);
 670
 671         u64_stats_update_begin(&stats->sync);
 672         stats->tx_reqs++;
 673         stats->tx_wrbs += wrb_cnt;
 674         stats->tx_bytes += copied;
 675         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 676         if (stopped)
 677                 stats->tx_stops++;
 678         u64_stats_update_end(&stats->sync);
 679 }
 680
 681 /* Determine number of WRB entries needed to xmit data in an skb */
 682 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 683                            bool *dummy)
 684 {
 685         int cnt = (skb->len > skb->data_len);
 686
 687         cnt += skb_shinfo(skb)->nr_frags;
 688
 689         /* to account for hdr wrb */
 690         cnt++;
 691         if (lancer_chip(adapter) || !(cnt & 1)) {
 692                 *dummy = false;
 693         } else {
 694                 /* add a dummy to make it an even num */
 695                 cnt++;
 696                 *dummy = true;
 697         }
 698         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 699         return cnt;
 700 }
 701
 702 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 703 {
 704         wrb->frag_pa_hi = upper_32_bits(addr);
 705         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 706         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 707         wrb->rsvd0 = 0;
 708 }
 709
 710 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 711                                      struct sk_buff *skb)
 712 {
 713         u8 vlan_prio;
 714         u16 vlan_tag;
 715
 716         vlan_tag = vlan_tx_tag_get(skb);
 717         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 718         /* If vlan priority provided by OS is NOT in available bmap */
 719         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 720                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 721                                 adapter->recommended_prio;
 722
 723         return vlan_tag;
 724 }
 725
 726 /* Used only for IP tunnel packets */
 727 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 728 {
 729         return (inner_ip_hdr(skb)->version == 4) ?
 730                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static u16 skb_ip_proto(struct sk_buff *skb)
 734 {
 735         return (ip_hdr(skb)->version == 4) ?
 736                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 737 }
 738
 739 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 740                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 741                          bool skip_hw_vlan)
 742 {
 743         u16 vlan_tag, proto;
 744
 745         memset(hdr, 0, sizeof(*hdr));
 746
 747         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 748
 749         if (skb_is_gso(skb)) {
 750                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 751                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 752                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 753                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 754         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 755                 if (skb->encapsulation) {
 756                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 757                         proto = skb_inner_ip_proto(skb);
 758                 } else {
 759                         proto = skb_ip_proto(skb);
 760                 }
 761                 if (proto == IPPROTO_TCP)
 762                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 763                 else if (proto == IPPROTO_UDP)
 764                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 765         }
 766
 767         if (vlan_tx_tag_present(skb)) {
 768                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 769                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 770                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 771         }
 772
 773         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 774         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 775         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 776         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 777         SET_TX_WRB_HDR_BITS(len, hdr, len);
 778 }
 779
 780 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 781                           bool unmap_single)
 782 {
 783         dma_addr_t dma;
 784
 785         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 786
 787         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 788         if (wrb->frag_len) {
 789                 if (unmap_single)
 790                         dma_unmap_single(dev, dma, wrb->frag_len,
 791                                          DMA_TO_DEVICE);
 792                 else
 793                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 794         }
 795 }
 796
 797 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 798                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 799                         bool skip_hw_vlan)
 800 {
 801         dma_addr_t busaddr;
 802         int i, copied = 0;
 803         struct device *dev = &adapter->pdev->dev;
 804         struct sk_buff *first_skb = skb;
 805         struct be_eth_wrb *wrb;
 806         struct be_eth_hdr_wrb *hdr;
 807         bool map_single = false;
 808         u16 map_head;
 809
 810         hdr = queue_head_node(txq);
 811         queue_head_inc(txq);
 812         map_head = txq->head;
 813
 814         if (skb->len > skb->data_len) {
 815                 int len = skb_headlen(skb);
 816
 817                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 818                 if (dma_mapping_error(dev, busaddr))
 819                         goto dma_err;
 820                 map_single = true;
 821                 wrb = queue_head_node(txq);
 822                 wrb_fill(wrb, busaddr, len);
 823                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 824                 queue_head_inc(txq);
 825                 copied += len;
 826         }
 827
 828         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 829                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 830
 831                 busaddr = skb_frag_dma_map(dev, frag, 0,
 832                                            skb_frag_size(frag), DMA_TO_DEVICE);
 833                 if (dma_mapping_error(dev, busaddr))
 834                         goto dma_err;
 835                 wrb = queue_head_node(txq);
 836                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 837                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 838                 queue_head_inc(txq);
 839                 copied += skb_frag_size(frag);
 840         }
 841
 842         if (dummy_wrb) {
 843                 wrb = queue_head_node(txq);
 844                 wrb_fill(wrb, 0, 0);
 845                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 846                 queue_head_inc(txq);
 847         }
 848
 849         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 850         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 851
 852         return copied;
 853 dma_err:
 854         txq->head = map_head;
 855         while (copied) {
 856                 wrb = queue_head_node(txq);
 857                 unmap_tx_frag(dev, wrb, map_single);
 858                 map_single = false;
 859                 copied -= wrb->frag_len;
 860                 adapter->drv_stats.dma_map_errors++;
 861                 queue_head_inc(txq);
 862         }
 863         return 0;
 864 }
 865
 866 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 867                                              struct sk_buff *skb,
 868                                              bool *skip_hw_vlan)
 869 {
 870         u16 vlan_tag = 0;
 871
 872         skb = skb_share_check(skb, GFP_ATOMIC);
 873         if (unlikely(!skb))
 874                 return skb;
 875
 876         if (vlan_tx_tag_present(skb))
 877                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 878
 879         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 880                 if (!vlan_tag)
 881                         vlan_tag = adapter->pvid;
 882                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 883                  * skip VLAN insertion
 884                  */
 885                 if (skip_hw_vlan)
 886                         *skip_hw_vlan = true;
 887         }
 888
 889         if (vlan_tag) {
 890                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 891                                                 vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 skb->vlan_tci = 0;
 895         }
 896
 897         /* Insert the outer VLAN, if any */
 898         if (adapter->qnq_vid) {
 899                 vlan_tag = adapter->qnq_vid;
 900                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 901                                                 vlan_tag);
 902                 if (unlikely(!skb))
 903                         return skb;
 904                 if (skip_hw_vlan)
 905                         *skip_hw_vlan = true;
 906         }
 907
 908         return skb;
 909 }
 910
 911 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 912 {
 913         struct ethhdr *eh = (struct ethhdr *)skb->data;
 914         u16 offset = ETH_HLEN;
 915
 916         if (eh->h_proto == htons(ETH_P_IPV6)) {
 917                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 918
 919                 offset += sizeof(struct ipv6hdr);
 920                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 921                     ip6h->nexthdr != NEXTHDR_UDP) {
 922                         struct ipv6_opt_hdr *ehdr =
 923                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 924
 925                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 926                         if (ehdr->hdrlen == 0xff)
 927                                 return true;
 928                 }
 929         }
 930         return false;
 931 }
 932
 933 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 934 {
 935         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 936 }
 937
 938 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 939 {
 940         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 941 }
 942
 943 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 944                                                   struct sk_buff *skb,
 945                                                   bool *skip_hw_vlan)
 946 {
 947         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 948         unsigned int eth_hdr_len;
 949         struct iphdr *ip;
 950
 951         /* For padded packets, BE HW modifies tot_len field in IP header
 952          * incorrecly when VLAN tag is inserted by HW.
 953          * For padded packets, Lancer computes incorrect checksum.
 954          */
 955         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 956                                                 VLAN_ETH_HLEN : ETH_HLEN;
 957         if (skb->len <= 60 &&
 958             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 959             is_ipv4_pkt(skb)) {
 960                 ip = (struct iphdr *)ip_hdr(skb);
 961                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 962         }
 963
 964         /* If vlan tag is already inlined in the packet, skip HW VLAN
 965          * tagging in pvid-tagging mode
 966          */
 967         if (be_pvid_tagging_enabled(adapter) &&
 968             veh->h_vlan_proto == htons(ETH_P_8021Q))
 969                 *skip_hw_vlan = true;
 970
 971         /* HW has a bug wherein it will calculate CSUM for VLAN
 972          * pkts even though it is disabled.
 973          * Manually insert VLAN in pkt.
 974          */
 975         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 976             vlan_tx_tag_present(skb)) {
 977                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 978                 if (unlikely(!skb))
 979                         goto err;
 980         }
 981
 982         /* HW may lockup when VLAN HW tagging is requested on
 983          * certain ipv6 packets. Drop such pkts if the HW workaround to
 984          * skip HW tagging is not enabled by FW.
 985          */
 986         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 987                      (adapter->pvid || adapter->qnq_vid) &&
 988                      !qnq_async_evt_rcvd(adapter)))
 989                 goto tx_drop;
 990
 991         /* Manual VLAN tag insertion to prevent:
 992          * ASIC lockup when the ASIC inserts VLAN tag into
 993          * certain ipv6 packets. Insert VLAN tags in driver,
 994          * and set event, completion, vlan bits accordingly
 995          * in the Tx WRB.
 996          */
 997         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 998             be_vlan_tag_tx_chk(adapter, skb)) {
 999                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1000                 if (unlikely(!skb))
1001                         goto err;
1002         }
1003
1004         return skb;
1005 tx_drop:
1006         dev_kfree_skb_any(skb);
1007 err:
1008         return NULL;
1009 }
1010
1011 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1012                                            struct sk_buff *skb,
1013                                            bool *skip_hw_vlan)
1014 {
1015         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1016          * less may cause a transmit stall on that port. So the work-around is
1017          * to pad short packets (<= 32 bytes) to a 36-byte length.
1018          */
1019         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1020                 if (skb_padto(skb, 36))
1021                         return NULL;
1022                 skb->len = 36;
1023         }
1024
1025         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1026                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1027                 if (!skb)
1028                         return NULL;
1029         }
1030
1031         return skb;
1032 }
1033
1034 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1035 {
1036         struct be_adapter *adapter = netdev_priv(netdev);
1037         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1038         struct be_queue_info *txq = &txo->q;
1039         bool dummy_wrb, stopped = false;
1040         u32 wrb_cnt = 0, copied = 0;
1041         bool skip_hw_vlan = false;
1042         u32 start = txq->head;
1043
1044         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1045         if (!skb) {
1046                 tx_stats(txo)->tx_drv_drops++;
1047                 return NETDEV_TX_OK;
1048         }
1049
1050         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1051
1052         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1053                               skip_hw_vlan);
1054         if (copied) {
1055                 int gso_segs = skb_shinfo(skb)->gso_segs;
1056
1057                 /* record the sent skb in the sent_skb table */
1058                 BUG_ON(txo->sent_skb_list[start]);
1059                 txo->sent_skb_list[start] = skb;
1060
1061                 /* Ensure txq has space for the next skb; Else stop the queue
1062                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1063                  * tx compls of the current transmit which'll wake up the queue
1064                  */
1065                 atomic_add(wrb_cnt, &txq->used);
1066                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1067                                                                 txq->len) {
1068                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1069                         stopped = true;
1070                 }
1071
1072                 be_txq_notify(adapter, txo, wrb_cnt);
1073
1074                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1075         } else {
1076                 txq->head = start;
1077                 tx_stats(txo)->tx_drv_drops++;
1078                 dev_kfree_skb_any(skb);
1079         }
1080         return NETDEV_TX_OK;
1081 }
1082
1083 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1084 {
1085         struct be_adapter *adapter = netdev_priv(netdev);
1086         struct device *dev = &adapter->pdev->dev;
1087
1088         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1089                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1090                          BE_MIN_MTU, BE_MAX_MTU);
1091                 return -EINVAL;
1092         }
1093
1094         dev_info(dev, "MTU changed from %d to %d bytes\n",
1095                  netdev->mtu, new_mtu);
1096         netdev->mtu = new_mtu;
1097         return 0;
1098 }
1099
1100 /*
1101  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1102  * If the user configures more, place BE in vlan promiscuous mode.
1103  */
1104 static int be_vid_config(struct be_adapter *adapter)
1105 {
1106         struct device *dev = &adapter->pdev->dev;
1107         u16 vids[BE_NUM_VLANS_SUPPORTED];
1108         u16 num = 0, i = 0;
1109         int status = 0;
1110
1111         /* No need to further configure vids if in promiscuous mode */
1112         if (adapter->promiscuous)
1113                 return 0;
1114
1115         if (adapter->vlans_added > be_max_vlans(adapter))
1116                 goto set_vlan_promisc;
1117
1118         /* Construct VLAN Table to give to HW */
1119         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1120                 vids[num++] = cpu_to_le16(i);
1121
1122         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1123         if (status) {
1124                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1125                 if (addl_status(status) ==
1126                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1127                         goto set_vlan_promisc;
1128                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1129         } else {
1130                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1131                         /* hw VLAN filtering re-enabled. */
1132                         status = be_cmd_rx_filter(adapter,
1133                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1134                         if (!status) {
1135                                 dev_info(dev,
1136                                          "Disabling VLAN Promiscuous mode\n");
1137                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1138                         }
1139                 }
1140         }
1141
1142         return status;
1143
1144 set_vlan_promisc:
1145         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1146                 return 0;
1147
1148         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1149         if (!status) {
1150                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1151                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1152         } else
1153                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1154         return status;
1155 }
1156
1157 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1158 {
1159         struct be_adapter *adapter = netdev_priv(netdev);
1160         int status = 0;
1161
1162         /* Packets with VID 0 are always received by Lancer by default */
1163         if (lancer_chip(adapter) && vid == 0)
1164                 return status;
1165
1166         if (test_bit(vid, adapter->vids))
1167                 return status;
1168
1169         set_bit(vid, adapter->vids);
1170         adapter->vlans_added++;
1171
1172         status = be_vid_config(adapter);
1173         if (status) {
1174                 adapter->vlans_added--;
1175                 clear_bit(vid, adapter->vids);
1176         }
1177
1178         return status;
1179 }
1180
1181 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1182 {
1183         struct be_adapter *adapter = netdev_priv(netdev);
1184
1185         /* Packets with VID 0 are always received by Lancer by default */
1186         if (lancer_chip(adapter) && vid == 0)
1187                 return 0;
1188
1189         clear_bit(vid, adapter->vids);
1190         adapter->vlans_added--;
1191
1192         return be_vid_config(adapter);
1193 }
1194
1195 static void be_clear_promisc(struct be_adapter *adapter)
1196 {
1197         adapter->promiscuous = false;
1198         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1199
1200         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1201 }
1202
1203 static void be_set_rx_mode(struct net_device *netdev)
1204 {
1205         struct be_adapter *adapter = netdev_priv(netdev);
1206         int status;
1207
1208         if (netdev->flags & IFF_PROMISC) {
1209                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1210                 adapter->promiscuous = true;
1211                 goto done;
1212         }
1213
1214         /* BE was previously in promiscuous mode; disable it */
1215         if (adapter->promiscuous) {
1216                 be_clear_promisc(adapter);
1217                 if (adapter->vlans_added)
1218                         be_vid_config(adapter);
1219         }
1220
1221         /* Enable multicast promisc if num configured exceeds what we support */
1222         if (netdev->flags & IFF_ALLMULTI ||
1223             netdev_mc_count(netdev) > be_max_mc(adapter))
1224                 goto set_mcast_promisc;
1225
1226         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1227                 struct netdev_hw_addr *ha;
1228                 int i = 1; /* First slot is claimed by the Primary MAC */
1229
1230                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1231                         be_cmd_pmac_del(adapter, adapter->if_handle,
1232                                         adapter->pmac_id[i], 0);
1233                 }
1234
1235                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1236                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1237                         adapter->promiscuous = true;
1238                         goto done;
1239                 }
1240
1241                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1242                         adapter->uc_macs++; /* First slot is for Primary MAC */
1243                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1244                                         adapter->if_handle,
1245                                         &adapter->pmac_id[adapter->uc_macs], 0);
1246                 }
1247         }
1248
1249         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1250         if (!status) {
1251                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1252                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1253                 goto done;
1254         }
1255
1256 set_mcast_promisc:
1257         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1258                 return;
1259
1260         /* Set to MCAST promisc mode if setting MULTICAST address fails
1261          * or if num configured exceeds what we support
1262          */
1263         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1264         if (!status)
1265                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1266 done:
1267         return;
1268 }
1269
1270 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1271 {
1272         struct be_adapter *adapter = netdev_priv(netdev);
1273         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1274         int status;
1275
1276         if (!sriov_enabled(adapter))
1277                 return -EPERM;
1278
1279         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1280                 return -EINVAL;
1281
1282         /* Proceed further only if user provided MAC is different
1283          * from active MAC
1284          */
1285         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1286                 return 0;
1287
1288         if (BEx_chip(adapter)) {
1289                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1290                                 vf + 1);
1291
1292                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1293                                          &vf_cfg->pmac_id, vf + 1);
1294         } else {
1295                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1296                                         vf + 1);
1297         }
1298
1299         if (status) {
1300                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1301                         mac, vf, status);
1302                 return be_cmd_status(status);
1303         }
1304
1305         ether_addr_copy(vf_cfg->mac_addr, mac);
1306
1307         return 0;
1308 }
1309
1310 static int be_get_vf_config(struct net_device *netdev, int vf,
1311                             struct ifla_vf_info *vi)
1312 {
1313         struct be_adapter *adapter = netdev_priv(netdev);
1314         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1315
1316         if (!sriov_enabled(adapter))
1317                 return -EPERM;
1318
1319         if (vf >= adapter->num_vfs)
1320                 return -EINVAL;
1321
1322         vi->vf = vf;
1323         vi->max_tx_rate = vf_cfg->tx_rate;
1324         vi->min_tx_rate = 0;
1325         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1326         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1327         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1328         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1329
1330         return 0;
1331 }
1332
1333 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1334 {
1335         struct be_adapter *adapter = netdev_priv(netdev);
1336         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1337         int status = 0;
1338
1339         if (!sriov_enabled(adapter))
1340                 return -EPERM;
1341
1342         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1343                 return -EINVAL;
1344
1345         if (vlan || qos) {
1346                 vlan |= qos << VLAN_PRIO_SHIFT;
1347                 if (vf_cfg->vlan_tag != vlan)
1348                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1349                                                        vf_cfg->if_handle, 0);
1350         } else {
1351                 /* Reset Transparent Vlan Tagging. */
1352                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1353                                                vf + 1, vf_cfg->if_handle, 0);
1354         }
1355
1356         if (status) {
1357                 dev_err(&adapter->pdev->dev,
1358                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1359                         vf, status);
1360                 return be_cmd_status(status);
1361         }
1362
1363         vf_cfg->vlan_tag = vlan;
1364
1365         return 0;
1366 }
1367
1368 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1369                              int min_tx_rate, int max_tx_rate)
1370 {
1371         struct be_adapter *adapter = netdev_priv(netdev);
1372         struct device *dev = &adapter->pdev->dev;
1373         int percent_rate, status = 0;
1374         u16 link_speed = 0;
1375         u8 link_status;
1376
1377         if (!sriov_enabled(adapter))
1378                 return -EPERM;
1379
1380         if (vf >= adapter->num_vfs)
1381                 return -EINVAL;
1382
1383         if (min_tx_rate)
1384                 return -EINVAL;
1385
1386         if (!max_tx_rate)
1387                 goto config_qos;
1388
1389         status = be_cmd_link_status_query(adapter, &link_speed,
1390                                           &link_status, 0);
1391         if (status)
1392                 goto err;
1393
1394         if (!link_status) {
1395                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1396                 status = -ENETDOWN;
1397                 goto err;
1398         }
1399
1400         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1401                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1402                         link_speed);
1403                 status = -EINVAL;
1404                 goto err;
1405         }
1406
1407         /* On Skyhawk the QOS setting must be done only as a % value */
1408         percent_rate = link_speed / 100;
1409         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1410                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1411                         percent_rate);
1412                 status = -EINVAL;
1413                 goto err;
1414         }
1415
1416 config_qos:
1417         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1418         if (status)
1419                 goto err;
1420
1421         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1422         return 0;
1423
1424 err:
1425         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1426                 max_tx_rate, vf);
1427         return be_cmd_status(status);
1428 }
1429
1430 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1431                                 int link_state)
1432 {
1433         struct be_adapter *adapter = netdev_priv(netdev);
1434         int status;
1435
1436         if (!sriov_enabled(adapter))
1437                 return -EPERM;
1438
1439         if (vf >= adapter->num_vfs)
1440                 return -EINVAL;
1441
1442         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1443         if (status) {
1444                 dev_err(&adapter->pdev->dev,
1445                         "Link state change on VF %d failed: %#x\n", vf, status);
1446                 return be_cmd_status(status);
1447         }
1448
1449         adapter->vf_cfg[vf].plink_tracking = link_state;
1450
1451         return 0;
1452 }
1453
1454 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1455                           ulong now)
1456 {
1457         aic->rx_pkts_prev = rx_pkts;
1458         aic->tx_reqs_prev = tx_pkts;
1459         aic->jiffies = now;
1460 }
1461
1462 static void be_eqd_update(struct be_adapter *adapter)
1463 {
1464         struct be_set_eqd set_eqd[MAX_EVT_QS];
1465         int eqd, i, num = 0, start;
1466         struct be_aic_obj *aic;
1467         struct be_eq_obj *eqo;
1468         struct be_rx_obj *rxo;
1469         struct be_tx_obj *txo;
1470         u64 rx_pkts, tx_pkts;
1471         ulong now;
1472         u32 pps, delta;
1473
1474         for_all_evt_queues(adapter, eqo, i) {
1475                 aic = &adapter->aic_obj[eqo->idx];
1476                 if (!aic->enable) {
1477                         if (aic->jiffies)
1478                                 aic->jiffies = 0;
1479                         eqd = aic->et_eqd;
1480                         goto modify_eqd;
1481                 }
1482
1483                 rxo = &adapter->rx_obj[eqo->idx];
1484                 do {
1485                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1486                         rx_pkts = rxo->stats.rx_pkts;
1487                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1488
1489                 txo = &adapter->tx_obj[eqo->idx];
1490                 do {
1491                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1492                         tx_pkts = txo->stats.tx_reqs;
1493                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1494
1495                 /* Skip, if wrapped around or first calculation */
1496                 now = jiffies;
1497                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1498                     rx_pkts < aic->rx_pkts_prev ||
1499                     tx_pkts < aic->tx_reqs_prev) {
1500                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1501                         continue;
1502                 }
1503
1504                 delta = jiffies_to_msecs(now - aic->jiffies);
1505                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1506                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1507                 eqd = (pps / 15000) << 2;
1508
1509                 if (eqd < 8)
1510                         eqd = 0;
1511                 eqd = min_t(u32, eqd, aic->max_eqd);
1512                 eqd = max_t(u32, eqd, aic->min_eqd);
1513
1514                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1515 modify_eqd:
1516                 if (eqd != aic->prev_eqd) {
1517                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1518                         set_eqd[num].eq_id = eqo->q.id;
1519                         aic->prev_eqd = eqd;
1520                         num++;
1521                 }
1522         }
1523
1524         if (num)
1525                 be_cmd_modify_eqd(adapter, set_eqd, num);
1526 }
1527
1528 static void be_rx_stats_update(struct be_rx_obj *rxo,
1529                                struct be_rx_compl_info *rxcp)
1530 {
1531         struct be_rx_stats *stats = rx_stats(rxo);
1532
1533         u64_stats_update_begin(&stats->sync);
1534         stats->rx_compl++;
1535         stats->rx_bytes += rxcp->pkt_size;
1536         stats->rx_pkts++;
1537         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1538                 stats->rx_mcast_pkts++;
1539         if (rxcp->err)
1540                 stats->rx_compl_err++;
1541         u64_stats_update_end(&stats->sync);
1542 }
1543
1544 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1545 {
1546         /* L4 checksum is not reliable for non TCP/UDP packets.
1547          * Also ignore ipcksm for ipv6 pkts
1548          */
1549         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1550                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1551 }
1552
1553 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1554 {
1555         struct be_adapter *adapter = rxo->adapter;
1556         struct be_rx_page_info *rx_page_info;
1557         struct be_queue_info *rxq = &rxo->q;
1558         u16 frag_idx = rxq->tail;
1559
1560         rx_page_info = &rxo->page_info_tbl[frag_idx];
1561         BUG_ON(!rx_page_info->page);
1562
1563         if (rx_page_info->last_frag) {
1564                 dma_unmap_page(&adapter->pdev->dev,
1565                                dma_unmap_addr(rx_page_info, bus),
1566                                adapter->big_page_size, DMA_FROM_DEVICE);
1567                 rx_page_info->last_frag = false;
1568         } else {
1569                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1570                                         dma_unmap_addr(rx_page_info, bus),
1571                                         rx_frag_size, DMA_FROM_DEVICE);
1572         }
1573
1574         queue_tail_inc(rxq);
1575         atomic_dec(&rxq->used);
1576         return rx_page_info;
1577 }
1578
1579 /* Throwaway the data in the Rx completion */
1580 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1581                                 struct be_rx_compl_info *rxcp)
1582 {
1583         struct be_rx_page_info *page_info;
1584         u16 i, num_rcvd = rxcp->num_rcvd;
1585
1586         for (i = 0; i < num_rcvd; i++) {
1587                 page_info = get_rx_page_info(rxo);
1588                 put_page(page_info->page);
1589                 memset(page_info, 0, sizeof(*page_info));
1590         }
1591 }
1592
1593 /*
1594  * skb_fill_rx_data forms a complete skb for an ether frame
1595  * indicated by rxcp.
1596  */
1597 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1598                              struct be_rx_compl_info *rxcp)
1599 {
1600         struct be_rx_page_info *page_info;
1601         u16 i, j;
1602         u16 hdr_len, curr_frag_len, remaining;
1603         u8 *start;
1604
1605         page_info = get_rx_page_info(rxo);
1606         start = page_address(page_info->page) + page_info->page_offset;
1607         prefetch(start);
1608
1609         /* Copy data in the first descriptor of this completion */
1610         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1611
1612         skb->len = curr_frag_len;
1613         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1614                 memcpy(skb->data, start, curr_frag_len);
1615                 /* Complete packet has now been moved to data */
1616                 put_page(page_info->page);
1617                 skb->data_len = 0;
1618                 skb->tail += curr_frag_len;
1619         } else {
1620                 hdr_len = ETH_HLEN;
1621                 memcpy(skb->data, start, hdr_len);
1622                 skb_shinfo(skb)->nr_frags = 1;
1623                 skb_frag_set_page(skb, 0, page_info->page);
1624                 skb_shinfo(skb)->frags[0].page_offset =
1625                                         page_info->page_offset + hdr_len;
1626                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1627                                   curr_frag_len - hdr_len);
1628                 skb->data_len = curr_frag_len - hdr_len;
1629                 skb->truesize += rx_frag_size;
1630                 skb->tail += hdr_len;
1631         }
1632         page_info->page = NULL;
1633
1634         if (rxcp->pkt_size <= rx_frag_size) {
1635                 BUG_ON(rxcp->num_rcvd != 1);
1636                 return;
1637         }
1638
1639         /* More frags present for this completion */
1640         remaining = rxcp->pkt_size - curr_frag_len;
1641         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1642                 page_info = get_rx_page_info(rxo);
1643                 curr_frag_len = min(remaining, rx_frag_size);
1644
1645                 /* Coalesce all frags from the same physical page in one slot */
1646                 if (page_info->page_offset == 0) {
1647                         /* Fresh page */
1648                         j++;
1649                         skb_frag_set_page(skb, j, page_info->page);
1650                         skb_shinfo(skb)->frags[j].page_offset =
1651                                                         page_info->page_offset;
1652                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1653                         skb_shinfo(skb)->nr_frags++;
1654                 } else {
1655                         put_page(page_info->page);
1656                 }
1657
1658                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1659                 skb->len += curr_frag_len;
1660                 skb->data_len += curr_frag_len;
1661                 skb->truesize += rx_frag_size;
1662                 remaining -= curr_frag_len;
1663                 page_info->page = NULL;
1664         }
1665         BUG_ON(j > MAX_SKB_FRAGS);
1666 }
1667
1668 /* Process the RX completion indicated by rxcp when GRO is disabled */
1669 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1670                                 struct be_rx_compl_info *rxcp)
1671 {
1672         struct be_adapter *adapter = rxo->adapter;
1673         struct net_device *netdev = adapter->netdev;
1674         struct sk_buff *skb;
1675
1676         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1677         if (unlikely(!skb)) {
1678                 rx_stats(rxo)->rx_drops_no_skbs++;
1679                 be_rx_compl_discard(rxo, rxcp);
1680                 return;
1681         }
1682
1683         skb_fill_rx_data(rxo, skb, rxcp);
1684
1685         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1686                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1687         else
1688                 skb_checksum_none_assert(skb);
1689
1690         skb->protocol = eth_type_trans(skb, netdev);
1691         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1692         if (netdev->features & NETIF_F_RXHASH)
1693                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1694
1695         skb->csum_level = rxcp->tunneled;
1696         skb_mark_napi_id(skb, napi);
1697
1698         if (rxcp->vlanf)
1699                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1700
1701         netif_receive_skb(skb);
1702 }
1703
1704 /* Process the RX completion indicated by rxcp when GRO is enabled */
1705 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1706                                     struct napi_struct *napi,
1707                                     struct be_rx_compl_info *rxcp)
1708 {
1709         struct be_adapter *adapter = rxo->adapter;
1710         struct be_rx_page_info *page_info;
1711         struct sk_buff *skb = NULL;
1712         u16 remaining, curr_frag_len;
1713         u16 i, j;
1714
1715         skb = napi_get_frags(napi);
1716         if (!skb) {
1717                 be_rx_compl_discard(rxo, rxcp);
1718                 return;
1719         }
1720
1721         remaining = rxcp->pkt_size;
1722         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1723                 page_info = get_rx_page_info(rxo);
1724
1725                 curr_frag_len = min(remaining, rx_frag_size);
1726
1727                 /* Coalesce all frags from the same physical page in one slot */
1728                 if (i == 0 || page_info->page_offset == 0) {
1729                         /* First frag or Fresh page */
1730                         j++;
1731                         skb_frag_set_page(skb, j, page_info->page);
1732                         skb_shinfo(skb)->frags[j].page_offset =
1733                                                         page_info->page_offset;
1734                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1735                 } else {
1736                         put_page(page_info->page);
1737                 }
1738                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1739                 skb->truesize += rx_frag_size;
1740                 remaining -= curr_frag_len;
1741                 memset(page_info, 0, sizeof(*page_info));
1742         }
1743         BUG_ON(j > MAX_SKB_FRAGS);
1744
1745         skb_shinfo(skb)->nr_frags = j + 1;
1746         skb->len = rxcp->pkt_size;
1747         skb->data_len = rxcp->pkt_size;
1748         skb->ip_summed = CHECKSUM_UNNECESSARY;
1749         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1750         if (adapter->netdev->features & NETIF_F_RXHASH)
1751                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1752
1753         skb->csum_level = rxcp->tunneled;
1754         skb_mark_napi_id(skb, napi);
1755
1756         if (rxcp->vlanf)
1757                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1758
1759         napi_gro_frags(napi);
1760 }
1761
1762 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1763                                  struct be_rx_compl_info *rxcp)
1764 {
1765         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1766         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1767         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1768         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1769         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1770         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1771         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1772         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1773         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1774         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1775         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1776         if (rxcp->vlanf) {
1777                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1778                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1779         }
1780         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1781         rxcp->tunneled =
1782                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1783 }
1784
1785 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1786                                  struct be_rx_compl_info *rxcp)
1787 {
1788         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1789         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1790         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1791         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1792         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1793         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1794         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1795         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1796         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1797         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1798         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1799         if (rxcp->vlanf) {
1800                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1801                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1802         }
1803         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1804         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1805 }
1806
1807 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1808 {
1809         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1810         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1811         struct be_adapter *adapter = rxo->adapter;
1812
1813         /* For checking the valid bit it is Ok to use either definition as the
1814          * valid bit is at the same position in both v0 and v1 Rx compl */
1815         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1816                 return NULL;
1817
1818         rmb();
1819         be_dws_le_to_cpu(compl, sizeof(*compl));
1820
1821         if (adapter->be3_native)
1822                 be_parse_rx_compl_v1(compl, rxcp);
1823         else
1824                 be_parse_rx_compl_v0(compl, rxcp);
1825
1826         if (rxcp->ip_frag)
1827                 rxcp->l4_csum = 0;
1828
1829         if (rxcp->vlanf) {
1830                 /* In QNQ modes, if qnq bit is not set, then the packet was
1831                  * tagged only with the transparent outer vlan-tag and must
1832                  * not be treated as a vlan packet by host
1833                  */
1834                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1835                         rxcp->vlanf = 0;
1836
1837                 if (!lancer_chip(adapter))
1838                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1839
1840                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1841                     !test_bit(rxcp->vlan_tag, adapter->vids))
1842                         rxcp->vlanf = 0;
1843         }
1844
1845         /* As the compl has been parsed, reset it; we wont touch it again */
1846         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1847
1848         queue_tail_inc(&rxo->cq);
1849         return rxcp;
1850 }
1851
1852 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1853 {
1854         u32 order = get_order(size);
1855
1856         if (order > 0)
1857                 gfp |= __GFP_COMP;
1858         return  alloc_pages(gfp, order);
1859 }
1860
1861 /*
1862  * Allocate a page, split it to fragments of size rx_frag_size and post as
1863  * receive buffers to BE
1864  */
1865 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1866 {
1867         struct be_adapter *adapter = rxo->adapter;
1868         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1869         struct be_queue_info *rxq = &rxo->q;
1870         struct page *pagep = NULL;
1871         struct device *dev = &adapter->pdev->dev;
1872         struct be_eth_rx_d *rxd;
1873         u64 page_dmaaddr = 0, frag_dmaaddr;
1874         u32 posted, page_offset = 0, notify = 0;
1875
1876         page_info = &rxo->page_info_tbl[rxq->head];
1877         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1878                 if (!pagep) {
1879                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1880                         if (unlikely(!pagep)) {
1881                                 rx_stats(rxo)->rx_post_fail++;
1882                                 break;
1883                         }
1884                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1885                                                     adapter->big_page_size,
1886                                                     DMA_FROM_DEVICE);
1887                         if (dma_mapping_error(dev, page_dmaaddr)) {
1888                                 put_page(pagep);
1889                                 pagep = NULL;
1890                                 adapter->drv_stats.dma_map_errors++;
1891                                 break;
1892                         }
1893                         page_offset = 0;
1894                 } else {
1895                         get_page(pagep);
1896                         page_offset += rx_frag_size;
1897                 }
1898                 page_info->page_offset = page_offset;
1899                 page_info->page = pagep;
1900
1901                 rxd = queue_head_node(rxq);
1902                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1903                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1904                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1905
1906                 /* Any space left in the current big page for another frag? */
1907                 if ((page_offset + rx_frag_size + rx_frag_size) >
1908                                         adapter->big_page_size) {
1909                         pagep = NULL;
1910                         page_info->last_frag = true;
1911                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1912                 } else {
1913                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1914                 }
1915
1916                 prev_page_info = page_info;
1917                 queue_head_inc(rxq);
1918                 page_info = &rxo->page_info_tbl[rxq->head];
1919         }
1920
1921         /* Mark the last frag of a page when we break out of the above loop
1922          * with no more slots available in the RXQ
1923          */
1924         if (pagep) {
1925                 prev_page_info->last_frag = true;
1926                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1927         }
1928
1929         if (posted) {
1930                 atomic_add(posted, &rxq->used);
1931                 if (rxo->rx_post_starved)
1932                         rxo->rx_post_starved = false;
1933                 do {
1934                         notify = min(256u, posted);
1935                         be_rxq_notify(adapter, rxq->id, notify);
1936                         posted -= notify;
1937                 } while (posted);
1938         } else if (atomic_read(&rxq->used) == 0) {
1939                 /* Let be_worker replenish when memory is available */
1940                 rxo->rx_post_starved = true;
1941         }
1942 }
1943
1944 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1945 {
1946         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1947
1948         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1949                 return NULL;
1950
1951         rmb();
1952         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1953
1954         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1955
1956         queue_tail_inc(tx_cq);
1957         return txcp;
1958 }
1959
1960 static u16 be_tx_compl_process(struct be_adapter *adapter,
1961                                struct be_tx_obj *txo, u16 last_index)
1962 {
1963         struct be_queue_info *txq = &txo->q;
1964         struct be_eth_wrb *wrb;
1965         struct sk_buff **sent_skbs = txo->sent_skb_list;
1966         struct sk_buff *sent_skb;
1967         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1968         bool unmap_skb_hdr = true;
1969
1970         sent_skb = sent_skbs[txq->tail];
1971         BUG_ON(!sent_skb);
1972         sent_skbs[txq->tail] = NULL;
1973
1974         /* skip header wrb */
1975         queue_tail_inc(txq);
1976
1977         do {
1978                 cur_index = txq->tail;
1979                 wrb = queue_tail_node(txq);
1980                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1981                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1982                 unmap_skb_hdr = false;
1983
1984                 num_wrbs++;
1985                 queue_tail_inc(txq);
1986         } while (cur_index != last_index);
1987
1988         dev_consume_skb_any(sent_skb);
1989         return num_wrbs;
1990 }
1991
1992 /* Return the number of events in the event queue */
1993 static inline int events_get(struct be_eq_obj *eqo)
1994 {
1995         struct be_eq_entry *eqe;
1996         int num = 0;
1997
1998         do {
1999                 eqe = queue_tail_node(&eqo->q);
2000                 if (eqe->evt == 0)
2001                         break;
2002
2003                 rmb();
2004                 eqe->evt = 0;
2005                 num++;
2006                 queue_tail_inc(&eqo->q);
2007         } while (true);
2008
2009         return num;
2010 }
2011
2012 /* Leaves the EQ is disarmed state */
2013 static void be_eq_clean(struct be_eq_obj *eqo)
2014 {
2015         int num = events_get(eqo);
2016
2017         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2018 }
2019
2020 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2021 {
2022         struct be_rx_page_info *page_info;
2023         struct be_queue_info *rxq = &rxo->q;
2024         struct be_queue_info *rx_cq = &rxo->cq;
2025         struct be_rx_compl_info *rxcp;
2026         struct be_adapter *adapter = rxo->adapter;
2027         int flush_wait = 0;
2028
2029         /* Consume pending rx completions.
2030          * Wait for the flush completion (identified by zero num_rcvd)
2031          * to arrive. Notify CQ even when there are no more CQ entries
2032          * for HW to flush partially coalesced CQ entries.
2033          * In Lancer, there is no need to wait for flush compl.
2034          */
2035         for (;;) {
2036                 rxcp = be_rx_compl_get(rxo);
2037                 if (!rxcp) {
2038                         if (lancer_chip(adapter))
2039                                 break;
2040
2041                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2042                                 dev_warn(&adapter->pdev->dev,
2043                                          "did not receive flush compl\n");
2044                                 break;
2045                         }
2046                         be_cq_notify(adapter, rx_cq->id, true, 0);
2047                         mdelay(1);
2048                 } else {
2049                         be_rx_compl_discard(rxo, rxcp);
2050                         be_cq_notify(adapter, rx_cq->id, false, 1);
2051                         if (rxcp->num_rcvd == 0)
2052                                 break;
2053                 }
2054         }
2055
2056         /* After cleanup, leave the CQ in unarmed state */
2057         be_cq_notify(adapter, rx_cq->id, false, 0);
2058
2059         /* Then free posted rx buffers that were not used */
2060         while (atomic_read(&rxq->used) > 0) {
2061                 page_info = get_rx_page_info(rxo);
2062                 put_page(page_info->page);
2063                 memset(page_info, 0, sizeof(*page_info));
2064         }
2065         BUG_ON(atomic_read(&rxq->used));
2066         rxq->tail = 0;
2067         rxq->head = 0;
2068 }
2069
2070 static void be_tx_compl_clean(struct be_adapter *adapter)
2071 {
2072         struct be_tx_obj *txo;
2073         struct be_queue_info *txq;
2074         struct be_eth_tx_compl *txcp;
2075         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2076         struct sk_buff *sent_skb;
2077         bool dummy_wrb;
2078         int i, pending_txqs;
2079
2080         /* Stop polling for compls when HW has been silent for 10ms */
2081         do {
2082                 pending_txqs = adapter->num_tx_qs;
2083
2084                 for_all_tx_queues(adapter, txo, i) {
2085                         cmpl = 0;
2086                         num_wrbs = 0;
2087                         txq = &txo->q;
2088                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2089                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2090                                 num_wrbs += be_tx_compl_process(adapter, txo,
2091                                                                 end_idx);
2092                                 cmpl++;
2093                         }
2094                         if (cmpl) {
2095                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2096                                 atomic_sub(num_wrbs, &txq->used);
2097                                 timeo = 0;
2098                         }
2099                         if (atomic_read(&txq->used) == 0)
2100                                 pending_txqs--;
2101                 }
2102
2103                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2104                         break;
2105
2106                 mdelay(1);
2107         } while (true);
2108
2109         for_all_tx_queues(adapter, txo, i) {
2110                 txq = &txo->q;
2111                 if (atomic_read(&txq->used))
2112                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2113                                 atomic_read(&txq->used));
2114
2115                 /* free posted tx for which compls will never arrive */
2116                 while (atomic_read(&txq->used)) {
2117                         sent_skb = txo->sent_skb_list[txq->tail];
2118                         end_idx = txq->tail;
2119                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2120                                                    &dummy_wrb);
2121                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2122                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2123                         atomic_sub(num_wrbs, &txq->used);
2124                 }
2125         }
2126 }
2127
2128 static void be_evt_queues_destroy(struct be_adapter *adapter)
2129 {
2130         struct be_eq_obj *eqo;
2131         int i;
2132
2133         for_all_evt_queues(adapter, eqo, i) {
2134                 if (eqo->q.created) {
2135                         be_eq_clean(eqo);
2136                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2137                         napi_hash_del(&eqo->napi);
2138                         netif_napi_del(&eqo->napi);
2139                 }
2140                 be_queue_free(adapter, &eqo->q);
2141         }
2142 }
2143
2144 static int be_evt_queues_create(struct be_adapter *adapter)
2145 {
2146         struct be_queue_info *eq;
2147         struct be_eq_obj *eqo;
2148         struct be_aic_obj *aic;
2149         int i, rc;
2150
2151         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2152                                     adapter->cfg_num_qs);
2153
2154         for_all_evt_queues(adapter, eqo, i) {
2155                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2156                                BE_NAPI_WEIGHT);
2157                 napi_hash_add(&eqo->napi);
2158                 aic = &adapter->aic_obj[i];
2159                 eqo->adapter = adapter;
2160                 eqo->idx = i;
2161                 aic->max_eqd = BE_MAX_EQD;
2162                 aic->enable = true;
2163
2164                 eq = &eqo->q;
2165                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2166                                     sizeof(struct be_eq_entry));
2167                 if (rc)
2168                         return rc;
2169
2170                 rc = be_cmd_eq_create(adapter, eqo);
2171                 if (rc)
2172                         return rc;
2173         }
2174         return 0;
2175 }
2176
2177 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2178 {
2179         struct be_queue_info *q;
2180
2181         q = &adapter->mcc_obj.q;
2182         if (q->created)
2183                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2184         be_queue_free(adapter, q);
2185
2186         q = &adapter->mcc_obj.cq;
2187         if (q->created)
2188                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2189         be_queue_free(adapter, q);
2190 }
2191
2192 /* Must be called only after TX qs are created as MCC shares TX EQ */
2193 static int be_mcc_queues_create(struct be_adapter *adapter)
2194 {
2195         struct be_queue_info *q, *cq;
2196
2197         cq = &adapter->mcc_obj.cq;
2198         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2199                            sizeof(struct be_mcc_compl)))
2200                 goto err;
2201
2202         /* Use the default EQ for MCC completions */
2203         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2204                 goto mcc_cq_free;
2205
2206         q = &adapter->mcc_obj.q;
2207         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2208                 goto mcc_cq_destroy;
2209
2210         if (be_cmd_mccq_create(adapter, q, cq))
2211                 goto mcc_q_free;
2212
2213         return 0;
2214
2215 mcc_q_free:
2216         be_queue_free(adapter, q);
2217 mcc_cq_destroy:
2218         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2219 mcc_cq_free:
2220         be_queue_free(adapter, cq);
2221 err:
2222         return -1;
2223 }
2224
2225 static void be_tx_queues_destroy(struct be_adapter *adapter)
2226 {
2227         struct be_queue_info *q;
2228         struct be_tx_obj *txo;
2229         u8 i;
2230
2231         for_all_tx_queues(adapter, txo, i) {
2232                 q = &txo->q;
2233                 if (q->created)
2234                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2235                 be_queue_free(adapter, q);
2236
2237                 q = &txo->cq;
2238                 if (q->created)
2239                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2240                 be_queue_free(adapter, q);
2241         }
2242 }
2243
2244 static int be_tx_qs_create(struct be_adapter *adapter)
2245 {
2246         struct be_queue_info *cq, *eq;
2247         struct be_tx_obj *txo;
2248         int status, i;
2249
2250         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2251
2252         for_all_tx_queues(adapter, txo, i) {
2253                 cq = &txo->cq;
2254                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2255                                         sizeof(struct be_eth_tx_compl));
2256                 if (status)
2257                         return status;
2258
2259                 u64_stats_init(&txo->stats.sync);
2260                 u64_stats_init(&txo->stats.sync_compl);
2261
2262                 /* If num_evt_qs is less than num_tx_qs, then more than
2263                  * one txq share an eq
2264                  */
2265                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2266                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2267                 if (status)
2268                         return status;
2269
2270                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2271                                         sizeof(struct be_eth_wrb));
2272                 if (status)
2273                         return status;
2274
2275                 status = be_cmd_txq_create(adapter, txo);
2276                 if (status)
2277                         return status;
2278         }
2279
2280         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2281                  adapter->num_tx_qs);
2282         return 0;
2283 }
2284
2285 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2286 {
2287         struct be_queue_info *q;
2288         struct be_rx_obj *rxo;
2289         int i;
2290
2291         for_all_rx_queues(adapter, rxo, i) {
2292                 q = &rxo->cq;
2293                 if (q->created)
2294                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2295                 be_queue_free(adapter, q);
2296         }
2297 }
2298
2299 static int be_rx_cqs_create(struct be_adapter *adapter)
2300 {
2301         struct be_queue_info *eq, *cq;
2302         struct be_rx_obj *rxo;
2303         int rc, i;
2304
2305         /* We can create as many RSS rings as there are EQs. */
2306         adapter->num_rx_qs = adapter->num_evt_qs;
2307
2308         /* We'll use RSS only if atleast 2 RSS rings are supported.
2309          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2310          */
2311         if (adapter->num_rx_qs > 1)
2312                 adapter->num_rx_qs++;
2313
2314         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2315         for_all_rx_queues(adapter, rxo, i) {
2316                 rxo->adapter = adapter;
2317                 cq = &rxo->cq;
2318                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2319                                     sizeof(struct be_eth_rx_compl));
2320                 if (rc)
2321                         return rc;
2322
2323                 u64_stats_init(&rxo->stats.sync);
2324                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2325                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2326                 if (rc)
2327                         return rc;
2328         }
2329
2330         dev_info(&adapter->pdev->dev,
2331                  "created %d RSS queue(s) and 1 default RX queue\n",
2332                  adapter->num_rx_qs - 1);
2333         return 0;
2334 }
2335
2336 static irqreturn_t be_intx(int irq, void *dev)
2337 {
2338         struct be_eq_obj *eqo = dev;
2339         struct be_adapter *adapter = eqo->adapter;
2340         int num_evts = 0;
2341
2342         /* IRQ is not expected when NAPI is scheduled as the EQ
2343          * will not be armed.
2344          * But, this can happen on Lancer INTx where it takes
2345          * a while to de-assert INTx or in BE2 where occasionaly
2346          * an interrupt may be raised even when EQ is unarmed.
2347          * If NAPI is already scheduled, then counting & notifying
2348          * events will orphan them.
2349          */
2350         if (napi_schedule_prep(&eqo->napi)) {
2351                 num_evts = events_get(eqo);
2352                 __napi_schedule(&eqo->napi);
2353                 if (num_evts)
2354                         eqo->spurious_intr = 0;
2355         }
2356         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2357
2358         /* Return IRQ_HANDLED only for the the first spurious intr
2359          * after a valid intr to stop the kernel from branding
2360          * this irq as a bad one!
2361          */
2362         if (num_evts || eqo->spurious_intr++ == 0)
2363                 return IRQ_HANDLED;
2364         else
2365                 return IRQ_NONE;
2366 }
2367
2368 static irqreturn_t be_msix(int irq, void *dev)
2369 {
2370         struct be_eq_obj *eqo = dev;
2371
2372         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2373         napi_schedule(&eqo->napi);
2374         return IRQ_HANDLED;
2375 }
2376
2377 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2378 {
2379         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2380 }
2381
2382 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2383                          int budget, int polling)
2384 {
2385         struct be_adapter *adapter = rxo->adapter;
2386         struct be_queue_info *rx_cq = &rxo->cq;
2387         struct be_rx_compl_info *rxcp;
2388         u32 work_done;
2389         u32 frags_consumed = 0;
2390
2391         for (work_done = 0; work_done < budget; work_done++) {
2392                 rxcp = be_rx_compl_get(rxo);
2393                 if (!rxcp)
2394                         break;
2395
2396                 /* Is it a flush compl that has no data */
2397                 if (unlikely(rxcp->num_rcvd == 0))
2398                         goto loop_continue;
2399
2400                 /* Discard compl with partial DMA Lancer B0 */
2401                 if (unlikely(!rxcp->pkt_size)) {
2402                         be_rx_compl_discard(rxo, rxcp);
2403                         goto loop_continue;
2404                 }
2405
2406                 /* On BE drop pkts that arrive due to imperfect filtering in
2407                  * promiscuous mode on some skews
2408                  */
2409                 if (unlikely(rxcp->port != adapter->port_num &&
2410                              !lancer_chip(adapter))) {
2411                         be_rx_compl_discard(rxo, rxcp);
2412                         goto loop_continue;
2413                 }
2414
2415                 /* Don't do gro when we're busy_polling */
2416                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2417                         be_rx_compl_process_gro(rxo, napi, rxcp);
2418                 else
2419                         be_rx_compl_process(rxo, napi, rxcp);
2420
2421 loop_continue:
2422                 frags_consumed += rxcp->num_rcvd;
2423                 be_rx_stats_update(rxo, rxcp);
2424         }
2425
2426         if (work_done) {
2427                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2428
2429                 /* When an rx-obj gets into post_starved state, just
2430                  * let be_worker do the posting.
2431                  */
2432                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2433                     !rxo->rx_post_starved)
2434                         be_post_rx_frags(rxo, GFP_ATOMIC,
2435                                          max_t(u32, MAX_RX_POST,
2436                                                frags_consumed));
2437         }
2438
2439         return work_done;
2440 }
2441
2442 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2443 {
2444         switch (status) {
2445         case BE_TX_COMP_HDR_PARSE_ERR:
2446                 tx_stats(txo)->tx_hdr_parse_err++;
2447                 break;
2448         case BE_TX_COMP_NDMA_ERR:
2449                 tx_stats(txo)->tx_dma_err++;
2450                 break;
2451         case BE_TX_COMP_ACL_ERR:
2452                 tx_stats(txo)->tx_spoof_check_err++;
2453                 break;
2454         }
2455 }
2456
2457 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2458 {
2459         switch (status) {
2460         case LANCER_TX_COMP_LSO_ERR:
2461                 tx_stats(txo)->tx_tso_err++;
2462                 break;
2463         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2464         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2465                 tx_stats(txo)->tx_spoof_check_err++;
2466                 break;
2467         case LANCER_TX_COMP_QINQ_ERR:
2468                 tx_stats(txo)->tx_qinq_err++;
2469                 break;
2470         case LANCER_TX_COMP_PARITY_ERR:
2471                 tx_stats(txo)->tx_internal_parity_err++;
2472                 break;
2473         case LANCER_TX_COMP_DMA_ERR:
2474                 tx_stats(txo)->tx_dma_err++;
2475                 break;
2476         }
2477 }
2478
2479 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2480                           int idx)
2481 {
2482         struct be_eth_tx_compl *txcp;
2483         int num_wrbs = 0, work_done = 0;
2484         u32 compl_status;
2485         u16 last_idx;
2486
2487         while ((txcp = be_tx_compl_get(&txo->cq))) {
2488                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2489                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2490                 work_done++;
2491
2492                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2493                 if (compl_status) {
2494                         if (lancer_chip(adapter))
2495                                 lancer_update_tx_err(txo, compl_status);
2496                         else
2497                                 be_update_tx_err(txo, compl_status);
2498                 }
2499         }
2500
2501         if (work_done) {
2502                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2503                 atomic_sub(num_wrbs, &txo->q.used);
2504
2505                 /* As Tx wrbs have been freed up, wake up netdev queue
2506                  * if it was stopped due to lack of tx wrbs.  */
2507                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2508                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2509                         netif_wake_subqueue(adapter->netdev, idx);
2510                 }
2511
2512                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2513                 tx_stats(txo)->tx_compl += work_done;
2514                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2515         }
2516 }
2517
2518 int be_poll(struct napi_struct *napi, int budget)
2519 {
2520         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2521         struct be_adapter *adapter = eqo->adapter;
2522         int max_work = 0, work, i, num_evts;
2523         struct be_rx_obj *rxo;
2524         struct be_tx_obj *txo;
2525
2526         num_evts = events_get(eqo);
2527
2528         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2529                 be_process_tx(adapter, txo, i);
2530
2531         if (be_lock_napi(eqo)) {
2532                 /* This loop will iterate twice for EQ0 in which
2533                  * completions of the last RXQ (default one) are also processed
2534                  * For other EQs the loop iterates only once
2535                  */
2536                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2537                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2538                         max_work = max(work, max_work);
2539                 }
2540                 be_unlock_napi(eqo);
2541         } else {
2542                 max_work = budget;
2543         }
2544
2545         if (is_mcc_eqo(eqo))
2546                 be_process_mcc(adapter);
2547
2548         if (max_work < budget) {
2549                 napi_complete(napi);
2550                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2551         } else {
2552                 /* As we'll continue in polling mode, count and clear events */
2553                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2554         }
2555         return max_work;
2556 }
2557
2558 #ifdef CONFIG_NET_RX_BUSY_POLL
2559 static int be_busy_poll(struct napi_struct *napi)
2560 {
2561         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2562         struct be_adapter *adapter = eqo->adapter;
2563         struct be_rx_obj *rxo;
2564         int i, work = 0;
2565
2566         if (!be_lock_busy_poll(eqo))
2567                 return LL_FLUSH_BUSY;
2568
2569         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2570                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2571                 if (work)
2572                         break;
2573         }
2574
2575         be_unlock_busy_poll(eqo);
2576         return work;
2577 }
2578 #endif
2579
2580 void be_detect_error(struct be_adapter *adapter)
2581 {
2582         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2583         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2584         u32 i;
2585         bool error_detected = false;
2586         struct device *dev = &adapter->pdev->dev;
2587         struct net_device *netdev = adapter->netdev;
2588
2589         if (be_hw_error(adapter))
2590                 return;
2591
2592         if (lancer_chip(adapter)) {
2593                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2594                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2595                         sliport_err1 = ioread32(adapter->db +
2596                                                 SLIPORT_ERROR1_OFFSET);
2597                         sliport_err2 = ioread32(adapter->db +
2598                                                 SLIPORT_ERROR2_OFFSET);
2599                         adapter->hw_error = true;
2600                         /* Do not log error messages if its a FW reset */
2601                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2602                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2603                                 dev_info(dev, "Firmware update in progress\n");
2604                         } else {
2605                                 error_detected = true;
2606                                 dev_err(dev, "Error detected in the card\n");
2607                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2608                                         sliport_status);
2609                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2610                                         sliport_err1);
2611                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2612                                         sliport_err2);
2613                         }
2614                 }
2615         } else {
2616                 pci_read_config_dword(adapter->pdev,
2617                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2618                 pci_read_config_dword(adapter->pdev,
2619                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2620                 pci_read_config_dword(adapter->pdev,
2621                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2622                 pci_read_config_dword(adapter->pdev,
2623                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2624
2625                 ue_lo = (ue_lo & ~ue_lo_mask);
2626                 ue_hi = (ue_hi & ~ue_hi_mask);
2627
2628                 /* On certain platforms BE hardware can indicate spurious UEs.
2629                  * Allow HW to stop working completely in case of a real UE.
2630                  * Hence not setting the hw_error for UE detection.
2631                  */
2632
2633                 if (ue_lo || ue_hi) {
2634                         error_detected = true;
2635                         dev_err(dev,
2636                                 "Unrecoverable Error detected in the adapter");
2637                         dev_err(dev, "Please reboot server to recover");
2638                         if (skyhawk_chip(adapter))
2639                                 adapter->hw_error = true;
2640                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2641                                 if (ue_lo & 1)
2642                                         dev_err(dev, "UE: %s bit set\n",
2643                                                 ue_status_low_desc[i]);
2644                         }
2645                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2646                                 if (ue_hi & 1)
2647                                         dev_err(dev, "UE: %s bit set\n",
2648                                                 ue_status_hi_desc[i]);
2649                         }
2650                 }
2651         }
2652         if (error_detected)
2653                 netif_carrier_off(netdev);
2654 }
2655
2656 static void be_msix_disable(struct be_adapter *adapter)
2657 {
2658         if (msix_enabled(adapter)) {
2659                 pci_disable_msix(adapter->pdev);
2660                 adapter->num_msix_vec = 0;
2661                 adapter->num_msix_roce_vec = 0;
2662         }
2663 }
2664
2665 static int be_msix_enable(struct be_adapter *adapter)
2666 {
2667         int i, num_vec;
2668         struct device *dev = &adapter->pdev->dev;
2669
2670         /* If RoCE is supported, program the max number of NIC vectors that
2671          * may be configured via set-channels, along with vectors needed for
2672          * RoCe. Else, just program the number we'll use initially.
2673          */
2674         if (be_roce_supported(adapter))
2675                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2676                                 2 * num_online_cpus());
2677         else
2678                 num_vec = adapter->cfg_num_qs;
2679
2680         for (i = 0; i < num_vec; i++)
2681                 adapter->msix_entries[i].entry = i;
2682
2683         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2684                                         MIN_MSIX_VECTORS, num_vec);
2685         if (num_vec < 0)
2686                 goto fail;
2687
2688         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2689                 adapter->num_msix_roce_vec = num_vec / 2;
2690                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2691                          adapter->num_msix_roce_vec);
2692         }
2693
2694         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2695
2696         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2697                  adapter->num_msix_vec);
2698         return 0;
2699
2700 fail:
2701         dev_warn(dev, "MSIx enable failed\n");
2702
2703         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2704         if (!be_physfn(adapter))
2705                 return num_vec;
2706         return 0;
2707 }
2708
2709 static inline int be_msix_vec_get(struct be_adapter *adapter,
2710                                   struct be_eq_obj *eqo)
2711 {
2712         return adapter->msix_entries[eqo->msix_idx].vector;
2713 }
2714
2715 static int be_msix_register(struct be_adapter *adapter)
2716 {
2717         struct net_device *netdev = adapter->netdev;
2718         struct be_eq_obj *eqo;
2719         int status, i, vec;
2720
2721         for_all_evt_queues(adapter, eqo, i) {
2722                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2723                 vec = be_msix_vec_get(adapter, eqo);
2724                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2725                 if (status)
2726                         goto err_msix;
2727         }
2728
2729         return 0;
2730 err_msix:
2731         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2732                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2733         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2734                  status);
2735         be_msix_disable(adapter);
2736         return status;
2737 }
2738
2739 static int be_irq_register(struct be_adapter *adapter)
2740 {
2741         struct net_device *netdev = adapter->netdev;
2742         int status;
2743
2744         if (msix_enabled(adapter)) {
2745                 status = be_msix_register(adapter);
2746                 if (status == 0)
2747                         goto done;
2748                 /* INTx is not supported for VF */
2749                 if (!be_physfn(adapter))
2750                         return status;
2751         }
2752
2753         /* INTx: only the first EQ is used */
2754         netdev->irq = adapter->pdev->irq;
2755         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2756                              &adapter->eq_obj[0]);
2757         if (status) {
2758                 dev_err(&adapter->pdev->dev,
2759                         "INTx request IRQ failed - err %d\n", status);
2760                 return status;
2761         }
2762 done:
2763         adapter->isr_registered = true;
2764         return 0;
2765 }
2766
2767 static void be_irq_unregister(struct be_adapter *adapter)
2768 {
2769         struct net_device *netdev = adapter->netdev;
2770         struct be_eq_obj *eqo;
2771         int i;
2772
2773         if (!adapter->isr_registered)
2774                 return;
2775
2776         /* INTx */
2777         if (!msix_enabled(adapter)) {
2778                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2779                 goto done;
2780         }
2781
2782         /* MSIx */
2783         for_all_evt_queues(adapter, eqo, i)
2784                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2785
2786 done:
2787         adapter->isr_registered = false;
2788 }
2789
2790 static void be_rx_qs_destroy(struct be_adapter *adapter)
2791 {
2792         struct be_queue_info *q;
2793         struct be_rx_obj *rxo;
2794         int i;
2795
2796         for_all_rx_queues(adapter, rxo, i) {
2797                 q = &rxo->q;
2798                 if (q->created) {
2799                         be_cmd_rxq_destroy(adapter, q);
2800                         be_rx_cq_clean(rxo);
2801                 }
2802                 be_queue_free(adapter, q);
2803         }
2804 }
2805
2806 static int be_close(struct net_device *netdev)
2807 {
2808         struct be_adapter *adapter = netdev_priv(netdev);
2809         struct be_eq_obj *eqo;
2810         int i;
2811
2812         /* This protection is needed as be_close() may be called even when the
2813          * adapter is in cleared state (after eeh perm failure)
2814          */
2815         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2816                 return 0;
2817
2818         be_roce_dev_close(adapter);
2819
2820         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2821                 for_all_evt_queues(adapter, eqo, i) {
2822                         napi_disable(&eqo->napi);
2823                         be_disable_busy_poll(eqo);
2824                 }
2825                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2826         }
2827
2828         be_async_mcc_disable(adapter);
2829
2830         /* Wait for all pending tx completions to arrive so that
2831          * all tx skbs are freed.
2832          */
2833         netif_tx_disable(netdev);
2834         be_tx_compl_clean(adapter);
2835
2836         be_rx_qs_destroy(adapter);
2837
2838         for (i = 1; i < (adapter->uc_macs + 1); i++)
2839                 be_cmd_pmac_del(adapter, adapter->if_handle,
2840                                 adapter->pmac_id[i], 0);
2841         adapter->uc_macs = 0;
2842
2843         for_all_evt_queues(adapter, eqo, i) {
2844                 if (msix_enabled(adapter))
2845                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2846                 else
2847                         synchronize_irq(netdev->irq);
2848                 be_eq_clean(eqo);
2849         }
2850
2851         be_irq_unregister(adapter);
2852
2853         return 0;
2854 }
2855
2856 static int be_rx_qs_create(struct be_adapter *adapter)
2857 {
2858         struct rss_info *rss = &adapter->rss_info;
2859         u8 rss_key[RSS_HASH_KEY_LEN];
2860         struct be_rx_obj *rxo;
2861         int rc, i, j;
2862
2863         for_all_rx_queues(adapter, rxo, i) {
2864                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2865                                     sizeof(struct be_eth_rx_d));
2866                 if (rc)
2867                         return rc;
2868         }
2869
2870         /* The FW would like the default RXQ to be created first */
2871         rxo = default_rxo(adapter);
2872         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2873                                adapter->if_handle, false, &rxo->rss_id);
2874         if (rc)
2875                 return rc;
2876
2877         for_all_rss_queues(adapter, rxo, i) {
2878                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2879                                        rx_frag_size, adapter->if_handle,
2880                                        true, &rxo->rss_id);
2881                 if (rc)
2882                         return rc;
2883         }
2884
2885         if (be_multi_rxq(adapter)) {
2886                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2887                         j += adapter->num_rx_qs - 1) {
2888                         for_all_rss_queues(adapter, rxo, i) {
2889                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2890                                         break;
2891                                 rss->rsstable[j + i] = rxo->rss_id;
2892                                 rss->rss_queue[j + i] = i;
2893                         }
2894                 }
2895                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2896                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2897
2898                 if (!BEx_chip(adapter))
2899                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2900                                 RSS_ENABLE_UDP_IPV6;
2901         } else {
2902                 /* Disable RSS, if only default RX Q is created */
2903                 rss->rss_flags = RSS_ENABLE_NONE;
2904         }
2905
2906         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
2907         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2908                                128, rss_key);
2909         if (rc) {
2910                 rss->rss_flags = RSS_ENABLE_NONE;
2911                 return rc;
2912         }
2913
2914         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
2915
2916         /* First time posting */
2917         for_all_rx_queues(adapter, rxo, i)
2918                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2919         return 0;
2920 }
2921
2922 static int be_open(struct net_device *netdev)
2923 {
2924         struct be_adapter *adapter = netdev_priv(netdev);
2925         struct be_eq_obj *eqo;
2926         struct be_rx_obj *rxo;
2927         struct be_tx_obj *txo;
2928         u8 link_status;
2929         int status, i;
2930
2931         status = be_rx_qs_create(adapter);
2932         if (status)
2933                 goto err;
2934
2935         status = be_irq_register(adapter);
2936         if (status)
2937                 goto err;
2938
2939         for_all_rx_queues(adapter, rxo, i)
2940                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2941
2942         for_all_tx_queues(adapter, txo, i)
2943                 be_cq_notify(adapter, txo->cq.id, true, 0);
2944
2945         be_async_mcc_enable(adapter);
2946
2947         for_all_evt_queues(adapter, eqo, i) {
2948                 napi_enable(&eqo->napi);
2949                 be_enable_busy_poll(eqo);
2950                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2951         }
2952         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2953
2954         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2955         if (!status)
2956                 be_link_status_update(adapter, link_status);
2957
2958         netif_tx_start_all_queues(netdev);
2959         be_roce_dev_open(adapter);
2960
2961 #ifdef CONFIG_BE2NET_VXLAN
2962         if (skyhawk_chip(adapter))
2963                 vxlan_get_rx_port(netdev);
2964 #endif
2965
2966         return 0;
2967 err:
2968         be_close(adapter->netdev);
2969         return -EIO;
2970 }
2971
2972 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2973 {
2974         struct be_dma_mem cmd;
2975         int status = 0;
2976         u8 mac[ETH_ALEN];
2977
2978         memset(mac, 0, ETH_ALEN);
2979
2980         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2981         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2982                                      GFP_KERNEL);
2983         if (!cmd.va)
2984                 return -ENOMEM;
2985
2986         if (enable) {
2987                 status = pci_write_config_dword(adapter->pdev,
2988                                                 PCICFG_PM_CONTROL_OFFSET,
2989                                                 PCICFG_PM_CONTROL_MASK);
2990                 if (status) {
2991                         dev_err(&adapter->pdev->dev,
2992                                 "Could not enable Wake-on-lan\n");
2993                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2994                                           cmd.dma);
2995                         return status;
2996                 }
2997                 status = be_cmd_enable_magic_wol(adapter,
2998                                                  adapter->netdev->dev_addr,
2999                                                  &cmd);
3000                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3001                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3002         } else {
3003                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3004                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3005                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3006         }
3007
3008         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3009         return status;
3010 }
3011
3012 /*
3013  * Generate a seed MAC address from the PF MAC Address using jhash.
3014  * MAC Address for VFs are assigned incrementally starting from the seed.
3015  * These addresses are programmed in the ASIC by the PF and the VF driver
3016  * queries for the MAC address during its probe.
3017  */
3018 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3019 {
3020         u32 vf;
3021         int status = 0;
3022         u8 mac[ETH_ALEN];
3023         struct be_vf_cfg *vf_cfg;
3024
3025         be_vf_eth_addr_generate(adapter, mac);
3026
3027         for_all_vfs(adapter, vf_cfg, vf) {
3028                 if (BEx_chip(adapter))
3029                         status = be_cmd_pmac_add(adapter, mac,
3030                                                  vf_cfg->if_handle,
3031                                                  &vf_cfg->pmac_id, vf + 1);
3032                 else
3033                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3034                                                 vf + 1);
3035
3036                 if (status)
3037                         dev_err(&adapter->pdev->dev,
3038                                 "Mac address assignment failed for VF %d\n",
3039                                 vf);
3040                 else
3041                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3042
3043                 mac[5] += 1;
3044         }
3045         return status;
3046 }
3047
3048 static int be_vfs_mac_query(struct be_adapter *adapter)
3049 {
3050         int status, vf;
3051         u8 mac[ETH_ALEN];
3052         struct be_vf_cfg *vf_cfg;
3053
3054         for_all_vfs(adapter, vf_cfg, vf) {
3055                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3056                                                mac, vf_cfg->if_handle,
3057                                                false, vf+1);
3058                 if (status)
3059                         return status;
3060                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3061         }
3062         return 0;
3063 }
3064
3065 static void be_vf_clear(struct be_adapter *adapter)
3066 {
3067         struct be_vf_cfg *vf_cfg;
3068         u32 vf;
3069
3070         if (pci_vfs_assigned(adapter->pdev)) {
3071                 dev_warn(&adapter->pdev->dev,
3072                          "VFs are assigned to VMs: not disabling VFs\n");
3073                 goto done;
3074         }
3075
3076         pci_disable_sriov(adapter->pdev);
3077
3078         for_all_vfs(adapter, vf_cfg, vf) {
3079                 if (BEx_chip(adapter))
3080                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3081                                         vf_cfg->pmac_id, vf + 1);
3082                 else
3083                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3084                                        vf + 1);
3085
3086                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3087         }
3088 done:
3089         kfree(adapter->vf_cfg);
3090         adapter->num_vfs = 0;
3091         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3092 }
3093
3094 static void be_clear_queues(struct be_adapter *adapter)
3095 {
3096         be_mcc_queues_destroy(adapter);
3097         be_rx_cqs_destroy(adapter);
3098         be_tx_queues_destroy(adapter);
3099         be_evt_queues_destroy(adapter);
3100 }
3101
3102 static void be_cancel_worker(struct be_adapter *adapter)
3103 {
3104         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3105                 cancel_delayed_work_sync(&adapter->work);
3106                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3107         }
3108 }
3109
3110 static void be_mac_clear(struct be_adapter *adapter)
3111 {
3112         int i;
3113
3114         if (adapter->pmac_id) {
3115                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3116                         be_cmd_pmac_del(adapter, adapter->if_handle,
3117                                         adapter->pmac_id[i], 0);
3118                 adapter->uc_macs = 0;
3119
3120                 kfree(adapter->pmac_id);
3121                 adapter->pmac_id = NULL;
3122         }
3123 }
3124
3125 #ifdef CONFIG_BE2NET_VXLAN
3126 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3127 {
3128         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3129                 be_cmd_manage_iface(adapter, adapter->if_handle,
3130                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3131
3132         if (adapter->vxlan_port)
3133                 be_cmd_set_vxlan_port(adapter, 0);
3134
3135         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3136         adapter->vxlan_port = 0;
3137 }
3138 #endif
3139
3140 static int be_clear(struct be_adapter *adapter)
3141 {
3142         be_cancel_worker(adapter);
3143
3144         if (sriov_enabled(adapter))
3145                 be_vf_clear(adapter);
3146
3147         /* Re-configure FW to distribute resources evenly across max-supported
3148          * number of VFs, only when VFs are not already enabled.
3149          */
3150         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3151                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3152                                         pci_sriov_get_totalvfs(adapter->pdev));
3153
3154 #ifdef CONFIG_BE2NET_VXLAN
3155         be_disable_vxlan_offloads(adapter);
3156 #endif
3157         /* delete the primary mac along with the uc-mac list */
3158         be_mac_clear(adapter);
3159
3160         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3161
3162         be_clear_queues(adapter);
3163
3164         be_msix_disable(adapter);
3165         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3166         return 0;
3167 }
3168
3169 static int be_vfs_if_create(struct be_adapter *adapter)
3170 {
3171         struct be_resources res = {0};
3172         struct be_vf_cfg *vf_cfg;
3173         u32 cap_flags, en_flags, vf;
3174         int status = 0;
3175
3176         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3177                     BE_IF_FLAGS_MULTICAST;
3178
3179         for_all_vfs(adapter, vf_cfg, vf) {
3180                 if (!BE3_chip(adapter)) {
3181                         status = be_cmd_get_profile_config(adapter, &res,
3182                                                            vf + 1);
3183                         if (!status)
3184                                 cap_flags = res.if_cap_flags;
3185                 }
3186
3187                 /* If a FW profile exists, then cap_flags are updated */
3188                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3189                                         BE_IF_FLAGS_BROADCAST |
3190                                         BE_IF_FLAGS_MULTICAST);
3191                 status =
3192                     be_cmd_if_create(adapter, cap_flags, en_flags,
3193                                      &vf_cfg->if_handle, vf + 1);
3194                 if (status)
3195                         goto err;
3196         }
3197 err:
3198         return status;
3199 }
3200
3201 static int be_vf_setup_init(struct be_adapter *adapter)
3202 {
3203         struct be_vf_cfg *vf_cfg;
3204         int vf;
3205
3206         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3207                                   GFP_KERNEL);
3208         if (!adapter->vf_cfg)
3209                 return -ENOMEM;
3210
3211         for_all_vfs(adapter, vf_cfg, vf) {
3212                 vf_cfg->if_handle = -1;
3213                 vf_cfg->pmac_id = -1;
3214         }
3215         return 0;
3216 }
3217
3218 static int be_vf_setup(struct be_adapter *adapter)
3219 {
3220         struct device *dev = &adapter->pdev->dev;
3221         struct be_vf_cfg *vf_cfg;
3222         int status, old_vfs, vf;
3223         u32 privileges;
3224
3225         old_vfs = pci_num_vf(adapter->pdev);
3226
3227         status = be_vf_setup_init(adapter);
3228         if (status)
3229                 goto err;
3230
3231         if (old_vfs) {
3232                 for_all_vfs(adapter, vf_cfg, vf) {
3233                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3234                         if (status)
3235                                 goto err;
3236                 }
3237
3238                 status = be_vfs_mac_query(adapter);
3239                 if (status)
3240                         goto err;
3241         } else {
3242                 status = be_vfs_if_create(adapter);
3243                 if (status)
3244                         goto err;
3245
3246                 status = be_vf_eth_addr_config(adapter);
3247                 if (status)
3248                         goto err;
3249         }
3250
3251         for_all_vfs(adapter, vf_cfg, vf) {
3252                 /* Allow VFs to programs MAC/VLAN filters */
3253                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3254                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3255                         status = be_cmd_set_fn_privileges(adapter,
3256                                                           privileges |
3257                                                           BE_PRIV_FILTMGMT,
3258                                                           vf + 1);
3259                         if (!status)
3260                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3261                                          vf);
3262                 }
3263
3264                 /* Allow full available bandwidth */
3265                 if (!old_vfs)
3266                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3267
3268                 if (!old_vfs) {
3269                         be_cmd_enable_vf(adapter, vf + 1);
3270                         be_cmd_set_logical_link_config(adapter,
3271                                                        IFLA_VF_LINK_STATE_AUTO,
3272                                                        vf+1);
3273                 }
3274         }
3275
3276         if (!old_vfs) {
3277                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3278                 if (status) {
3279                         dev_err(dev, "SRIOV enable failed\n");
3280                         adapter->num_vfs = 0;
3281                         goto err;
3282                 }
3283         }
3284
3285         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3286         return 0;
3287 err:
3288         dev_err(dev, "VF setup failed\n");
3289         be_vf_clear(adapter);
3290         return status;
3291 }
3292
3293 /* Converting function_mode bits on BE3 to SH mc_type enums */
3294
3295 static u8 be_convert_mc_type(u32 function_mode)
3296 {
3297         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3298                 return vNIC1;
3299         else if (function_mode & QNQ_MODE)
3300                 return FLEX10;
3301         else if (function_mode & VNIC_MODE)
3302                 return vNIC2;
3303         else if (function_mode & UMC_ENABLED)
3304                 return UMC;
3305         else
3306                 return MC_NONE;
3307 }
3308
3309 /* On BE2/BE3 FW does not suggest the supported limits */
3310 static void BEx_get_resources(struct be_adapter *adapter,
3311                               struct be_resources *res)
3312 {
3313         bool use_sriov = adapter->num_vfs ? 1 : 0;
3314
3315         if (be_physfn(adapter))
3316                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3317         else
3318                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3319
3320         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3321
3322         if (be_is_mc(adapter)) {
3323                 /* Assuming that there are 4 channels per port,
3324                  * when multi-channel is enabled
3325                  */
3326                 if (be_is_qnq_mode(adapter))
3327                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3328                 else
3329                         /* In a non-qnq multichannel mode, the pvid
3330                          * takes up one vlan entry
3331                          */
3332                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3333         } else {
3334                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3335         }
3336
3337         res->max_mcast_mac = BE_MAX_MC;
3338
3339         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3340          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3341          *    *only* if it is RSS-capable.
3342          */
3343         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3344             !be_physfn(adapter) || (be_is_mc(adapter) &&
3345             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3346                 res->max_tx_qs = 1;
3347         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3348                 struct be_resources super_nic_res = {0};
3349
3350                 /* On a SuperNIC profile, the driver needs to use the
3351                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3352                  */
3353                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3354                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3355                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3356         } else {
3357                 res->max_tx_qs = BE3_MAX_TX_QS;
3358         }
3359
3360         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3361             !use_sriov && be_physfn(adapter))
3362                 res->max_rss_qs = (adapter->be3_native) ?
3363                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3364         res->max_rx_qs = res->max_rss_qs + 1;
3365
3366         if (be_physfn(adapter))
3367                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3368                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3369         else
3370                 res->max_evt_qs = 1;
3371
3372         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3373         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3374                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3375 }
3376
3377 static void be_setup_init(struct be_adapter *adapter)
3378 {
3379         adapter->vlan_prio_bmap = 0xff;
3380         adapter->phy.link_speed = -1;
3381         adapter->if_handle = -1;
3382         adapter->be3_native = false;
3383         adapter->promiscuous = false;
3384         if (be_physfn(adapter))
3385                 adapter->cmd_privileges = MAX_PRIVILEGES;
3386         else
3387                 adapter->cmd_privileges = MIN_PRIVILEGES;
3388 }
3389
3390 static int be_get_sriov_config(struct be_adapter *adapter)
3391 {
3392         struct device *dev = &adapter->pdev->dev;
3393         struct be_resources res = {0};
3394         int max_vfs, old_vfs;
3395
3396         /* Some old versions of BE3 FW don't report max_vfs value */
3397         be_cmd_get_profile_config(adapter, &res, 0);
3398
3399         if (BE3_chip(adapter) && !res.max_vfs) {
3400                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3401                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3402         }
3403
3404         adapter->pool_res = res;
3405
3406         if (!be_max_vfs(adapter)) {
3407                 if (num_vfs)
3408                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3409                 adapter->num_vfs = 0;
3410                 return 0;
3411         }
3412
3413         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3414
3415         /* validate num_vfs module param */
3416         old_vfs = pci_num_vf(adapter->pdev);
3417         if (old_vfs) {
3418                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3419                 if (old_vfs != num_vfs)
3420                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3421                 adapter->num_vfs = old_vfs;
3422         } else {
3423                 if (num_vfs > be_max_vfs(adapter)) {
3424                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3425                                  num_vfs);
3426                         dev_info(dev, "Limiting to %d VFs\n",
3427                                  be_max_vfs(adapter));
3428                 }
3429                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3430         }
3431
3432         return 0;
3433 }
3434
3435 static int be_get_resources(struct be_adapter *adapter)
3436 {
3437         struct device *dev = &adapter->pdev->dev;
3438         struct be_resources res = {0};
3439         int status;
3440
3441         if (BEx_chip(adapter)) {
3442                 BEx_get_resources(adapter, &res);
3443                 adapter->res = res;
3444         }
3445
3446         /* For Lancer, SH etc read per-function resource limits from FW.
3447          * GET_FUNC_CONFIG returns per function guaranteed limits.
3448          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3449          */
3450         if (!BEx_chip(adapter)) {
3451                 status = be_cmd_get_func_config(adapter, &res);
3452                 if (status)
3453                         return status;
3454
3455                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3456                 if (be_roce_supported(adapter))
3457                         res.max_evt_qs /= 2;
3458                 adapter->res = res;
3459         }
3460
3461         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3462                  be_max_txqs(adapter), be_max_rxqs(adapter),
3463                  be_max_rss(adapter), be_max_eqs(adapter),
3464                  be_max_vfs(adapter));
3465         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3466                  be_max_uc(adapter), be_max_mc(adapter),
3467                  be_max_vlans(adapter));
3468
3469         return 0;
3470 }
3471
3472 static void be_sriov_config(struct be_adapter *adapter)
3473 {
3474         struct device *dev = &adapter->pdev->dev;
3475         int status;
3476
3477         status = be_get_sriov_config(adapter);
3478         if (status) {
3479                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3480                 dev_err(dev, "SR-IOV cannot be enabled\n");
3481                 return;
3482         }
3483
3484         /* When the HW is in SRIOV capable configuration, the PF-pool
3485          * resources are equally distributed across the max-number of
3486          * VFs. The user may request only a subset of the max-vfs to be
3487          * enabled. Based on num_vfs, redistribute the resources across
3488          * num_vfs so that each VF will have access to more number of
3489          * resources. This facility is not available in BE3 FW.
3490          * Also, this is done by FW in Lancer chip.
3491          */
3492         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3493                 status = be_cmd_set_sriov_config(adapter,
3494                                                  adapter->pool_res,
3495                                                  adapter->num_vfs);
3496                 if (status)
3497                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3498         }
3499 }
3500
3501 static int be_get_config(struct be_adapter *adapter)
3502 {
3503         u16 profile_id;
3504         int status;
3505
3506         status = be_cmd_query_fw_cfg(adapter);
3507         if (status)
3508                 return status;
3509
3510          if (be_physfn(adapter)) {
3511                 status = be_cmd_get_active_profile(adapter, &profile_id);
3512                 if (!status)
3513                         dev_info(&adapter->pdev->dev,
3514                                  "Using profile 0x%x\n", profile_id);
3515         }
3516
3517         if (!BE2_chip(adapter) && be_physfn(adapter))
3518                 be_sriov_config(adapter);
3519
3520         status = be_get_resources(adapter);
3521         if (status)
3522                 return status;
3523
3524         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3525                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3526         if (!adapter->pmac_id)
3527                 return -ENOMEM;
3528
3529         /* Sanitize cfg_num_qs based on HW and platform limits */
3530         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3531
3532         return 0;
3533 }
3534
3535 static int be_mac_setup(struct be_adapter *adapter)
3536 {
3537         u8 mac[ETH_ALEN];
3538         int status;
3539
3540         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3541                 status = be_cmd_get_perm_mac(adapter, mac);
3542                 if (status)
3543                         return status;
3544
3545                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3546                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3547         } else {
3548                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3549                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3550         }
3551
3552         /* For BE3-R VFs, the PF programs the initial MAC address */
3553         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3554                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3555                                 &adapter->pmac_id[0], 0);
3556         return 0;
3557 }
3558
3559 static void be_schedule_worker(struct be_adapter *adapter)
3560 {
3561         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3562         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3563 }
3564
3565 static int be_setup_queues(struct be_adapter *adapter)
3566 {
3567         struct net_device *netdev = adapter->netdev;
3568         int status;
3569
3570         status = be_evt_queues_create(adapter);
3571         if (status)
3572                 goto err;
3573
3574         status = be_tx_qs_create(adapter);
3575         if (status)
3576                 goto err;
3577
3578         status = be_rx_cqs_create(adapter);
3579         if (status)
3580                 goto err;
3581
3582         status = be_mcc_queues_create(adapter);
3583         if (status)
3584                 goto err;
3585
3586         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3587         if (status)
3588                 goto err;
3589
3590         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3591         if (status)
3592                 goto err;
3593
3594         return 0;
3595 err:
3596         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3597         return status;
3598 }
3599
3600 int be_update_queues(struct be_adapter *adapter)
3601 {
3602         struct net_device *netdev = adapter->netdev;
3603         int status;
3604
3605         if (netif_running(netdev))
3606                 be_close(netdev);
3607
3608         be_cancel_worker(adapter);
3609
3610         /* If any vectors have been shared with RoCE we cannot re-program
3611          * the MSIx table.
3612          */
3613         if (!adapter->num_msix_roce_vec)
3614                 be_msix_disable(adapter);
3615
3616         be_clear_queues(adapter);
3617
3618         if (!msix_enabled(adapter)) {
3619                 status = be_msix_enable(adapter);
3620                 if (status)
3621                         return status;
3622         }
3623
3624         status = be_setup_queues(adapter);
3625         if (status)
3626                 return status;
3627
3628         be_schedule_worker(adapter);
3629
3630         if (netif_running(netdev))
3631                 status = be_open(netdev);
3632
3633         return status;
3634 }
3635
3636 static int be_setup(struct be_adapter *adapter)
3637 {
3638         struct device *dev = &adapter->pdev->dev;
3639         u32 tx_fc, rx_fc, en_flags;
3640         int status;
3641
3642         be_setup_init(adapter);
3643
3644         if (!lancer_chip(adapter))
3645                 be_cmd_req_native_mode(adapter);
3646
3647         status = be_get_config(adapter);
3648         if (status)
3649                 goto err;
3650
3651         status = be_msix_enable(adapter);
3652         if (status)
3653                 goto err;
3654
3655         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3656                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3657         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3658                 en_flags |= BE_IF_FLAGS_RSS;
3659         en_flags = en_flags & be_if_cap_flags(adapter);
3660         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3661                                   &adapter->if_handle, 0);
3662         if (status)
3663                 goto err;
3664
3665         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3666         rtnl_lock();
3667         status = be_setup_queues(adapter);
3668         rtnl_unlock();
3669         if (status)
3670                 goto err;
3671
3672         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3673
3674         status = be_mac_setup(adapter);
3675         if (status)
3676                 goto err;
3677
3678         be_cmd_get_fw_ver(adapter);
3679         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3680
3681         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3682                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3683                         adapter->fw_ver);
3684                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3685         }
3686
3687         if (adapter->vlans_added)
3688                 be_vid_config(adapter);
3689
3690         be_set_rx_mode(adapter->netdev);
3691
3692         be_cmd_get_acpi_wol_cap(adapter);
3693
3694         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3695
3696         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3697                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3698                                         adapter->rx_fc);
3699
3700         if (be_physfn(adapter))
3701                 be_cmd_set_logical_link_config(adapter,
3702                                                IFLA_VF_LINK_STATE_AUTO, 0);
3703
3704         if (adapter->num_vfs)
3705                 be_vf_setup(adapter);
3706
3707         status = be_cmd_get_phy_info(adapter);
3708         if (!status && be_pause_supported(adapter))
3709                 adapter->phy.fc_autoneg = 1;
3710
3711         be_schedule_worker(adapter);
3712         adapter->flags |= BE_FLAGS_SETUP_DONE;
3713         return 0;
3714 err:
3715         be_clear(adapter);
3716         return status;
3717 }
3718
3719 #ifdef CONFIG_NET_POLL_CONTROLLER
3720 static void be_netpoll(struct net_device *netdev)
3721 {
3722         struct be_adapter *adapter = netdev_priv(netdev);
3723         struct be_eq_obj *eqo;
3724         int i;
3725
3726         for_all_evt_queues(adapter, eqo, i) {
3727                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3728                 napi_schedule(&eqo->napi);
3729         }
3730 }
3731 #endif
3732
3733 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3734
3735 static bool phy_flashing_required(struct be_adapter *adapter)
3736 {
3737         return (adapter->phy.phy_type == TN_8022 &&
3738                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3739 }
3740
3741 static bool is_comp_in_ufi(struct be_adapter *adapter,
3742                            struct flash_section_info *fsec, int type)
3743 {
3744         int i = 0, img_type = 0;
3745         struct flash_section_info_g2 *fsec_g2 = NULL;
3746
3747         if (BE2_chip(adapter))
3748                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3749
3750         for (i = 0; i < MAX_FLASH_COMP; i++) {
3751                 if (fsec_g2)
3752                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3753                 else
3754                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3755
3756                 if (img_type == type)
3757                         return true;
3758         }
3759         return false;
3760
3761 }
3762
3763 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3764                                                 int header_size,
3765                                                 const struct firmware *fw)
3766 {
3767         struct flash_section_info *fsec = NULL;
3768         const u8 *p = fw->data;
3769
3770         p += header_size;
3771         while (p < (fw->data + fw->size)) {
3772                 fsec = (struct flash_section_info *)p;
3773                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3774                         return fsec;
3775                 p += 32;
3776         }
3777         return NULL;
3778 }
3779
3780 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3781                               u32 img_offset, u32 img_size, int hdr_size,
3782                               u16 img_optype, bool *crc_match)
3783 {
3784         u32 crc_offset;
3785         int status;
3786         u8 crc[4];
3787
3788         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3789         if (status)
3790                 return status;
3791
3792         crc_offset = hdr_size + img_offset + img_size - 4;
3793
3794         /* Skip flashing, if crc of flashed region matches */
3795         if (!memcmp(crc, p + crc_offset, 4))
3796                 *crc_match = true;
3797         else
3798                 *crc_match = false;
3799
3800         return status;
3801 }
3802
3803 static int be_flash(struct be_adapter *adapter, const u8 *img,
3804                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3805 {
3806         struct be_cmd_write_flashrom *req = flash_cmd->va;
3807         u32 total_bytes, flash_op, num_bytes;
3808         int status;
3809
3810         total_bytes = img_size;
3811         while (total_bytes) {
3812                 num_bytes = min_t(u32, 32*1024, total_bytes);
3813
3814                 total_bytes -= num_bytes;
3815
3816                 if (!total_bytes) {
3817                         if (optype == OPTYPE_PHY_FW)
3818                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3819                         else
3820                                 flash_op = FLASHROM_OPER_FLASH;
3821                 } else {
3822                         if (optype == OPTYPE_PHY_FW)
3823                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3824                         else
3825                                 flash_op = FLASHROM_OPER_SAVE;
3826                 }
3827
3828                 memcpy(req->data_buf, img, num_bytes);
3829                 img += num_bytes;
3830                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3831                                                flash_op, num_bytes);
3832                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3833                     optype == OPTYPE_PHY_FW)
3834                         break;
3835                 else if (status)
3836                         return status;
3837         }
3838         return 0;
3839 }
3840
3841 /* For BE2, BE3 and BE3-R */
3842 static int be_flash_BEx(struct be_adapter *adapter,
3843                         const struct firmware *fw,
3844                         struct be_dma_mem *flash_cmd, int num_of_images)
3845 {
3846         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3847         struct device *dev = &adapter->pdev->dev;
3848         struct flash_section_info *fsec = NULL;
3849         int status, i, filehdr_size, num_comp;
3850         const struct flash_comp *pflashcomp;
3851         bool crc_match;
3852         const u8 *p;
3853
3854         struct flash_comp gen3_flash_types[] = {
3855                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3856                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3857                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3858                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3859                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3860                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3861                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3862                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3863                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3864                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3865                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3866                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3867                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3868                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3869                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3870                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3871                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3872                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3873                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3874                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3875         };
3876
3877         struct flash_comp gen2_flash_types[] = {
3878                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3879                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3880                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3881                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3882                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3883                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3884                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3885                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3886                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3887                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3888                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3889                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3890                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3891                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3892                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3893                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3894         };
3895
3896         if (BE3_chip(adapter)) {
3897                 pflashcomp = gen3_flash_types;
3898                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3899                 num_comp = ARRAY_SIZE(gen3_flash_types);
3900         } else {
3901                 pflashcomp = gen2_flash_types;
3902                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3903                 num_comp = ARRAY_SIZE(gen2_flash_types);
3904         }
3905
3906         /* Get flash section info*/
3907         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3908         if (!fsec) {
3909                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3910                 return -1;
3911         }
3912         for (i = 0; i < num_comp; i++) {
3913                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3914                         continue;
3915
3916                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3917                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3918                         continue;
3919
3920                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3921                     !phy_flashing_required(adapter))
3922                                 continue;
3923
3924                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3925                         status = be_check_flash_crc(adapter, fw->data,
3926                                                     pflashcomp[i].offset,
3927                                                     pflashcomp[i].size,
3928                                                     filehdr_size +
3929                                                     img_hdrs_size,
3930                                                     OPTYPE_REDBOOT, &crc_match);
3931                         if (status) {
3932                                 dev_err(dev,
3933                                         "Could not get CRC for 0x%x region\n",
3934                                         pflashcomp[i].optype);
3935                                 continue;
3936                         }
3937
3938                         if (crc_match)
3939                                 continue;
3940                 }
3941
3942                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3943                         img_hdrs_size;
3944                 if (p + pflashcomp[i].size > fw->data + fw->size)
3945                         return -1;
3946
3947                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3948                                   pflashcomp[i].size);
3949                 if (status) {
3950                         dev_err(dev, "Flashing section type 0x%x failed\n",
3951                                 pflashcomp[i].img_type);
3952                         return status;
3953                 }
3954         }
3955         return 0;
3956 }
3957
3958 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3959 {
3960         u32 img_type = le32_to_cpu(fsec_entry.type);
3961         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3962
3963         if (img_optype != 0xFFFF)
3964                 return img_optype;
3965
3966         switch (img_type) {
3967         case IMAGE_FIRMWARE_iSCSI:
3968                 img_optype = OPTYPE_ISCSI_ACTIVE;
3969                 break;
3970         case IMAGE_BOOT_CODE:
3971                 img_optype = OPTYPE_REDBOOT;
3972                 break;
3973         case IMAGE_OPTION_ROM_ISCSI:
3974                 img_optype = OPTYPE_BIOS;
3975                 break;
3976         case IMAGE_OPTION_ROM_PXE:
3977                 img_optype = OPTYPE_PXE_BIOS;
3978                 break;
3979         case IMAGE_OPTION_ROM_FCoE:
3980                 img_optype = OPTYPE_FCOE_BIOS;
3981                 break;
3982         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3983                 img_optype = OPTYPE_ISCSI_BACKUP;
3984                 break;
3985         case IMAGE_NCSI:
3986                 img_optype = OPTYPE_NCSI_FW;
3987                 break;
3988         case IMAGE_FLASHISM_JUMPVECTOR:
3989                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3990                 break;
3991         case IMAGE_FIRMWARE_PHY:
3992                 img_optype = OPTYPE_SH_PHY_FW;
3993                 break;
3994         case IMAGE_REDBOOT_DIR:
3995                 img_optype = OPTYPE_REDBOOT_DIR;
3996                 break;
3997         case IMAGE_REDBOOT_CONFIG:
3998                 img_optype = OPTYPE_REDBOOT_CONFIG;
3999                 break;
4000         case IMAGE_UFI_DIR:
4001                 img_optype = OPTYPE_UFI_DIR;
4002                 break;
4003         default:
4004                 break;
4005         }
4006
4007         return img_optype;
4008 }
4009
4010 static int be_flash_skyhawk(struct be_adapter *adapter,
4011                             const struct firmware *fw,
4012                             struct be_dma_mem *flash_cmd, int num_of_images)
4013 {
4014         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4015         struct device *dev = &adapter->pdev->dev;
4016         struct flash_section_info *fsec = NULL;
4017         u32 img_offset, img_size, img_type;
4018         int status, i, filehdr_size;
4019         bool crc_match, old_fw_img;
4020         u16 img_optype;
4021         const u8 *p;
4022
4023         filehdr_size = sizeof(struct flash_file_hdr_g3);
4024         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4025         if (!fsec) {
4026                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4027                 return -EINVAL;
4028         }
4029
4030         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4031                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4032                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4033                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4034                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4035                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4036
4037                 if (img_optype == 0xFFFF)
4038                         continue;
4039                 /* Don't bother verifying CRC if an old FW image is being
4040                  * flashed
4041                  */
4042                 if (old_fw_img)
4043                         goto flash;
4044
4045                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4046                                             img_size, filehdr_size +
4047                                             img_hdrs_size, img_optype,
4048                                             &crc_match);
4049                 /* The current FW image on the card does not recognize the new
4050                  * FLASH op_type. The FW download is partially complete.
4051                  * Reboot the server now to enable FW image to recognize the
4052                  * new FLASH op_type. To complete the remaining process,
4053                  * download the same FW again after the reboot.
4054                  */
4055                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4056                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4057                         dev_err(dev, "Flash incomplete. Reset the server\n");
4058                         dev_err(dev, "Download FW image again after reset\n");
4059                         return -EAGAIN;
4060                 } else if (status) {
4061                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4062                                 img_optype);
4063                         return -EFAULT;
4064                 }
4065
4066                 if (crc_match)
4067                         continue;
4068
4069 flash:
4070                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4071                 if (p + img_size > fw->data + fw->size)
4072                         return -1;
4073
4074                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4075                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4076                  * UFI_DIR region
4077                  */
4078                 if (old_fw_img &&
4079                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4080                      (img_optype == OPTYPE_UFI_DIR &&
4081                       base_status(status) == MCC_STATUS_FAILED))) {
4082                         continue;
4083                 } else if (status) {
4084                         dev_err(dev, "Flashing section type 0x%x failed\n",
4085                                 img_type);
4086                         return -EFAULT;
4087                 }
4088         }
4089         return 0;
4090 }
4091
4092 static int lancer_fw_download(struct be_adapter *adapter,
4093                               const struct firmware *fw)
4094 {
4095 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4096 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4097         struct device *dev = &adapter->pdev->dev;
4098         struct be_dma_mem flash_cmd;
4099         const u8 *data_ptr = NULL;
4100         u8 *dest_image_ptr = NULL;
4101         size_t image_size = 0;
4102         u32 chunk_size = 0;
4103         u32 data_written = 0;
4104         u32 offset = 0;
4105         int status = 0;
4106         u8 add_status = 0;
4107         u8 change_status;
4108
4109         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4110                 dev_err(dev, "FW image size should be multiple of 4\n");
4111                 return -EINVAL;
4112         }
4113
4114         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4115                                 + LANCER_FW_DOWNLOAD_CHUNK;
4116         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4117                                           &flash_cmd.dma, GFP_KERNEL);
4118         if (!flash_cmd.va)
4119                 return -ENOMEM;
4120
4121         dest_image_ptr = flash_cmd.va +
4122                                 sizeof(struct lancer_cmd_req_write_object);
4123         image_size = fw->size;
4124         data_ptr = fw->data;
4125
4126         while (image_size) {
4127                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4128
4129                 /* Copy the image chunk content. */
4130                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4131
4132                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4133                                                  chunk_size, offset,
4134                                                  LANCER_FW_DOWNLOAD_LOCATION,
4135                                                  &data_written, &change_status,
4136                                                  &add_status);
4137                 if (status)
4138                         break;
4139
4140                 offset += data_written;
4141                 data_ptr += data_written;
4142                 image_size -= data_written;
4143         }
4144
4145         if (!status) {
4146                 /* Commit the FW written */
4147                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4148                                                  0, offset,
4149                                                  LANCER_FW_DOWNLOAD_LOCATION,
4150                                                  &data_written, &change_status,
4151                                                  &add_status);
4152         }
4153
4154         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4155         if (status) {
4156                 dev_err(dev, "Firmware load error\n");
4157                 return be_cmd_status(status);
4158         }
4159
4160         dev_info(dev, "Firmware flashed successfully\n");
4161
4162         if (change_status == LANCER_FW_RESET_NEEDED) {
4163                 dev_info(dev, "Resetting adapter to activate new FW\n");
4164                 status = lancer_physdev_ctrl(adapter,
4165                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4166                 if (status) {
4167                         dev_err(dev, "Adapter busy, could not reset FW\n");
4168                         dev_err(dev, "Reboot server to activate new FW\n");
4169                 }
4170         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4171                 dev_info(dev, "Reboot server to activate new FW\n");
4172         }
4173
4174         return 0;
4175 }
4176
4177 #define UFI_TYPE2               2
4178 #define UFI_TYPE3               3
4179 #define UFI_TYPE3R              10
4180 #define UFI_TYPE4               4
4181 static int be_get_ufi_type(struct be_adapter *adapter,
4182                            struct flash_file_hdr_g3 *fhdr)
4183 {
4184         if (!fhdr)
4185                 goto be_get_ufi_exit;
4186
4187         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4188                 return UFI_TYPE4;
4189         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4190                 if (fhdr->asic_type_rev == 0x10)
4191                         return UFI_TYPE3R;
4192                 else
4193                         return UFI_TYPE3;
4194         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4195                 return UFI_TYPE2;
4196
4197 be_get_ufi_exit:
4198         dev_err(&adapter->pdev->dev,
4199                 "UFI and Interface are not compatible for flashing\n");
4200         return -1;
4201 }
4202
4203 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4204 {
4205         struct flash_file_hdr_g3 *fhdr3;
4206         struct image_hdr *img_hdr_ptr = NULL;
4207         struct be_dma_mem flash_cmd;
4208         const u8 *p;
4209         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4210
4211         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4212         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4213                                           &flash_cmd.dma, GFP_KERNEL);
4214         if (!flash_cmd.va) {
4215                 status = -ENOMEM;
4216                 goto be_fw_exit;
4217         }
4218
4219         p = fw->data;
4220         fhdr3 = (struct flash_file_hdr_g3 *)p;
4221
4222         ufi_type = be_get_ufi_type(adapter, fhdr3);
4223
4224         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4225         for (i = 0; i < num_imgs; i++) {
4226                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4227                                 (sizeof(struct flash_file_hdr_g3) +
4228                                  i * sizeof(struct image_hdr)));
4229                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4230                         switch (ufi_type) {
4231                         case UFI_TYPE4:
4232                                 status = be_flash_skyhawk(adapter, fw,
4233                                                           &flash_cmd, num_imgs);
4234                                 break;
4235                         case UFI_TYPE3R:
4236                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4237                                                       num_imgs);
4238                                 break;
4239                         case UFI_TYPE3:
4240                                 /* Do not flash this ufi on BE3-R cards */
4241                                 if (adapter->asic_rev < 0x10)
4242                                         status = be_flash_BEx(adapter, fw,
4243                                                               &flash_cmd,
4244                                                               num_imgs);
4245                                 else {
4246                                         status = -EINVAL;
4247                                         dev_err(&adapter->pdev->dev,
4248                                                 "Can't load BE3 UFI on BE3R\n");
4249                                 }
4250                         }
4251                 }
4252         }
4253
4254         if (ufi_type == UFI_TYPE2)
4255                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4256         else if (ufi_type == -1)
4257                 status = -EINVAL;
4258
4259         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4260                           flash_cmd.dma);
4261         if (status) {
4262                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4263                 goto be_fw_exit;
4264         }
4265
4266         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4267
4268 be_fw_exit:
4269         return status;
4270 }
4271
4272 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4273 {
4274         const struct firmware *fw;
4275         int status;
4276
4277         if (!netif_running(adapter->netdev)) {
4278                 dev_err(&adapter->pdev->dev,
4279                         "Firmware load not allowed (interface is down)\n");
4280                 return -ENETDOWN;
4281         }
4282
4283         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4284         if (status)
4285                 goto fw_exit;
4286
4287         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4288
4289         if (lancer_chip(adapter))
4290                 status = lancer_fw_download(adapter, fw);
4291         else
4292                 status = be_fw_download(adapter, fw);
4293
4294         if (!status)
4295                 be_cmd_get_fw_ver(adapter);
4296
4297 fw_exit:
4298         release_firmware(fw);
4299         return status;
4300 }
4301
4302 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4303 {
4304         struct be_adapter *adapter = netdev_priv(dev);
4305         struct nlattr *attr, *br_spec;
4306         int rem;
4307         int status = 0;
4308         u16 mode = 0;
4309
4310         if (!sriov_enabled(adapter))
4311                 return -EOPNOTSUPP;
4312
4313         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4314         if (!br_spec)
4315                 return -EINVAL;
4316
4317         nla_for_each_nested(attr, br_spec, rem) {
4318                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4319                         continue;
4320
4321                 if (nla_len(attr) < sizeof(mode))
4322                         return -EINVAL;
4323
4324                 mode = nla_get_u16(attr);
4325                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4326                         return -EINVAL;
4327
4328                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4329                                                adapter->if_handle,
4330                                                mode == BRIDGE_MODE_VEPA ?
4331                                                PORT_FWD_TYPE_VEPA :
4332                                                PORT_FWD_TYPE_VEB);
4333                 if (status)
4334                         goto err;
4335
4336                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4337                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4338
4339                 return status;
4340         }
4341 err:
4342         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4343                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4344
4345         return status;
4346 }
4347
4348 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4349                                  struct net_device *dev, u32 filter_mask)
4350 {
4351         struct be_adapter *adapter = netdev_priv(dev);
4352         int status = 0;
4353         u8 hsw_mode;
4354
4355         if (!sriov_enabled(adapter))
4356                 return 0;
4357
4358         /* BE and Lancer chips support VEB mode only */
4359         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4360                 hsw_mode = PORT_FWD_TYPE_VEB;
4361         } else {
4362                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4363                                                adapter->if_handle, &hsw_mode);
4364                 if (status)
4365                         return 0;
4366         }
4367
4368         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4369                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4370                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4371 }
4372
4373 #ifdef CONFIG_BE2NET_VXLAN
4374 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4375                               __be16 port)
4376 {
4377         struct be_adapter *adapter = netdev_priv(netdev);
4378         struct device *dev = &adapter->pdev->dev;
4379         int status;
4380
4381         if (lancer_chip(adapter) || BEx_chip(adapter))
4382                 return;
4383
4384         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4385                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4386                          be16_to_cpu(port));
4387                 dev_info(dev,
4388                          "Only one UDP port supported for VxLAN offloads\n");
4389                 return;
4390         }
4391
4392         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4393                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4394         if (status) {
4395                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4396                 goto err;
4397         }
4398
4399         status = be_cmd_set_vxlan_port(adapter, port);
4400         if (status) {
4401                 dev_warn(dev, "Failed to add VxLAN port\n");
4402                 goto err;
4403         }
4404         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4405         adapter->vxlan_port = port;
4406
4407         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4408                  be16_to_cpu(port));
4409         return;
4410 err:
4411         be_disable_vxlan_offloads(adapter);
4412 }
4413
4414 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4415                               __be16 port)
4416 {
4417         struct be_adapter *adapter = netdev_priv(netdev);
4418
4419         if (lancer_chip(adapter) || BEx_chip(adapter))
4420                 return;
4421
4422         if (adapter->vxlan_port != port)
4423                 return;
4424
4425         be_disable_vxlan_offloads(adapter);
4426
4427         dev_info(&adapter->pdev->dev,
4428                  "Disabled VxLAN offloads for UDP port %d\n",
4429                  be16_to_cpu(port));
4430 }
4431
4432 static bool be_gso_check(struct sk_buff *skb, struct net_device *dev)
4433 {
4434         return vxlan_gso_check(skb);
4435 }
4436 #endif
4437
4438 static const struct net_device_ops be_netdev_ops = {
4439         .ndo_open               = be_open,
4440         .ndo_stop               = be_close,
4441         .ndo_start_xmit         = be_xmit,
4442         .ndo_set_rx_mode        = be_set_rx_mode,
4443         .ndo_set_mac_address    = be_mac_addr_set,
4444         .ndo_change_mtu         = be_change_mtu,
4445         .ndo_get_stats64        = be_get_stats64,
4446         .ndo_validate_addr      = eth_validate_addr,
4447         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4448         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4449         .ndo_set_vf_mac         = be_set_vf_mac,
4450         .ndo_set_vf_vlan        = be_set_vf_vlan,
4451         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4452         .ndo_get_vf_config      = be_get_vf_config,
4453         .ndo_set_vf_link_state  = be_set_vf_link_state,
4454 #ifdef CONFIG_NET_POLL_CONTROLLER
4455         .ndo_poll_controller    = be_netpoll,
4456 #endif
4457         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4458         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4459 #ifdef CONFIG_NET_RX_BUSY_POLL
4460         .ndo_busy_poll          = be_busy_poll,
4461 #endif
4462 #ifdef CONFIG_BE2NET_VXLAN
4463         .ndo_add_vxlan_port     = be_add_vxlan_port,
4464         .ndo_del_vxlan_port     = be_del_vxlan_port,
4465         .ndo_gso_check          = be_gso_check,
4466 #endif
4467 };
4468
4469 static void be_netdev_init(struct net_device *netdev)
4470 {
4471         struct be_adapter *adapter = netdev_priv(netdev);
4472
4473         if (skyhawk_chip(adapter)) {
4474                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4475                                            NETIF_F_TSO | NETIF_F_TSO6 |
4476                                            NETIF_F_GSO_UDP_TUNNEL;
4477                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4478         }
4479         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4480                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4481                 NETIF_F_HW_VLAN_CTAG_TX;
4482         if (be_multi_rxq(adapter))
4483                 netdev->hw_features |= NETIF_F_RXHASH;
4484
4485         netdev->features |= netdev->hw_features |
4486                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4487
4488         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4489                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4490
4491         netdev->priv_flags |= IFF_UNICAST_FLT;
4492
4493         netdev->flags |= IFF_MULTICAST;
4494
4495         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4496
4497         netdev->netdev_ops = &be_netdev_ops;
4498
4499         netdev->ethtool_ops = &be_ethtool_ops;
4500 }
4501
4502 static void be_unmap_pci_bars(struct be_adapter *adapter)
4503 {
4504         if (adapter->csr)
4505                 pci_iounmap(adapter->pdev, adapter->csr);
4506         if (adapter->db)
4507                 pci_iounmap(adapter->pdev, adapter->db);
4508 }
4509
4510 static int db_bar(struct be_adapter *adapter)
4511 {
4512         if (lancer_chip(adapter) || !be_physfn(adapter))
4513                 return 0;
4514         else
4515                 return 4;
4516 }
4517
4518 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4519 {
4520         if (skyhawk_chip(adapter)) {
4521                 adapter->roce_db.size = 4096;
4522                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4523                                                               db_bar(adapter));
4524                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4525                                                                db_bar(adapter));
4526         }
4527         return 0;
4528 }
4529
4530 static int be_map_pci_bars(struct be_adapter *adapter)
4531 {
4532         u8 __iomem *addr;
4533
4534         if (BEx_chip(adapter) && be_physfn(adapter)) {
4535                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4536                 if (!adapter->csr)
4537                         return -ENOMEM;
4538         }
4539
4540         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4541         if (!addr)
4542                 goto pci_map_err;
4543         adapter->db = addr;
4544
4545         be_roce_map_pci_bars(adapter);
4546         return 0;
4547
4548 pci_map_err:
4549         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4550         be_unmap_pci_bars(adapter);
4551         return -ENOMEM;
4552 }
4553
4554 static void be_ctrl_cleanup(struct be_adapter *adapter)
4555 {
4556         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4557
4558         be_unmap_pci_bars(adapter);
4559
4560         if (mem->va)
4561                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4562                                   mem->dma);
4563
4564         mem = &adapter->rx_filter;
4565         if (mem->va)
4566                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4567                                   mem->dma);
4568 }
4569
4570 static int be_ctrl_init(struct be_adapter *adapter)
4571 {
4572         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4573         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4574         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4575         u32 sli_intf;
4576         int status;
4577
4578         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4579         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4580                                  SLI_INTF_FAMILY_SHIFT;
4581         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4582
4583         status = be_map_pci_bars(adapter);
4584         if (status)
4585                 goto done;
4586
4587         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4588         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4589                                                 mbox_mem_alloc->size,
4590                                                 &mbox_mem_alloc->dma,
4591                                                 GFP_KERNEL);
4592         if (!mbox_mem_alloc->va) {
4593                 status = -ENOMEM;
4594                 goto unmap_pci_bars;
4595         }
4596         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4597         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4598         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4599         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4600
4601         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4602         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4603                                             rx_filter->size, &rx_filter->dma,
4604                                             GFP_KERNEL);
4605         if (!rx_filter->va) {
4606                 status = -ENOMEM;
4607                 goto free_mbox;
4608         }
4609
4610         mutex_init(&adapter->mbox_lock);
4611         spin_lock_init(&adapter->mcc_lock);
4612         spin_lock_init(&adapter->mcc_cq_lock);
4613
4614         init_completion(&adapter->et_cmd_compl);
4615         pci_save_state(adapter->pdev);
4616         return 0;
4617
4618 free_mbox:
4619         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4620                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4621
4622 unmap_pci_bars:
4623         be_unmap_pci_bars(adapter);
4624
4625 done:
4626         return status;
4627 }
4628
4629 static void be_stats_cleanup(struct be_adapter *adapter)
4630 {
4631         struct be_dma_mem *cmd = &adapter->stats_cmd;
4632
4633         if (cmd->va)
4634                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4635                                   cmd->va, cmd->dma);
4636 }
4637
4638 static int be_stats_init(struct be_adapter *adapter)
4639 {
4640         struct be_dma_mem *cmd = &adapter->stats_cmd;
4641
4642         if (lancer_chip(adapter))
4643                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4644         else if (BE2_chip(adapter))
4645                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4646         else if (BE3_chip(adapter))
4647                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4648         else
4649                 /* ALL non-BE ASICs */
4650                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4651
4652         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4653                                       GFP_KERNEL);
4654         if (!cmd->va)
4655                 return -ENOMEM;
4656         return 0;
4657 }
4658
4659 static void be_remove(struct pci_dev *pdev)
4660 {
4661         struct be_adapter *adapter = pci_get_drvdata(pdev);
4662
4663         if (!adapter)
4664                 return;
4665
4666         be_roce_dev_remove(adapter);
4667         be_intr_set(adapter, false);
4668
4669         cancel_delayed_work_sync(&adapter->func_recovery_work);
4670
4671         unregister_netdev(adapter->netdev);
4672
4673         be_clear(adapter);
4674
4675         /* tell fw we're done with firing cmds */
4676         be_cmd_fw_clean(adapter);
4677
4678         be_stats_cleanup(adapter);
4679
4680         be_ctrl_cleanup(adapter);
4681
4682         pci_disable_pcie_error_reporting(pdev);
4683
4684         pci_release_regions(pdev);
4685         pci_disable_device(pdev);
4686
4687         free_netdev(adapter->netdev);
4688 }
4689
4690 static int be_get_initial_config(struct be_adapter *adapter)
4691 {
4692         int status, level;
4693
4694         status = be_cmd_get_cntl_attributes(adapter);
4695         if (status)
4696                 return status;
4697
4698         /* Must be a power of 2 or else MODULO will BUG_ON */
4699         adapter->be_get_temp_freq = 64;
4700
4701         if (BEx_chip(adapter)) {
4702                 level = be_cmd_get_fw_log_level(adapter);
4703                 adapter->msg_enable =
4704                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4705         }
4706
4707         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4708         return 0;
4709 }
4710
4711 static int lancer_recover_func(struct be_adapter *adapter)
4712 {
4713         struct device *dev = &adapter->pdev->dev;
4714         int status;
4715
4716         status = lancer_test_and_set_rdy_state(adapter);
4717         if (status)
4718                 goto err;
4719
4720         if (netif_running(adapter->netdev))
4721                 be_close(adapter->netdev);
4722
4723         be_clear(adapter);
4724
4725         be_clear_all_error(adapter);
4726
4727         status = be_setup(adapter);
4728         if (status)
4729                 goto err;
4730
4731         if (netif_running(adapter->netdev)) {
4732                 status = be_open(adapter->netdev);
4733                 if (status)
4734                         goto err;
4735         }
4736
4737         dev_err(dev, "Adapter recovery successful\n");
4738         return 0;
4739 err:
4740         if (status == -EAGAIN)
4741                 dev_err(dev, "Waiting for resource provisioning\n");
4742         else
4743                 dev_err(dev, "Adapter recovery failed\n");
4744
4745         return status;
4746 }
4747
4748 static void be_func_recovery_task(struct work_struct *work)
4749 {
4750         struct be_adapter *adapter =
4751                 container_of(work, struct be_adapter,  func_recovery_work.work);
4752         int status = 0;
4753
4754         be_detect_error(adapter);
4755
4756         if (adapter->hw_error && lancer_chip(adapter)) {
4757                 rtnl_lock();
4758                 netif_device_detach(adapter->netdev);
4759                 rtnl_unlock();
4760
4761                 status = lancer_recover_func(adapter);
4762                 if (!status)
4763                         netif_device_attach(adapter->netdev);
4764         }
4765
4766         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4767          * no need to attempt further recovery.
4768          */
4769         if (!status || status == -EAGAIN)
4770                 schedule_delayed_work(&adapter->func_recovery_work,
4771                                       msecs_to_jiffies(1000));
4772 }
4773
4774 static void be_worker(struct work_struct *work)
4775 {
4776         struct be_adapter *adapter =
4777                 container_of(work, struct be_adapter, work.work);
4778         struct be_rx_obj *rxo;
4779         int i;
4780
4781         /* when interrupts are not yet enabled, just reap any pending
4782         * mcc completions */
4783         if (!netif_running(adapter->netdev)) {
4784                 local_bh_disable();
4785                 be_process_mcc(adapter);
4786                 local_bh_enable();
4787                 goto reschedule;
4788         }
4789
4790         if (!adapter->stats_cmd_sent) {
4791                 if (lancer_chip(adapter))
4792                         lancer_cmd_get_pport_stats(adapter,
4793                                                    &adapter->stats_cmd);
4794                 else
4795                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4796         }
4797
4798         if (be_physfn(adapter) &&
4799             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4800                 be_cmd_get_die_temperature(adapter);
4801
4802         for_all_rx_queues(adapter, rxo, i) {
4803                 /* Replenish RX-queues starved due to memory
4804                  * allocation failures.
4805                  */
4806                 if (rxo->rx_post_starved)
4807                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4808         }
4809
4810         be_eqd_update(adapter);
4811
4812 reschedule:
4813         adapter->work_counter++;
4814         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4815 }
4816
4817 /* If any VFs are already enabled don't FLR the PF */
4818 static bool be_reset_required(struct be_adapter *adapter)
4819 {
4820         return pci_num_vf(adapter->pdev) ? false : true;
4821 }
4822
4823 static char *mc_name(struct be_adapter *adapter)
4824 {
4825         char *str = ""; /* default */
4826
4827         switch (adapter->mc_type) {
4828         case UMC:
4829                 str = "UMC";
4830                 break;
4831         case FLEX10:
4832                 str = "FLEX10";
4833                 break;
4834         case vNIC1:
4835                 str = "vNIC-1";
4836                 break;
4837         case nPAR:
4838                 str = "nPAR";
4839                 break;
4840         case UFP:
4841                 str = "UFP";
4842                 break;
4843         case vNIC2:
4844                 str = "vNIC-2";
4845                 break;
4846         default:
4847                 str = "";
4848         }
4849
4850         return str;
4851 }
4852
4853 static inline char *func_name(struct be_adapter *adapter)
4854 {
4855         return be_physfn(adapter) ? "PF" : "VF";
4856 }
4857
4858 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4859 {
4860         int status = 0;
4861         struct be_adapter *adapter;
4862         struct net_device *netdev;
4863         char port_name;
4864
4865         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4866
4867         status = pci_enable_device(pdev);
4868         if (status)
4869                 goto do_none;
4870
4871         status = pci_request_regions(pdev, DRV_NAME);
4872         if (status)
4873                 goto disable_dev;
4874         pci_set_master(pdev);
4875
4876         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4877         if (!netdev) {
4878                 status = -ENOMEM;
4879                 goto rel_reg;
4880         }
4881         adapter = netdev_priv(netdev);
4882         adapter->pdev = pdev;
4883         pci_set_drvdata(pdev, adapter);
4884         adapter->netdev = netdev;
4885         SET_NETDEV_DEV(netdev, &pdev->dev);
4886
4887         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4888         if (!status) {
4889                 netdev->features |= NETIF_F_HIGHDMA;
4890         } else {
4891                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4892                 if (status) {
4893                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4894                         goto free_netdev;
4895                 }
4896         }
4897
4898         status = pci_enable_pcie_error_reporting(pdev);
4899         if (!status)
4900                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4901
4902         status = be_ctrl_init(adapter);
4903         if (status)
4904                 goto free_netdev;
4905
4906         /* sync up with fw's ready state */
4907         if (be_physfn(adapter)) {
4908                 status = be_fw_wait_ready(adapter);
4909                 if (status)
4910                         goto ctrl_clean;
4911         }
4912
4913         if (be_reset_required(adapter)) {
4914                 status = be_cmd_reset_function(adapter);
4915                 if (status)
4916                         goto ctrl_clean;
4917
4918                 /* Wait for interrupts to quiesce after an FLR */
4919                 msleep(100);
4920         }
4921
4922         /* Allow interrupts for other ULPs running on NIC function */
4923         be_intr_set(adapter, true);
4924
4925         /* tell fw we're ready to fire cmds */
4926         status = be_cmd_fw_init(adapter);
4927         if (status)
4928                 goto ctrl_clean;
4929
4930         status = be_stats_init(adapter);
4931         if (status)
4932                 goto ctrl_clean;
4933
4934         status = be_get_initial_config(adapter);
4935         if (status)
4936                 goto stats_clean;
4937
4938         INIT_DELAYED_WORK(&adapter->work, be_worker);
4939         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4940         adapter->rx_fc = true;
4941         adapter->tx_fc = true;
4942
4943         status = be_setup(adapter);
4944         if (status)
4945                 goto stats_clean;
4946
4947         be_netdev_init(netdev);
4948         status = register_netdev(netdev);
4949         if (status != 0)
4950                 goto unsetup;
4951
4952         be_roce_dev_add(adapter);
4953
4954         schedule_delayed_work(&adapter->func_recovery_work,
4955                               msecs_to_jiffies(1000));
4956
4957         be_cmd_query_port_name(adapter, &port_name);
4958
4959         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4960                  func_name(adapter), mc_name(adapter), port_name);
4961
4962         return 0;
4963
4964 unsetup:
4965         be_clear(adapter);
4966 stats_clean:
4967         be_stats_cleanup(adapter);
4968 ctrl_clean:
4969         be_ctrl_cleanup(adapter);
4970 free_netdev:
4971         free_netdev(netdev);
4972 rel_reg:
4973         pci_release_regions(pdev);
4974 disable_dev:
4975         pci_disable_device(pdev);
4976 do_none:
4977         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4978         return status;
4979 }
4980
4981 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4982 {
4983         struct be_adapter *adapter = pci_get_drvdata(pdev);
4984         struct net_device *netdev =  adapter->netdev;
4985
4986         if (adapter->wol_en)
4987                 be_setup_wol(adapter, true);
4988
4989         be_intr_set(adapter, false);
4990         cancel_delayed_work_sync(&adapter->func_recovery_work);
4991
4992         netif_device_detach(netdev);
4993         if (netif_running(netdev)) {
4994                 rtnl_lock();
4995                 be_close(netdev);
4996                 rtnl_unlock();
4997         }
4998         be_clear(adapter);
4999
5000         pci_save_state(pdev);
5001         pci_disable_device(pdev);
5002         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5003         return 0;
5004 }
5005
5006 static int be_resume(struct pci_dev *pdev)
5007 {
5008         int status = 0;
5009         struct be_adapter *adapter = pci_get_drvdata(pdev);
5010         struct net_device *netdev =  adapter->netdev;
5011
5012         netif_device_detach(netdev);
5013
5014         status = pci_enable_device(pdev);
5015         if (status)
5016                 return status;
5017
5018         pci_set_power_state(pdev, PCI_D0);
5019         pci_restore_state(pdev);
5020
5021         status = be_fw_wait_ready(adapter);
5022         if (status)
5023                 return status;
5024
5025         be_intr_set(adapter, true);
5026         /* tell fw we're ready to fire cmds */
5027         status = be_cmd_fw_init(adapter);
5028         if (status)
5029                 return status;
5030
5031         be_setup(adapter);
5032         if (netif_running(netdev)) {
5033                 rtnl_lock();
5034                 be_open(netdev);
5035                 rtnl_unlock();
5036         }
5037
5038         schedule_delayed_work(&adapter->func_recovery_work,
5039                               msecs_to_jiffies(1000));
5040         netif_device_attach(netdev);
5041
5042         if (adapter->wol_en)
5043                 be_setup_wol(adapter, false);
5044
5045         return 0;
5046 }
5047
5048 /*
5049  * An FLR will stop BE from DMAing any data.
5050  */
5051 static void be_shutdown(struct pci_dev *pdev)
5052 {
5053         struct be_adapter *adapter = pci_get_drvdata(pdev);
5054
5055         if (!adapter)
5056                 return;
5057
5058         be_roce_dev_shutdown(adapter);
5059         cancel_delayed_work_sync(&adapter->work);
5060         cancel_delayed_work_sync(&adapter->func_recovery_work);
5061
5062         netif_device_detach(adapter->netdev);
5063
5064         be_cmd_reset_function(adapter);
5065
5066         pci_disable_device(pdev);
5067 }
5068
5069 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5070                                             pci_channel_state_t state)
5071 {
5072         struct be_adapter *adapter = pci_get_drvdata(pdev);
5073         struct net_device *netdev =  adapter->netdev;
5074
5075         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5076
5077         if (!adapter->eeh_error) {
5078                 adapter->eeh_error = true;
5079
5080                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5081
5082                 rtnl_lock();
5083                 netif_device_detach(netdev);
5084                 if (netif_running(netdev))
5085                         be_close(netdev);
5086                 rtnl_unlock();
5087
5088                 be_clear(adapter);
5089         }
5090
5091         if (state == pci_channel_io_perm_failure)
5092                 return PCI_ERS_RESULT_DISCONNECT;
5093
5094         pci_disable_device(pdev);
5095
5096         /* The error could cause the FW to trigger a flash debug dump.
5097          * Resetting the card while flash dump is in progress
5098          * can cause it not to recover; wait for it to finish.
5099          * Wait only for first function as it is needed only once per
5100          * adapter.
5101          */
5102         if (pdev->devfn == 0)
5103                 ssleep(30);
5104
5105         return PCI_ERS_RESULT_NEED_RESET;
5106 }
5107
5108 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5109 {
5110         struct be_adapter *adapter = pci_get_drvdata(pdev);
5111         int status;
5112
5113         dev_info(&adapter->pdev->dev, "EEH reset\n");
5114
5115         status = pci_enable_device(pdev);
5116         if (status)
5117                 return PCI_ERS_RESULT_DISCONNECT;
5118
5119         pci_set_master(pdev);
5120         pci_set_power_state(pdev, PCI_D0);
5121         pci_restore_state(pdev);
5122
5123         /* Check if card is ok and fw is ready */
5124         dev_info(&adapter->pdev->dev,
5125                  "Waiting for FW to be ready after EEH reset\n");
5126         status = be_fw_wait_ready(adapter);
5127         if (status)
5128                 return PCI_ERS_RESULT_DISCONNECT;
5129
5130         pci_cleanup_aer_uncorrect_error_status(pdev);
5131         be_clear_all_error(adapter);
5132         return PCI_ERS_RESULT_RECOVERED;
5133 }
5134
5135 static void be_eeh_resume(struct pci_dev *pdev)
5136 {
5137         int status = 0;
5138         struct be_adapter *adapter = pci_get_drvdata(pdev);
5139         struct net_device *netdev =  adapter->netdev;
5140
5141         dev_info(&adapter->pdev->dev, "EEH resume\n");
5142
5143         pci_save_state(pdev);
5144
5145         status = be_cmd_reset_function(adapter);
5146         if (status)
5147                 goto err;
5148
5149         /* On some BE3 FW versions, after a HW reset,
5150          * interrupts will remain disabled for each function.
5151          * So, explicitly enable interrupts
5152          */
5153         be_intr_set(adapter, true);
5154
5155         /* tell fw we're ready to fire cmds */
5156         status = be_cmd_fw_init(adapter);
5157         if (status)
5158                 goto err;
5159
5160         status = be_setup(adapter);
5161         if (status)
5162                 goto err;
5163
5164         if (netif_running(netdev)) {
5165                 status = be_open(netdev);
5166                 if (status)
5167                         goto err;
5168         }
5169
5170         schedule_delayed_work(&adapter->func_recovery_work,
5171                               msecs_to_jiffies(1000));
5172         netif_device_attach(netdev);
5173         return;
5174 err:
5175         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5176 }
5177
5178 static const struct pci_error_handlers be_eeh_handlers = {
5179         .error_detected = be_eeh_err_detected,
5180         .slot_reset = be_eeh_reset,
5181         .resume = be_eeh_resume,
5182 };
5183
5184 static struct pci_driver be_driver = {
5185         .name = DRV_NAME,
5186         .id_table = be_dev_ids,
5187         .probe = be_probe,
5188         .remove = be_remove,
5189         .suspend = be_suspend,
5190         .resume = be_resume,
5191         .shutdown = be_shutdown,
5192         .err_handler = &be_eeh_handlers
5193 };
5194
5195 static int __init be_init_module(void)
5196 {
5197         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5198             rx_frag_size != 2048) {
5199                 printk(KERN_WARNING DRV_NAME
5200                         " : Module param rx_frag_size must be 2048/4096/8192."
5201                         " Using 2048\n");
5202                 rx_frag_size = 2048;
5203         }
5204
5205         return pci_register_driver(&be_driver);
5206 }
5207 module_init(be_init_module);
5208
5209 static void __exit be_exit_module(void)
5210 {
5211         pci_unregister_driver(&be_driver);
5212 }
5213 module_exit(be_exit_module);