drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo,
 666                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 667                                bool stopped)
 668 {
 669         struct be_tx_stats *stats = tx_stats(txo);
 670
 671         u64_stats_update_begin(&stats->sync);
 672         stats->tx_reqs++;
 673         stats->tx_wrbs += wrb_cnt;
 674         stats->tx_bytes += copied;
 675         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 676         if (stopped)
 677                 stats->tx_stops++;
 678         u64_stats_update_end(&stats->sync);
 679 }
 680
 681 /* Determine number of WRB entries needed to xmit data in an skb */
 682 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 683                            bool *dummy)
 684 {
 685         int cnt = (skb->len > skb->data_len);
 686
 687         cnt += skb_shinfo(skb)->nr_frags;
 688
 689         /* to account for hdr wrb */
 690         cnt++;
 691         if (lancer_chip(adapter) || !(cnt & 1)) {
 692                 *dummy = false;
 693         } else {
 694                 /* add a dummy to make it an even num */
 695                 cnt++;
 696                 *dummy = true;
 697         }
 698         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 699         return cnt;
 700 }
 701
 702 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 703 {
 704         wrb->frag_pa_hi = upper_32_bits(addr);
 705         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 706         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 707         wrb->rsvd0 = 0;
 708 }
 709
 710 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 711                                      struct sk_buff *skb)
 712 {
 713         u8 vlan_prio;
 714         u16 vlan_tag;
 715
 716         vlan_tag = vlan_tx_tag_get(skb);
 717         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 718         /* If vlan priority provided by OS is NOT in available bmap */
 719         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 720                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 721                                 adapter->recommended_prio;
 722
 723         return vlan_tag;
 724 }
 725
 726 /* Used only for IP tunnel packets */
 727 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 728 {
 729         return (inner_ip_hdr(skb)->version == 4) ?
 730                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static u16 skb_ip_proto(struct sk_buff *skb)
 734 {
 735         return (ip_hdr(skb)->version == 4) ?
 736                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 737 }
 738
 739 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 740                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 741                          bool skip_hw_vlan)
 742 {
 743         u16 vlan_tag, proto;
 744
 745         memset(hdr, 0, sizeof(*hdr));
 746
 747         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 748
 749         if (skb_is_gso(skb)) {
 750                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 751                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 752                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 753                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 754         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 755                 if (skb->encapsulation) {
 756                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 757                         proto = skb_inner_ip_proto(skb);
 758                 } else {
 759                         proto = skb_ip_proto(skb);
 760                 }
 761                 if (proto == IPPROTO_TCP)
 762                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 763                 else if (proto == IPPROTO_UDP)
 764                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 765         }
 766
 767         if (vlan_tx_tag_present(skb)) {
 768                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 769                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 770                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 771         }
 772
 773         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 774         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 775         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 776         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 777         SET_TX_WRB_HDR_BITS(len, hdr, len);
 778 }
 779
 780 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 781                           bool unmap_single)
 782 {
 783         dma_addr_t dma;
 784
 785         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 786
 787         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 788         if (wrb->frag_len) {
 789                 if (unmap_single)
 790                         dma_unmap_single(dev, dma, wrb->frag_len,
 791                                          DMA_TO_DEVICE);
 792                 else
 793                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 794         }
 795 }
 796
 797 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 798                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 799                         bool skip_hw_vlan)
 800 {
 801         dma_addr_t busaddr;
 802         int i, copied = 0;
 803         struct device *dev = &adapter->pdev->dev;
 804         struct sk_buff *first_skb = skb;
 805         struct be_eth_wrb *wrb;
 806         struct be_eth_hdr_wrb *hdr;
 807         bool map_single = false;
 808         u16 map_head;
 809
 810         hdr = queue_head_node(txq);
 811         queue_head_inc(txq);
 812         map_head = txq->head;
 813
 814         if (skb->len > skb->data_len) {
 815                 int len = skb_headlen(skb);
 816
 817                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 818                 if (dma_mapping_error(dev, busaddr))
 819                         goto dma_err;
 820                 map_single = true;
 821                 wrb = queue_head_node(txq);
 822                 wrb_fill(wrb, busaddr, len);
 823                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 824                 queue_head_inc(txq);
 825                 copied += len;
 826         }
 827
 828         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 829                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 830
 831                 busaddr = skb_frag_dma_map(dev, frag, 0,
 832                                            skb_frag_size(frag), DMA_TO_DEVICE);
 833                 if (dma_mapping_error(dev, busaddr))
 834                         goto dma_err;
 835                 wrb = queue_head_node(txq);
 836                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 837                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 838                 queue_head_inc(txq);
 839                 copied += skb_frag_size(frag);
 840         }
 841
 842         if (dummy_wrb) {
 843                 wrb = queue_head_node(txq);
 844                 wrb_fill(wrb, 0, 0);
 845                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 846                 queue_head_inc(txq);
 847         }
 848
 849         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 850         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 851
 852         return copied;
 853 dma_err:
 854         txq->head = map_head;
 855         while (copied) {
 856                 wrb = queue_head_node(txq);
 857                 unmap_tx_frag(dev, wrb, map_single);
 858                 map_single = false;
 859                 copied -= wrb->frag_len;
 860                 adapter->drv_stats.dma_map_errors++;
 861                 queue_head_inc(txq);
 862         }
 863         return 0;
 864 }
 865
 866 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 867                                              struct sk_buff *skb,
 868                                              bool *skip_hw_vlan)
 869 {
 870         u16 vlan_tag = 0;
 871
 872         skb = skb_share_check(skb, GFP_ATOMIC);
 873         if (unlikely(!skb))
 874                 return skb;
 875
 876         if (vlan_tx_tag_present(skb))
 877                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 878
 879         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 880                 if (!vlan_tag)
 881                         vlan_tag = adapter->pvid;
 882                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 883                  * skip VLAN insertion
 884                  */
 885                 if (skip_hw_vlan)
 886                         *skip_hw_vlan = true;
 887         }
 888
 889         if (vlan_tag) {
 890                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 891                                                 vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 skb->vlan_tci = 0;
 895         }
 896
 897         /* Insert the outer VLAN, if any */
 898         if (adapter->qnq_vid) {
 899                 vlan_tag = adapter->qnq_vid;
 900                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 901                                                 vlan_tag);
 902                 if (unlikely(!skb))
 903                         return skb;
 904                 if (skip_hw_vlan)
 905                         *skip_hw_vlan = true;
 906         }
 907
 908         return skb;
 909 }
 910
 911 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 912 {
 913         struct ethhdr *eh = (struct ethhdr *)skb->data;
 914         u16 offset = ETH_HLEN;
 915
 916         if (eh->h_proto == htons(ETH_P_IPV6)) {
 917                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 918
 919                 offset += sizeof(struct ipv6hdr);
 920                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 921                     ip6h->nexthdr != NEXTHDR_UDP) {
 922                         struct ipv6_opt_hdr *ehdr =
 923                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 924
 925                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 926                         if (ehdr->hdrlen == 0xff)
 927                                 return true;
 928                 }
 929         }
 930         return false;
 931 }
 932
 933 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 934 {
 935         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 936 }
 937
 938 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 939 {
 940         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 941 }
 942
 943 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 944                                                   struct sk_buff *skb,
 945                                                   bool *skip_hw_vlan)
 946 {
 947         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 948         unsigned int eth_hdr_len;
 949         struct iphdr *ip;
 950
 951         /* For padded packets, BE HW modifies tot_len field in IP header
 952          * incorrecly when VLAN tag is inserted by HW.
 953          * For padded packets, Lancer computes incorrect checksum.
 954          */
 955         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 956                                                 VLAN_ETH_HLEN : ETH_HLEN;
 957         if (skb->len <= 60 &&
 958             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 959             is_ipv4_pkt(skb)) {
 960                 ip = (struct iphdr *)ip_hdr(skb);
 961                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 962         }
 963
 964         /* If vlan tag is already inlined in the packet, skip HW VLAN
 965          * tagging in pvid-tagging mode
 966          */
 967         if (be_pvid_tagging_enabled(adapter) &&
 968             veh->h_vlan_proto == htons(ETH_P_8021Q))
 969                 *skip_hw_vlan = true;
 970
 971         /* HW has a bug wherein it will calculate CSUM for VLAN
 972          * pkts even though it is disabled.
 973          * Manually insert VLAN in pkt.
 974          */
 975         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 976             vlan_tx_tag_present(skb)) {
 977                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 978                 if (unlikely(!skb))
 979                         goto err;
 980         }
 981
 982         /* HW may lockup when VLAN HW tagging is requested on
 983          * certain ipv6 packets. Drop such pkts if the HW workaround to
 984          * skip HW tagging is not enabled by FW.
 985          */
 986         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 987                      (adapter->pvid || adapter->qnq_vid) &&
 988                      !qnq_async_evt_rcvd(adapter)))
 989                 goto tx_drop;
 990
 991         /* Manual VLAN tag insertion to prevent:
 992          * ASIC lockup when the ASIC inserts VLAN tag into
 993          * certain ipv6 packets. Insert VLAN tags in driver,
 994          * and set event, completion, vlan bits accordingly
 995          * in the Tx WRB.
 996          */
 997         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 998             be_vlan_tag_tx_chk(adapter, skb)) {
 999                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1000                 if (unlikely(!skb))
1001                         goto err;
1002         }
1003
1004         return skb;
1005 tx_drop:
1006         dev_kfree_skb_any(skb);
1007 err:
1008         return NULL;
1009 }
1010
1011 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1012                                            struct sk_buff *skb,
1013                                            bool *skip_hw_vlan)
1014 {
1015         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1016          * less may cause a transmit stall on that port. So the work-around is
1017          * to pad short packets (<= 32 bytes) to a 36-byte length.
1018          */
1019         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1020                 if (skb_put_padto(skb, 36))
1021                         return NULL;
1022         }
1023
1024         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1025                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1026                 if (!skb)
1027                         return NULL;
1028         }
1029
1030         return skb;
1031 }
1032
1033 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1034 {
1035         struct be_adapter *adapter = netdev_priv(netdev);
1036         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1037         struct be_queue_info *txq = &txo->q;
1038         bool dummy_wrb, stopped = false;
1039         u32 wrb_cnt = 0, copied = 0;
1040         bool skip_hw_vlan = false;
1041         u32 start = txq->head;
1042
1043         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1044         if (!skb) {
1045                 tx_stats(txo)->tx_drv_drops++;
1046                 return NETDEV_TX_OK;
1047         }
1048
1049         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1050
1051         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1052                               skip_hw_vlan);
1053         if (copied) {
1054                 int gso_segs = skb_shinfo(skb)->gso_segs;
1055
1056                 /* record the sent skb in the sent_skb table */
1057                 BUG_ON(txo->sent_skb_list[start]);
1058                 txo->sent_skb_list[start] = skb;
1059
1060                 /* Ensure txq has space for the next skb; Else stop the queue
1061                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1062                  * tx compls of the current transmit which'll wake up the queue
1063                  */
1064                 atomic_add(wrb_cnt, &txq->used);
1065                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1066                                                                 txq->len) {
1067                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1068                         stopped = true;
1069                 }
1070
1071                 be_txq_notify(adapter, txo, wrb_cnt);
1072
1073                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1074         } else {
1075                 txq->head = start;
1076                 tx_stats(txo)->tx_drv_drops++;
1077                 dev_kfree_skb_any(skb);
1078         }
1079         return NETDEV_TX_OK;
1080 }
1081
1082 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1083 {
1084         struct be_adapter *adapter = netdev_priv(netdev);
1085         struct device *dev = &adapter->pdev->dev;
1086
1087         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1088                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1089                          BE_MIN_MTU, BE_MAX_MTU);
1090                 return -EINVAL;
1091         }
1092
1093         dev_info(dev, "MTU changed from %d to %d bytes\n",
1094                  netdev->mtu, new_mtu);
1095         netdev->mtu = new_mtu;
1096         return 0;
1097 }
1098
1099 /*
1100  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1101  * If the user configures more, place BE in vlan promiscuous mode.
1102  */
1103 static int be_vid_config(struct be_adapter *adapter)
1104 {
1105         struct device *dev = &adapter->pdev->dev;
1106         u16 vids[BE_NUM_VLANS_SUPPORTED];
1107         u16 num = 0, i = 0;
1108         int status = 0;
1109
1110         /* No need to further configure vids if in promiscuous mode */
1111         if (adapter->promiscuous)
1112                 return 0;
1113
1114         if (adapter->vlans_added > be_max_vlans(adapter))
1115                 goto set_vlan_promisc;
1116
1117         /* Construct VLAN Table to give to HW */
1118         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1119                 vids[num++] = cpu_to_le16(i);
1120
1121         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1122         if (status) {
1123                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1124                 if (addl_status(status) ==
1125                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1126                         goto set_vlan_promisc;
1127                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1128         } else {
1129                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1130                         /* hw VLAN filtering re-enabled. */
1131                         status = be_cmd_rx_filter(adapter,
1132                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1133                         if (!status) {
1134                                 dev_info(dev,
1135                                          "Disabling VLAN Promiscuous mode\n");
1136                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1137                         }
1138                 }
1139         }
1140
1141         return status;
1142
1143 set_vlan_promisc:
1144         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1145                 return 0;
1146
1147         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1148         if (!status) {
1149                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1150                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1151         } else
1152                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1153         return status;
1154 }
1155
1156 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1157 {
1158         struct be_adapter *adapter = netdev_priv(netdev);
1159         int status = 0;
1160
1161         /* Packets with VID 0 are always received by Lancer by default */
1162         if (lancer_chip(adapter) && vid == 0)
1163                 return status;
1164
1165         if (test_bit(vid, adapter->vids))
1166                 return status;
1167
1168         set_bit(vid, adapter->vids);
1169         adapter->vlans_added++;
1170
1171         status = be_vid_config(adapter);
1172         if (status) {
1173                 adapter->vlans_added--;
1174                 clear_bit(vid, adapter->vids);
1175         }
1176
1177         return status;
1178 }
1179
1180 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1181 {
1182         struct be_adapter *adapter = netdev_priv(netdev);
1183
1184         /* Packets with VID 0 are always received by Lancer by default */
1185         if (lancer_chip(adapter) && vid == 0)
1186                 return 0;
1187
1188         clear_bit(vid, adapter->vids);
1189         adapter->vlans_added--;
1190
1191         return be_vid_config(adapter);
1192 }
1193
1194 static void be_clear_promisc(struct be_adapter *adapter)
1195 {
1196         adapter->promiscuous = false;
1197         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1198
1199         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1200 }
1201
1202 static void be_set_rx_mode(struct net_device *netdev)
1203 {
1204         struct be_adapter *adapter = netdev_priv(netdev);
1205         int status;
1206
1207         if (netdev->flags & IFF_PROMISC) {
1208                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1209                 adapter->promiscuous = true;
1210                 goto done;
1211         }
1212
1213         /* BE was previously in promiscuous mode; disable it */
1214         if (adapter->promiscuous) {
1215                 be_clear_promisc(adapter);
1216                 if (adapter->vlans_added)
1217                         be_vid_config(adapter);
1218         }
1219
1220         /* Enable multicast promisc if num configured exceeds what we support */
1221         if (netdev->flags & IFF_ALLMULTI ||
1222             netdev_mc_count(netdev) > be_max_mc(adapter))
1223                 goto set_mcast_promisc;
1224
1225         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1226                 struct netdev_hw_addr *ha;
1227                 int i = 1; /* First slot is claimed by the Primary MAC */
1228
1229                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1230                         be_cmd_pmac_del(adapter, adapter->if_handle,
1231                                         adapter->pmac_id[i], 0);
1232                 }
1233
1234                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1235                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1236                         adapter->promiscuous = true;
1237                         goto done;
1238                 }
1239
1240                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1241                         adapter->uc_macs++; /* First slot is for Primary MAC */
1242                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1243                                         adapter->if_handle,
1244                                         &adapter->pmac_id[adapter->uc_macs], 0);
1245                 }
1246         }
1247
1248         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1249         if (!status) {
1250                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1251                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1252                 goto done;
1253         }
1254
1255 set_mcast_promisc:
1256         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1257                 return;
1258
1259         /* Set to MCAST promisc mode if setting MULTICAST address fails
1260          * or if num configured exceeds what we support
1261          */
1262         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1263         if (!status)
1264                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1265 done:
1266         return;
1267 }
1268
1269 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1270 {
1271         struct be_adapter *adapter = netdev_priv(netdev);
1272         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1273         int status;
1274
1275         if (!sriov_enabled(adapter))
1276                 return -EPERM;
1277
1278         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1279                 return -EINVAL;
1280
1281         /* Proceed further only if user provided MAC is different
1282          * from active MAC
1283          */
1284         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1285                 return 0;
1286
1287         if (BEx_chip(adapter)) {
1288                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1289                                 vf + 1);
1290
1291                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1292                                          &vf_cfg->pmac_id, vf + 1);
1293         } else {
1294                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1295                                         vf + 1);
1296         }
1297
1298         if (status) {
1299                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1300                         mac, vf, status);
1301                 return be_cmd_status(status);
1302         }
1303
1304         ether_addr_copy(vf_cfg->mac_addr, mac);
1305
1306         return 0;
1307 }
1308
1309 static int be_get_vf_config(struct net_device *netdev, int vf,
1310                             struct ifla_vf_info *vi)
1311 {
1312         struct be_adapter *adapter = netdev_priv(netdev);
1313         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1314
1315         if (!sriov_enabled(adapter))
1316                 return -EPERM;
1317
1318         if (vf >= adapter->num_vfs)
1319                 return -EINVAL;
1320
1321         vi->vf = vf;
1322         vi->max_tx_rate = vf_cfg->tx_rate;
1323         vi->min_tx_rate = 0;
1324         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1325         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1326         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1327         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1328
1329         return 0;
1330 }
1331
1332 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1333 {
1334         struct be_adapter *adapter = netdev_priv(netdev);
1335         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1336         int status = 0;
1337
1338         if (!sriov_enabled(adapter))
1339                 return -EPERM;
1340
1341         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1342                 return -EINVAL;
1343
1344         if (vlan || qos) {
1345                 vlan |= qos << VLAN_PRIO_SHIFT;
1346                 if (vf_cfg->vlan_tag != vlan)
1347                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1348                                                        vf_cfg->if_handle, 0);
1349         } else {
1350                 /* Reset Transparent Vlan Tagging. */
1351                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1352                                                vf + 1, vf_cfg->if_handle, 0);
1353         }
1354
1355         if (status) {
1356                 dev_err(&adapter->pdev->dev,
1357                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1358                         vf, status);
1359                 return be_cmd_status(status);
1360         }
1361
1362         vf_cfg->vlan_tag = vlan;
1363
1364         return 0;
1365 }
1366
1367 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1368                              int min_tx_rate, int max_tx_rate)
1369 {
1370         struct be_adapter *adapter = netdev_priv(netdev);
1371         struct device *dev = &adapter->pdev->dev;
1372         int percent_rate, status = 0;
1373         u16 link_speed = 0;
1374         u8 link_status;
1375
1376         if (!sriov_enabled(adapter))
1377                 return -EPERM;
1378
1379         if (vf >= adapter->num_vfs)
1380                 return -EINVAL;
1381
1382         if (min_tx_rate)
1383                 return -EINVAL;
1384
1385         if (!max_tx_rate)
1386                 goto config_qos;
1387
1388         status = be_cmd_link_status_query(adapter, &link_speed,
1389                                           &link_status, 0);
1390         if (status)
1391                 goto err;
1392
1393         if (!link_status) {
1394                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1395                 status = -ENETDOWN;
1396                 goto err;
1397         }
1398
1399         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1400                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1401                         link_speed);
1402                 status = -EINVAL;
1403                 goto err;
1404         }
1405
1406         /* On Skyhawk the QOS setting must be done only as a % value */
1407         percent_rate = link_speed / 100;
1408         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1409                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1410                         percent_rate);
1411                 status = -EINVAL;
1412                 goto err;
1413         }
1414
1415 config_qos:
1416         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1417         if (status)
1418                 goto err;
1419
1420         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1421         return 0;
1422
1423 err:
1424         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1425                 max_tx_rate, vf);
1426         return be_cmd_status(status);
1427 }
1428
1429 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1430                                 int link_state)
1431 {
1432         struct be_adapter *adapter = netdev_priv(netdev);
1433         int status;
1434
1435         if (!sriov_enabled(adapter))
1436                 return -EPERM;
1437
1438         if (vf >= adapter->num_vfs)
1439                 return -EINVAL;
1440
1441         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1442         if (status) {
1443                 dev_err(&adapter->pdev->dev,
1444                         "Link state change on VF %d failed: %#x\n", vf, status);
1445                 return be_cmd_status(status);
1446         }
1447
1448         adapter->vf_cfg[vf].plink_tracking = link_state;
1449
1450         return 0;
1451 }
1452
1453 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1454                           ulong now)
1455 {
1456         aic->rx_pkts_prev = rx_pkts;
1457         aic->tx_reqs_prev = tx_pkts;
1458         aic->jiffies = now;
1459 }
1460
1461 static void be_eqd_update(struct be_adapter *adapter)
1462 {
1463         struct be_set_eqd set_eqd[MAX_EVT_QS];
1464         int eqd, i, num = 0, start;
1465         struct be_aic_obj *aic;
1466         struct be_eq_obj *eqo;
1467         struct be_rx_obj *rxo;
1468         struct be_tx_obj *txo;
1469         u64 rx_pkts, tx_pkts;
1470         ulong now;
1471         u32 pps, delta;
1472
1473         for_all_evt_queues(adapter, eqo, i) {
1474                 aic = &adapter->aic_obj[eqo->idx];
1475                 if (!aic->enable) {
1476                         if (aic->jiffies)
1477                                 aic->jiffies = 0;
1478                         eqd = aic->et_eqd;
1479                         goto modify_eqd;
1480                 }
1481
1482                 rxo = &adapter->rx_obj[eqo->idx];
1483                 do {
1484                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1485                         rx_pkts = rxo->stats.rx_pkts;
1486                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1487
1488                 txo = &adapter->tx_obj[eqo->idx];
1489                 do {
1490                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1491                         tx_pkts = txo->stats.tx_reqs;
1492                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1493
1494                 /* Skip, if wrapped around or first calculation */
1495                 now = jiffies;
1496                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1497                     rx_pkts < aic->rx_pkts_prev ||
1498                     tx_pkts < aic->tx_reqs_prev) {
1499                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1500                         continue;
1501                 }
1502
1503                 delta = jiffies_to_msecs(now - aic->jiffies);
1504                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1505                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1506                 eqd = (pps / 15000) << 2;
1507
1508                 if (eqd < 8)
1509                         eqd = 0;
1510                 eqd = min_t(u32, eqd, aic->max_eqd);
1511                 eqd = max_t(u32, eqd, aic->min_eqd);
1512
1513                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1514 modify_eqd:
1515                 if (eqd != aic->prev_eqd) {
1516                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1517                         set_eqd[num].eq_id = eqo->q.id;
1518                         aic->prev_eqd = eqd;
1519                         num++;
1520                 }
1521         }
1522
1523         if (num)
1524                 be_cmd_modify_eqd(adapter, set_eqd, num);
1525 }
1526
1527 static void be_rx_stats_update(struct be_rx_obj *rxo,
1528                                struct be_rx_compl_info *rxcp)
1529 {
1530         struct be_rx_stats *stats = rx_stats(rxo);
1531
1532         u64_stats_update_begin(&stats->sync);
1533         stats->rx_compl++;
1534         stats->rx_bytes += rxcp->pkt_size;
1535         stats->rx_pkts++;
1536         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1537                 stats->rx_mcast_pkts++;
1538         if (rxcp->err)
1539                 stats->rx_compl_err++;
1540         u64_stats_update_end(&stats->sync);
1541 }
1542
1543 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1544 {
1545         /* L4 checksum is not reliable for non TCP/UDP packets.
1546          * Also ignore ipcksm for ipv6 pkts
1547          */
1548         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1549                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1550 }
1551
1552 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1553 {
1554         struct be_adapter *adapter = rxo->adapter;
1555         struct be_rx_page_info *rx_page_info;
1556         struct be_queue_info *rxq = &rxo->q;
1557         u16 frag_idx = rxq->tail;
1558
1559         rx_page_info = &rxo->page_info_tbl[frag_idx];
1560         BUG_ON(!rx_page_info->page);
1561
1562         if (rx_page_info->last_frag) {
1563                 dma_unmap_page(&adapter->pdev->dev,
1564                                dma_unmap_addr(rx_page_info, bus),
1565                                adapter->big_page_size, DMA_FROM_DEVICE);
1566                 rx_page_info->last_frag = false;
1567         } else {
1568                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1569                                         dma_unmap_addr(rx_page_info, bus),
1570                                         rx_frag_size, DMA_FROM_DEVICE);
1571         }
1572
1573         queue_tail_inc(rxq);
1574         atomic_dec(&rxq->used);
1575         return rx_page_info;
1576 }
1577
1578 /* Throwaway the data in the Rx completion */
1579 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1580                                 struct be_rx_compl_info *rxcp)
1581 {
1582         struct be_rx_page_info *page_info;
1583         u16 i, num_rcvd = rxcp->num_rcvd;
1584
1585         for (i = 0; i < num_rcvd; i++) {
1586                 page_info = get_rx_page_info(rxo);
1587                 put_page(page_info->page);
1588                 memset(page_info, 0, sizeof(*page_info));
1589         }
1590 }
1591
1592 /*
1593  * skb_fill_rx_data forms a complete skb for an ether frame
1594  * indicated by rxcp.
1595  */
1596 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1597                              struct be_rx_compl_info *rxcp)
1598 {
1599         struct be_rx_page_info *page_info;
1600         u16 i, j;
1601         u16 hdr_len, curr_frag_len, remaining;
1602         u8 *start;
1603
1604         page_info = get_rx_page_info(rxo);
1605         start = page_address(page_info->page) + page_info->page_offset;
1606         prefetch(start);
1607
1608         /* Copy data in the first descriptor of this completion */
1609         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1610
1611         skb->len = curr_frag_len;
1612         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1613                 memcpy(skb->data, start, curr_frag_len);
1614                 /* Complete packet has now been moved to data */
1615                 put_page(page_info->page);
1616                 skb->data_len = 0;
1617                 skb->tail += curr_frag_len;
1618         } else {
1619                 hdr_len = ETH_HLEN;
1620                 memcpy(skb->data, start, hdr_len);
1621                 skb_shinfo(skb)->nr_frags = 1;
1622                 skb_frag_set_page(skb, 0, page_info->page);
1623                 skb_shinfo(skb)->frags[0].page_offset =
1624                                         page_info->page_offset + hdr_len;
1625                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1626                                   curr_frag_len - hdr_len);
1627                 skb->data_len = curr_frag_len - hdr_len;
1628                 skb->truesize += rx_frag_size;
1629                 skb->tail += hdr_len;
1630         }
1631         page_info->page = NULL;
1632
1633         if (rxcp->pkt_size <= rx_frag_size) {
1634                 BUG_ON(rxcp->num_rcvd != 1);
1635                 return;
1636         }
1637
1638         /* More frags present for this completion */
1639         remaining = rxcp->pkt_size - curr_frag_len;
1640         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1641                 page_info = get_rx_page_info(rxo);
1642                 curr_frag_len = min(remaining, rx_frag_size);
1643
1644                 /* Coalesce all frags from the same physical page in one slot */
1645                 if (page_info->page_offset == 0) {
1646                         /* Fresh page */
1647                         j++;
1648                         skb_frag_set_page(skb, j, page_info->page);
1649                         skb_shinfo(skb)->frags[j].page_offset =
1650                                                         page_info->page_offset;
1651                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1652                         skb_shinfo(skb)->nr_frags++;
1653                 } else {
1654                         put_page(page_info->page);
1655                 }
1656
1657                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1658                 skb->len += curr_frag_len;
1659                 skb->data_len += curr_frag_len;
1660                 skb->truesize += rx_frag_size;
1661                 remaining -= curr_frag_len;
1662                 page_info->page = NULL;
1663         }
1664         BUG_ON(j > MAX_SKB_FRAGS);
1665 }
1666
1667 /* Process the RX completion indicated by rxcp when GRO is disabled */
1668 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1669                                 struct be_rx_compl_info *rxcp)
1670 {
1671         struct be_adapter *adapter = rxo->adapter;
1672         struct net_device *netdev = adapter->netdev;
1673         struct sk_buff *skb;
1674
1675         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1676         if (unlikely(!skb)) {
1677                 rx_stats(rxo)->rx_drops_no_skbs++;
1678                 be_rx_compl_discard(rxo, rxcp);
1679                 return;
1680         }
1681
1682         skb_fill_rx_data(rxo, skb, rxcp);
1683
1684         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1685                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1686         else
1687                 skb_checksum_none_assert(skb);
1688
1689         skb->protocol = eth_type_trans(skb, netdev);
1690         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1691         if (netdev->features & NETIF_F_RXHASH)
1692                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1693
1694         skb->csum_level = rxcp->tunneled;
1695         skb_mark_napi_id(skb, napi);
1696
1697         if (rxcp->vlanf)
1698                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1699
1700         netif_receive_skb(skb);
1701 }
1702
1703 /* Process the RX completion indicated by rxcp when GRO is enabled */
1704 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1705                                     struct napi_struct *napi,
1706                                     struct be_rx_compl_info *rxcp)
1707 {
1708         struct be_adapter *adapter = rxo->adapter;
1709         struct be_rx_page_info *page_info;
1710         struct sk_buff *skb = NULL;
1711         u16 remaining, curr_frag_len;
1712         u16 i, j;
1713
1714         skb = napi_get_frags(napi);
1715         if (!skb) {
1716                 be_rx_compl_discard(rxo, rxcp);
1717                 return;
1718         }
1719
1720         remaining = rxcp->pkt_size;
1721         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1722                 page_info = get_rx_page_info(rxo);
1723
1724                 curr_frag_len = min(remaining, rx_frag_size);
1725
1726                 /* Coalesce all frags from the same physical page in one slot */
1727                 if (i == 0 || page_info->page_offset == 0) {
1728                         /* First frag or Fresh page */
1729                         j++;
1730                         skb_frag_set_page(skb, j, page_info->page);
1731                         skb_shinfo(skb)->frags[j].page_offset =
1732                                                         page_info->page_offset;
1733                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1734                 } else {
1735                         put_page(page_info->page);
1736                 }
1737                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1738                 skb->truesize += rx_frag_size;
1739                 remaining -= curr_frag_len;
1740                 memset(page_info, 0, sizeof(*page_info));
1741         }
1742         BUG_ON(j > MAX_SKB_FRAGS);
1743
1744         skb_shinfo(skb)->nr_frags = j + 1;
1745         skb->len = rxcp->pkt_size;
1746         skb->data_len = rxcp->pkt_size;
1747         skb->ip_summed = CHECKSUM_UNNECESSARY;
1748         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1749         if (adapter->netdev->features & NETIF_F_RXHASH)
1750                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1751
1752         skb->csum_level = rxcp->tunneled;
1753         skb_mark_napi_id(skb, napi);
1754
1755         if (rxcp->vlanf)
1756                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1757
1758         napi_gro_frags(napi);
1759 }
1760
1761 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1762                                  struct be_rx_compl_info *rxcp)
1763 {
1764         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1765         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1766         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1767         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1768         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1769         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1770         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1771         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1772         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1773         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1774         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1775         if (rxcp->vlanf) {
1776                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1777                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1778         }
1779         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1780         rxcp->tunneled =
1781                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1782 }
1783
1784 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1785                                  struct be_rx_compl_info *rxcp)
1786 {
1787         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1788         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1789         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1790         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1791         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1792         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1793         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1794         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1795         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1796         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1797         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1798         if (rxcp->vlanf) {
1799                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1800                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1801         }
1802         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1803         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1804 }
1805
1806 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1807 {
1808         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1809         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1810         struct be_adapter *adapter = rxo->adapter;
1811
1812         /* For checking the valid bit it is Ok to use either definition as the
1813          * valid bit is at the same position in both v0 and v1 Rx compl */
1814         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1815                 return NULL;
1816
1817         rmb();
1818         be_dws_le_to_cpu(compl, sizeof(*compl));
1819
1820         if (adapter->be3_native)
1821                 be_parse_rx_compl_v1(compl, rxcp);
1822         else
1823                 be_parse_rx_compl_v0(compl, rxcp);
1824
1825         if (rxcp->ip_frag)
1826                 rxcp->l4_csum = 0;
1827
1828         if (rxcp->vlanf) {
1829                 /* In QNQ modes, if qnq bit is not set, then the packet was
1830                  * tagged only with the transparent outer vlan-tag and must
1831                  * not be treated as a vlan packet by host
1832                  */
1833                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1834                         rxcp->vlanf = 0;
1835
1836                 if (!lancer_chip(adapter))
1837                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1838
1839                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1840                     !test_bit(rxcp->vlan_tag, adapter->vids))
1841                         rxcp->vlanf = 0;
1842         }
1843
1844         /* As the compl has been parsed, reset it; we wont touch it again */
1845         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1846
1847         queue_tail_inc(&rxo->cq);
1848         return rxcp;
1849 }
1850
1851 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1852 {
1853         u32 order = get_order(size);
1854
1855         if (order > 0)
1856                 gfp |= __GFP_COMP;
1857         return  alloc_pages(gfp, order);
1858 }
1859
1860 /*
1861  * Allocate a page, split it to fragments of size rx_frag_size and post as
1862  * receive buffers to BE
1863  */
1864 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1865 {
1866         struct be_adapter *adapter = rxo->adapter;
1867         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1868         struct be_queue_info *rxq = &rxo->q;
1869         struct page *pagep = NULL;
1870         struct device *dev = &adapter->pdev->dev;
1871         struct be_eth_rx_d *rxd;
1872         u64 page_dmaaddr = 0, frag_dmaaddr;
1873         u32 posted, page_offset = 0, notify = 0;
1874
1875         page_info = &rxo->page_info_tbl[rxq->head];
1876         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1877                 if (!pagep) {
1878                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1879                         if (unlikely(!pagep)) {
1880                                 rx_stats(rxo)->rx_post_fail++;
1881                                 break;
1882                         }
1883                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1884                                                     adapter->big_page_size,
1885                                                     DMA_FROM_DEVICE);
1886                         if (dma_mapping_error(dev, page_dmaaddr)) {
1887                                 put_page(pagep);
1888                                 pagep = NULL;
1889                                 adapter->drv_stats.dma_map_errors++;
1890                                 break;
1891                         }
1892                         page_offset = 0;
1893                 } else {
1894                         get_page(pagep);
1895                         page_offset += rx_frag_size;
1896                 }
1897                 page_info->page_offset = page_offset;
1898                 page_info->page = pagep;
1899
1900                 rxd = queue_head_node(rxq);
1901                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1902                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1903                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1904
1905                 /* Any space left in the current big page for another frag? */
1906                 if ((page_offset + rx_frag_size + rx_frag_size) >
1907                                         adapter->big_page_size) {
1908                         pagep = NULL;
1909                         page_info->last_frag = true;
1910                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1911                 } else {
1912                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1913                 }
1914
1915                 prev_page_info = page_info;
1916                 queue_head_inc(rxq);
1917                 page_info = &rxo->page_info_tbl[rxq->head];
1918         }
1919
1920         /* Mark the last frag of a page when we break out of the above loop
1921          * with no more slots available in the RXQ
1922          */
1923         if (pagep) {
1924                 prev_page_info->last_frag = true;
1925                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1926         }
1927
1928         if (posted) {
1929                 atomic_add(posted, &rxq->used);
1930                 if (rxo->rx_post_starved)
1931                         rxo->rx_post_starved = false;
1932                 do {
1933                         notify = min(256u, posted);
1934                         be_rxq_notify(adapter, rxq->id, notify);
1935                         posted -= notify;
1936                 } while (posted);
1937         } else if (atomic_read(&rxq->used) == 0) {
1938                 /* Let be_worker replenish when memory is available */
1939                 rxo->rx_post_starved = true;
1940         }
1941 }
1942
1943 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1944 {
1945         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1946
1947         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1948                 return NULL;
1949
1950         rmb();
1951         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1952
1953         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1954
1955         queue_tail_inc(tx_cq);
1956         return txcp;
1957 }
1958
1959 static u16 be_tx_compl_process(struct be_adapter *adapter,
1960                                struct be_tx_obj *txo, u16 last_index)
1961 {
1962         struct be_queue_info *txq = &txo->q;
1963         struct be_eth_wrb *wrb;
1964         struct sk_buff **sent_skbs = txo->sent_skb_list;
1965         struct sk_buff *sent_skb;
1966         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1967         bool unmap_skb_hdr = true;
1968
1969         sent_skb = sent_skbs[txq->tail];
1970         BUG_ON(!sent_skb);
1971         sent_skbs[txq->tail] = NULL;
1972
1973         /* skip header wrb */
1974         queue_tail_inc(txq);
1975
1976         do {
1977                 cur_index = txq->tail;
1978                 wrb = queue_tail_node(txq);
1979                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1980                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1981                 unmap_skb_hdr = false;
1982
1983                 num_wrbs++;
1984                 queue_tail_inc(txq);
1985         } while (cur_index != last_index);
1986
1987         dev_consume_skb_any(sent_skb);
1988         return num_wrbs;
1989 }
1990
1991 /* Return the number of events in the event queue */
1992 static inline int events_get(struct be_eq_obj *eqo)
1993 {
1994         struct be_eq_entry *eqe;
1995         int num = 0;
1996
1997         do {
1998                 eqe = queue_tail_node(&eqo->q);
1999                 if (eqe->evt == 0)
2000                         break;
2001
2002                 rmb();
2003                 eqe->evt = 0;
2004                 num++;
2005                 queue_tail_inc(&eqo->q);
2006         } while (true);
2007
2008         return num;
2009 }
2010
2011 /* Leaves the EQ is disarmed state */
2012 static void be_eq_clean(struct be_eq_obj *eqo)
2013 {
2014         int num = events_get(eqo);
2015
2016         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2017 }
2018
2019 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2020 {
2021         struct be_rx_page_info *page_info;
2022         struct be_queue_info *rxq = &rxo->q;
2023         struct be_queue_info *rx_cq = &rxo->cq;
2024         struct be_rx_compl_info *rxcp;
2025         struct be_adapter *adapter = rxo->adapter;
2026         int flush_wait = 0;
2027
2028         /* Consume pending rx completions.
2029          * Wait for the flush completion (identified by zero num_rcvd)
2030          * to arrive. Notify CQ even when there are no more CQ entries
2031          * for HW to flush partially coalesced CQ entries.
2032          * In Lancer, there is no need to wait for flush compl.
2033          */
2034         for (;;) {
2035                 rxcp = be_rx_compl_get(rxo);
2036                 if (!rxcp) {
2037                         if (lancer_chip(adapter))
2038                                 break;
2039
2040                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2041                                 dev_warn(&adapter->pdev->dev,
2042                                          "did not receive flush compl\n");
2043                                 break;
2044                         }
2045                         be_cq_notify(adapter, rx_cq->id, true, 0);
2046                         mdelay(1);
2047                 } else {
2048                         be_rx_compl_discard(rxo, rxcp);
2049                         be_cq_notify(adapter, rx_cq->id, false, 1);
2050                         if (rxcp->num_rcvd == 0)
2051                                 break;
2052                 }
2053         }
2054
2055         /* After cleanup, leave the CQ in unarmed state */
2056         be_cq_notify(adapter, rx_cq->id, false, 0);
2057
2058         /* Then free posted rx buffers that were not used */
2059         while (atomic_read(&rxq->used) > 0) {
2060                 page_info = get_rx_page_info(rxo);
2061                 put_page(page_info->page);
2062                 memset(page_info, 0, sizeof(*page_info));
2063         }
2064         BUG_ON(atomic_read(&rxq->used));
2065         rxq->tail = 0;
2066         rxq->head = 0;
2067 }
2068
2069 static void be_tx_compl_clean(struct be_adapter *adapter)
2070 {
2071         struct be_tx_obj *txo;
2072         struct be_queue_info *txq;
2073         struct be_eth_tx_compl *txcp;
2074         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2075         struct sk_buff *sent_skb;
2076         bool dummy_wrb;
2077         int i, pending_txqs;
2078
2079         /* Stop polling for compls when HW has been silent for 10ms */
2080         do {
2081                 pending_txqs = adapter->num_tx_qs;
2082
2083                 for_all_tx_queues(adapter, txo, i) {
2084                         cmpl = 0;
2085                         num_wrbs = 0;
2086                         txq = &txo->q;
2087                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2088                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2089                                 num_wrbs += be_tx_compl_process(adapter, txo,
2090                                                                 end_idx);
2091                                 cmpl++;
2092                         }
2093                         if (cmpl) {
2094                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2095                                 atomic_sub(num_wrbs, &txq->used);
2096                                 timeo = 0;
2097                         }
2098                         if (atomic_read(&txq->used) == 0)
2099                                 pending_txqs--;
2100                 }
2101
2102                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2103                         break;
2104
2105                 mdelay(1);
2106         } while (true);
2107
2108         for_all_tx_queues(adapter, txo, i) {
2109                 txq = &txo->q;
2110                 if (atomic_read(&txq->used))
2111                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2112                                 atomic_read(&txq->used));
2113
2114                 /* free posted tx for which compls will never arrive */
2115                 while (atomic_read(&txq->used)) {
2116                         sent_skb = txo->sent_skb_list[txq->tail];
2117                         end_idx = txq->tail;
2118                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2119                                                    &dummy_wrb);
2120                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2121                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2122                         atomic_sub(num_wrbs, &txq->used);
2123                 }
2124         }
2125 }
2126
2127 static void be_evt_queues_destroy(struct be_adapter *adapter)
2128 {
2129         struct be_eq_obj *eqo;
2130         int i;
2131
2132         for_all_evt_queues(adapter, eqo, i) {
2133                 if (eqo->q.created) {
2134                         be_eq_clean(eqo);
2135                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2136                         napi_hash_del(&eqo->napi);
2137                         netif_napi_del(&eqo->napi);
2138                 }
2139                 be_queue_free(adapter, &eqo->q);
2140         }
2141 }
2142
2143 static int be_evt_queues_create(struct be_adapter *adapter)
2144 {
2145         struct be_queue_info *eq;
2146         struct be_eq_obj *eqo;
2147         struct be_aic_obj *aic;
2148         int i, rc;
2149
2150         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2151                                     adapter->cfg_num_qs);
2152
2153         for_all_evt_queues(adapter, eqo, i) {
2154                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2155                                BE_NAPI_WEIGHT);
2156                 napi_hash_add(&eqo->napi);
2157                 aic = &adapter->aic_obj[i];
2158                 eqo->adapter = adapter;
2159                 eqo->idx = i;
2160                 aic->max_eqd = BE_MAX_EQD;
2161                 aic->enable = true;
2162
2163                 eq = &eqo->q;
2164                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2165                                     sizeof(struct be_eq_entry));
2166                 if (rc)
2167                         return rc;
2168
2169                 rc = be_cmd_eq_create(adapter, eqo);
2170                 if (rc)
2171                         return rc;
2172         }
2173         return 0;
2174 }
2175
2176 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2177 {
2178         struct be_queue_info *q;
2179
2180         q = &adapter->mcc_obj.q;
2181         if (q->created)
2182                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2183         be_queue_free(adapter, q);
2184
2185         q = &adapter->mcc_obj.cq;
2186         if (q->created)
2187                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2188         be_queue_free(adapter, q);
2189 }
2190
2191 /* Must be called only after TX qs are created as MCC shares TX EQ */
2192 static int be_mcc_queues_create(struct be_adapter *adapter)
2193 {
2194         struct be_queue_info *q, *cq;
2195
2196         cq = &adapter->mcc_obj.cq;
2197         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2198                            sizeof(struct be_mcc_compl)))
2199                 goto err;
2200
2201         /* Use the default EQ for MCC completions */
2202         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2203                 goto mcc_cq_free;
2204
2205         q = &adapter->mcc_obj.q;
2206         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2207                 goto mcc_cq_destroy;
2208
2209         if (be_cmd_mccq_create(adapter, q, cq))
2210                 goto mcc_q_free;
2211
2212         return 0;
2213
2214 mcc_q_free:
2215         be_queue_free(adapter, q);
2216 mcc_cq_destroy:
2217         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2218 mcc_cq_free:
2219         be_queue_free(adapter, cq);
2220 err:
2221         return -1;
2222 }
2223
2224 static void be_tx_queues_destroy(struct be_adapter *adapter)
2225 {
2226         struct be_queue_info *q;
2227         struct be_tx_obj *txo;
2228         u8 i;
2229
2230         for_all_tx_queues(adapter, txo, i) {
2231                 q = &txo->q;
2232                 if (q->created)
2233                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2234                 be_queue_free(adapter, q);
2235
2236                 q = &txo->cq;
2237                 if (q->created)
2238                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2239                 be_queue_free(adapter, q);
2240         }
2241 }
2242
2243 static int be_tx_qs_create(struct be_adapter *adapter)
2244 {
2245         struct be_queue_info *cq, *eq;
2246         struct be_tx_obj *txo;
2247         int status, i;
2248
2249         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2250
2251         for_all_tx_queues(adapter, txo, i) {
2252                 cq = &txo->cq;
2253                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2254                                         sizeof(struct be_eth_tx_compl));
2255                 if (status)
2256                         return status;
2257
2258                 u64_stats_init(&txo->stats.sync);
2259                 u64_stats_init(&txo->stats.sync_compl);
2260
2261                 /* If num_evt_qs is less than num_tx_qs, then more than
2262                  * one txq share an eq
2263                  */
2264                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2265                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2266                 if (status)
2267                         return status;
2268
2269                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2270                                         sizeof(struct be_eth_wrb));
2271                 if (status)
2272                         return status;
2273
2274                 status = be_cmd_txq_create(adapter, txo);
2275                 if (status)
2276                         return status;
2277         }
2278
2279         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2280                  adapter->num_tx_qs);
2281         return 0;
2282 }
2283
2284 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2285 {
2286         struct be_queue_info *q;
2287         struct be_rx_obj *rxo;
2288         int i;
2289
2290         for_all_rx_queues(adapter, rxo, i) {
2291                 q = &rxo->cq;
2292                 if (q->created)
2293                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2294                 be_queue_free(adapter, q);
2295         }
2296 }
2297
2298 static int be_rx_cqs_create(struct be_adapter *adapter)
2299 {
2300         struct be_queue_info *eq, *cq;
2301         struct be_rx_obj *rxo;
2302         int rc, i;
2303
2304         /* We can create as many RSS rings as there are EQs. */
2305         adapter->num_rx_qs = adapter->num_evt_qs;
2306
2307         /* We'll use RSS only if atleast 2 RSS rings are supported.
2308          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2309          */
2310         if (adapter->num_rx_qs > 1)
2311                 adapter->num_rx_qs++;
2312
2313         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2314         for_all_rx_queues(adapter, rxo, i) {
2315                 rxo->adapter = adapter;
2316                 cq = &rxo->cq;
2317                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2318                                     sizeof(struct be_eth_rx_compl));
2319                 if (rc)
2320                         return rc;
2321
2322                 u64_stats_init(&rxo->stats.sync);
2323                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2324                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2325                 if (rc)
2326                         return rc;
2327         }
2328
2329         dev_info(&adapter->pdev->dev,
2330                  "created %d RSS queue(s) and 1 default RX queue\n",
2331                  adapter->num_rx_qs - 1);
2332         return 0;
2333 }
2334
2335 static irqreturn_t be_intx(int irq, void *dev)
2336 {
2337         struct be_eq_obj *eqo = dev;
2338         struct be_adapter *adapter = eqo->adapter;
2339         int num_evts = 0;
2340
2341         /* IRQ is not expected when NAPI is scheduled as the EQ
2342          * will not be armed.
2343          * But, this can happen on Lancer INTx where it takes
2344          * a while to de-assert INTx or in BE2 where occasionaly
2345          * an interrupt may be raised even when EQ is unarmed.
2346          * If NAPI is already scheduled, then counting & notifying
2347          * events will orphan them.
2348          */
2349         if (napi_schedule_prep(&eqo->napi)) {
2350                 num_evts = events_get(eqo);
2351                 __napi_schedule(&eqo->napi);
2352                 if (num_evts)
2353                         eqo->spurious_intr = 0;
2354         }
2355         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2356
2357         /* Return IRQ_HANDLED only for the the first spurious intr
2358          * after a valid intr to stop the kernel from branding
2359          * this irq as a bad one!
2360          */
2361         if (num_evts || eqo->spurious_intr++ == 0)
2362                 return IRQ_HANDLED;
2363         else
2364                 return IRQ_NONE;
2365 }
2366
2367 static irqreturn_t be_msix(int irq, void *dev)
2368 {
2369         struct be_eq_obj *eqo = dev;
2370
2371         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2372         napi_schedule(&eqo->napi);
2373         return IRQ_HANDLED;
2374 }
2375
2376 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2377 {
2378         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2379 }
2380
2381 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2382                          int budget, int polling)
2383 {
2384         struct be_adapter *adapter = rxo->adapter;
2385         struct be_queue_info *rx_cq = &rxo->cq;
2386         struct be_rx_compl_info *rxcp;
2387         u32 work_done;
2388         u32 frags_consumed = 0;
2389
2390         for (work_done = 0; work_done < budget; work_done++) {
2391                 rxcp = be_rx_compl_get(rxo);
2392                 if (!rxcp)
2393                         break;
2394
2395                 /* Is it a flush compl that has no data */
2396                 if (unlikely(rxcp->num_rcvd == 0))
2397                         goto loop_continue;
2398
2399                 /* Discard compl with partial DMA Lancer B0 */
2400                 if (unlikely(!rxcp->pkt_size)) {
2401                         be_rx_compl_discard(rxo, rxcp);
2402                         goto loop_continue;
2403                 }
2404
2405                 /* On BE drop pkts that arrive due to imperfect filtering in
2406                  * promiscuous mode on some skews
2407                  */
2408                 if (unlikely(rxcp->port != adapter->port_num &&
2409                              !lancer_chip(adapter))) {
2410                         be_rx_compl_discard(rxo, rxcp);
2411                         goto loop_continue;
2412                 }
2413
2414                 /* Don't do gro when we're busy_polling */
2415                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2416                         be_rx_compl_process_gro(rxo, napi, rxcp);
2417                 else
2418                         be_rx_compl_process(rxo, napi, rxcp);
2419
2420 loop_continue:
2421                 frags_consumed += rxcp->num_rcvd;
2422                 be_rx_stats_update(rxo, rxcp);
2423         }
2424
2425         if (work_done) {
2426                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2427
2428                 /* When an rx-obj gets into post_starved state, just
2429                  * let be_worker do the posting.
2430                  */
2431                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2432                     !rxo->rx_post_starved)
2433                         be_post_rx_frags(rxo, GFP_ATOMIC,
2434                                          max_t(u32, MAX_RX_POST,
2435                                                frags_consumed));
2436         }
2437
2438         return work_done;
2439 }
2440
2441 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2442 {
2443         switch (status) {
2444         case BE_TX_COMP_HDR_PARSE_ERR:
2445                 tx_stats(txo)->tx_hdr_parse_err++;
2446                 break;
2447         case BE_TX_COMP_NDMA_ERR:
2448                 tx_stats(txo)->tx_dma_err++;
2449                 break;
2450         case BE_TX_COMP_ACL_ERR:
2451                 tx_stats(txo)->tx_spoof_check_err++;
2452                 break;
2453         }
2454 }
2455
2456 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2457 {
2458         switch (status) {
2459         case LANCER_TX_COMP_LSO_ERR:
2460                 tx_stats(txo)->tx_tso_err++;
2461                 break;
2462         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2463         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2464                 tx_stats(txo)->tx_spoof_check_err++;
2465                 break;
2466         case LANCER_TX_COMP_QINQ_ERR:
2467                 tx_stats(txo)->tx_qinq_err++;
2468                 break;
2469         case LANCER_TX_COMP_PARITY_ERR:
2470                 tx_stats(txo)->tx_internal_parity_err++;
2471                 break;
2472         case LANCER_TX_COMP_DMA_ERR:
2473                 tx_stats(txo)->tx_dma_err++;
2474                 break;
2475         }
2476 }
2477
2478 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2479                           int idx)
2480 {
2481         struct be_eth_tx_compl *txcp;
2482         int num_wrbs = 0, work_done = 0;
2483         u32 compl_status;
2484         u16 last_idx;
2485
2486         while ((txcp = be_tx_compl_get(&txo->cq))) {
2487                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2488                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2489                 work_done++;
2490
2491                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2492                 if (compl_status) {
2493                         if (lancer_chip(adapter))
2494                                 lancer_update_tx_err(txo, compl_status);
2495                         else
2496                                 be_update_tx_err(txo, compl_status);
2497                 }
2498         }
2499
2500         if (work_done) {
2501                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2502                 atomic_sub(num_wrbs, &txo->q.used);
2503
2504                 /* As Tx wrbs have been freed up, wake up netdev queue
2505                  * if it was stopped due to lack of tx wrbs.  */
2506                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2507                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2508                         netif_wake_subqueue(adapter->netdev, idx);
2509                 }
2510
2511                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2512                 tx_stats(txo)->tx_compl += work_done;
2513                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2514         }
2515 }
2516
2517 int be_poll(struct napi_struct *napi, int budget)
2518 {
2519         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2520         struct be_adapter *adapter = eqo->adapter;
2521         int max_work = 0, work, i, num_evts;
2522         struct be_rx_obj *rxo;
2523         struct be_tx_obj *txo;
2524
2525         num_evts = events_get(eqo);
2526
2527         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2528                 be_process_tx(adapter, txo, i);
2529
2530         if (be_lock_napi(eqo)) {
2531                 /* This loop will iterate twice for EQ0 in which
2532                  * completions of the last RXQ (default one) are also processed
2533                  * For other EQs the loop iterates only once
2534                  */
2535                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2536                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2537                         max_work = max(work, max_work);
2538                 }
2539                 be_unlock_napi(eqo);
2540         } else {
2541                 max_work = budget;
2542         }
2543
2544         if (is_mcc_eqo(eqo))
2545                 be_process_mcc(adapter);
2546
2547         if (max_work < budget) {
2548                 napi_complete(napi);
2549                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2550         } else {
2551                 /* As we'll continue in polling mode, count and clear events */
2552                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2553         }
2554         return max_work;
2555 }
2556
2557 #ifdef CONFIG_NET_RX_BUSY_POLL
2558 static int be_busy_poll(struct napi_struct *napi)
2559 {
2560         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2561         struct be_adapter *adapter = eqo->adapter;
2562         struct be_rx_obj *rxo;
2563         int i, work = 0;
2564
2565         if (!be_lock_busy_poll(eqo))
2566                 return LL_FLUSH_BUSY;
2567
2568         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2569                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2570                 if (work)
2571                         break;
2572         }
2573
2574         be_unlock_busy_poll(eqo);
2575         return work;
2576 }
2577 #endif
2578
2579 void be_detect_error(struct be_adapter *adapter)
2580 {
2581         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2582         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2583         u32 i;
2584         bool error_detected = false;
2585         struct device *dev = &adapter->pdev->dev;
2586         struct net_device *netdev = adapter->netdev;
2587
2588         if (be_hw_error(adapter))
2589                 return;
2590
2591         if (lancer_chip(adapter)) {
2592                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2593                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2594                         sliport_err1 = ioread32(adapter->db +
2595                                                 SLIPORT_ERROR1_OFFSET);
2596                         sliport_err2 = ioread32(adapter->db +
2597                                                 SLIPORT_ERROR2_OFFSET);
2598                         adapter->hw_error = true;
2599                         /* Do not log error messages if its a FW reset */
2600                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2601                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2602                                 dev_info(dev, "Firmware update in progress\n");
2603                         } else {
2604                                 error_detected = true;
2605                                 dev_err(dev, "Error detected in the card\n");
2606                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2607                                         sliport_status);
2608                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2609                                         sliport_err1);
2610                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2611                                         sliport_err2);
2612                         }
2613                 }
2614         } else {
2615                 pci_read_config_dword(adapter->pdev,
2616                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2617                 pci_read_config_dword(adapter->pdev,
2618                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2619                 pci_read_config_dword(adapter->pdev,
2620                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2621                 pci_read_config_dword(adapter->pdev,
2622                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2623
2624                 ue_lo = (ue_lo & ~ue_lo_mask);
2625                 ue_hi = (ue_hi & ~ue_hi_mask);
2626
2627                 /* On certain platforms BE hardware can indicate spurious UEs.
2628                  * Allow HW to stop working completely in case of a real UE.
2629                  * Hence not setting the hw_error for UE detection.
2630                  */
2631
2632                 if (ue_lo || ue_hi) {
2633                         error_detected = true;
2634                         dev_err(dev,
2635                                 "Unrecoverable Error detected in the adapter");
2636                         dev_err(dev, "Please reboot server to recover");
2637                         if (skyhawk_chip(adapter))
2638                                 adapter->hw_error = true;
2639                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2640                                 if (ue_lo & 1)
2641                                         dev_err(dev, "UE: %s bit set\n",
2642                                                 ue_status_low_desc[i]);
2643                         }
2644                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2645                                 if (ue_hi & 1)
2646                                         dev_err(dev, "UE: %s bit set\n",
2647                                                 ue_status_hi_desc[i]);
2648                         }
2649                 }
2650         }
2651         if (error_detected)
2652                 netif_carrier_off(netdev);
2653 }
2654
2655 static void be_msix_disable(struct be_adapter *adapter)
2656 {
2657         if (msix_enabled(adapter)) {
2658                 pci_disable_msix(adapter->pdev);
2659                 adapter->num_msix_vec = 0;
2660                 adapter->num_msix_roce_vec = 0;
2661         }
2662 }
2663
2664 static int be_msix_enable(struct be_adapter *adapter)
2665 {
2666         int i, num_vec;
2667         struct device *dev = &adapter->pdev->dev;
2668
2669         /* If RoCE is supported, program the max number of NIC vectors that
2670          * may be configured via set-channels, along with vectors needed for
2671          * RoCe. Else, just program the number we'll use initially.
2672          */
2673         if (be_roce_supported(adapter))
2674                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2675                                 2 * num_online_cpus());
2676         else
2677                 num_vec = adapter->cfg_num_qs;
2678
2679         for (i = 0; i < num_vec; i++)
2680                 adapter->msix_entries[i].entry = i;
2681
2682         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2683                                         MIN_MSIX_VECTORS, num_vec);
2684         if (num_vec < 0)
2685                 goto fail;
2686
2687         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2688                 adapter->num_msix_roce_vec = num_vec / 2;
2689                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2690                          adapter->num_msix_roce_vec);
2691         }
2692
2693         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2694
2695         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2696                  adapter->num_msix_vec);
2697         return 0;
2698
2699 fail:
2700         dev_warn(dev, "MSIx enable failed\n");
2701
2702         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2703         if (!be_physfn(adapter))
2704                 return num_vec;
2705         return 0;
2706 }
2707
2708 static inline int be_msix_vec_get(struct be_adapter *adapter,
2709                                   struct be_eq_obj *eqo)
2710 {
2711         return adapter->msix_entries[eqo->msix_idx].vector;
2712 }
2713
2714 static int be_msix_register(struct be_adapter *adapter)
2715 {
2716         struct net_device *netdev = adapter->netdev;
2717         struct be_eq_obj *eqo;
2718         int status, i, vec;
2719
2720         for_all_evt_queues(adapter, eqo, i) {
2721                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2722                 vec = be_msix_vec_get(adapter, eqo);
2723                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2724                 if (status)
2725                         goto err_msix;
2726         }
2727
2728         return 0;
2729 err_msix:
2730         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2731                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2732         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2733                  status);
2734         be_msix_disable(adapter);
2735         return status;
2736 }
2737
2738 static int be_irq_register(struct be_adapter *adapter)
2739 {
2740         struct net_device *netdev = adapter->netdev;
2741         int status;
2742
2743         if (msix_enabled(adapter)) {
2744                 status = be_msix_register(adapter);
2745                 if (status == 0)
2746                         goto done;
2747                 /* INTx is not supported for VF */
2748                 if (!be_physfn(adapter))
2749                         return status;
2750         }
2751
2752         /* INTx: only the first EQ is used */
2753         netdev->irq = adapter->pdev->irq;
2754         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2755                              &adapter->eq_obj[0]);
2756         if (status) {
2757                 dev_err(&adapter->pdev->dev,
2758                         "INTx request IRQ failed - err %d\n", status);
2759                 return status;
2760         }
2761 done:
2762         adapter->isr_registered = true;
2763         return 0;
2764 }
2765
2766 static void be_irq_unregister(struct be_adapter *adapter)
2767 {
2768         struct net_device *netdev = adapter->netdev;
2769         struct be_eq_obj *eqo;
2770         int i;
2771
2772         if (!adapter->isr_registered)
2773                 return;
2774
2775         /* INTx */
2776         if (!msix_enabled(adapter)) {
2777                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2778                 goto done;
2779         }
2780
2781         /* MSIx */
2782         for_all_evt_queues(adapter, eqo, i)
2783                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2784
2785 done:
2786         adapter->isr_registered = false;
2787 }
2788
2789 static void be_rx_qs_destroy(struct be_adapter *adapter)
2790 {
2791         struct be_queue_info *q;
2792         struct be_rx_obj *rxo;
2793         int i;
2794
2795         for_all_rx_queues(adapter, rxo, i) {
2796                 q = &rxo->q;
2797                 if (q->created) {
2798                         be_cmd_rxq_destroy(adapter, q);
2799                         be_rx_cq_clean(rxo);
2800                 }
2801                 be_queue_free(adapter, q);
2802         }
2803 }
2804
2805 static int be_close(struct net_device *netdev)
2806 {
2807         struct be_adapter *adapter = netdev_priv(netdev);
2808         struct be_eq_obj *eqo;
2809         int i;
2810
2811         /* This protection is needed as be_close() may be called even when the
2812          * adapter is in cleared state (after eeh perm failure)
2813          */
2814         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2815                 return 0;
2816
2817         be_roce_dev_close(adapter);
2818
2819         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2820                 for_all_evt_queues(adapter, eqo, i) {
2821                         napi_disable(&eqo->napi);
2822                         be_disable_busy_poll(eqo);
2823                 }
2824                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2825         }
2826
2827         be_async_mcc_disable(adapter);
2828
2829         /* Wait for all pending tx completions to arrive so that
2830          * all tx skbs are freed.
2831          */
2832         netif_tx_disable(netdev);
2833         be_tx_compl_clean(adapter);
2834
2835         be_rx_qs_destroy(adapter);
2836
2837         for (i = 1; i < (adapter->uc_macs + 1); i++)
2838                 be_cmd_pmac_del(adapter, adapter->if_handle,
2839                                 adapter->pmac_id[i], 0);
2840         adapter->uc_macs = 0;
2841
2842         for_all_evt_queues(adapter, eqo, i) {
2843                 if (msix_enabled(adapter))
2844                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2845                 else
2846                         synchronize_irq(netdev->irq);
2847                 be_eq_clean(eqo);
2848         }
2849
2850         be_irq_unregister(adapter);
2851
2852         return 0;
2853 }
2854
2855 static int be_rx_qs_create(struct be_adapter *adapter)
2856 {
2857         struct rss_info *rss = &adapter->rss_info;
2858         u8 rss_key[RSS_HASH_KEY_LEN];
2859         struct be_rx_obj *rxo;
2860         int rc, i, j;
2861
2862         for_all_rx_queues(adapter, rxo, i) {
2863                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2864                                     sizeof(struct be_eth_rx_d));
2865                 if (rc)
2866                         return rc;
2867         }
2868
2869         /* The FW would like the default RXQ to be created first */
2870         rxo = default_rxo(adapter);
2871         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2872                                adapter->if_handle, false, &rxo->rss_id);
2873         if (rc)
2874                 return rc;
2875
2876         for_all_rss_queues(adapter, rxo, i) {
2877                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2878                                        rx_frag_size, adapter->if_handle,
2879                                        true, &rxo->rss_id);
2880                 if (rc)
2881                         return rc;
2882         }
2883
2884         if (be_multi_rxq(adapter)) {
2885                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2886                         j += adapter->num_rx_qs - 1) {
2887                         for_all_rss_queues(adapter, rxo, i) {
2888                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2889                                         break;
2890                                 rss->rsstable[j + i] = rxo->rss_id;
2891                                 rss->rss_queue[j + i] = i;
2892                         }
2893                 }
2894                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2895                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2896
2897                 if (!BEx_chip(adapter))
2898                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2899                                 RSS_ENABLE_UDP_IPV6;
2900         } else {
2901                 /* Disable RSS, if only default RX Q is created */
2902                 rss->rss_flags = RSS_ENABLE_NONE;
2903         }
2904
2905         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
2906         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2907                                128, rss_key);
2908         if (rc) {
2909                 rss->rss_flags = RSS_ENABLE_NONE;
2910                 return rc;
2911         }
2912
2913         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
2914
2915         /* First time posting */
2916         for_all_rx_queues(adapter, rxo, i)
2917                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2918         return 0;
2919 }
2920
2921 static int be_open(struct net_device *netdev)
2922 {
2923         struct be_adapter *adapter = netdev_priv(netdev);
2924         struct be_eq_obj *eqo;
2925         struct be_rx_obj *rxo;
2926         struct be_tx_obj *txo;
2927         u8 link_status;
2928         int status, i;
2929
2930         status = be_rx_qs_create(adapter);
2931         if (status)
2932                 goto err;
2933
2934         status = be_irq_register(adapter);
2935         if (status)
2936                 goto err;
2937
2938         for_all_rx_queues(adapter, rxo, i)
2939                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2940
2941         for_all_tx_queues(adapter, txo, i)
2942                 be_cq_notify(adapter, txo->cq.id, true, 0);
2943
2944         be_async_mcc_enable(adapter);
2945
2946         for_all_evt_queues(adapter, eqo, i) {
2947                 napi_enable(&eqo->napi);
2948                 be_enable_busy_poll(eqo);
2949                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2950         }
2951         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2952
2953         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2954         if (!status)
2955                 be_link_status_update(adapter, link_status);
2956
2957         netif_tx_start_all_queues(netdev);
2958         be_roce_dev_open(adapter);
2959
2960 #ifdef CONFIG_BE2NET_VXLAN
2961         if (skyhawk_chip(adapter))
2962                 vxlan_get_rx_port(netdev);
2963 #endif
2964
2965         return 0;
2966 err:
2967         be_close(adapter->netdev);
2968         return -EIO;
2969 }
2970
2971 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2972 {
2973         struct be_dma_mem cmd;
2974         int status = 0;
2975         u8 mac[ETH_ALEN];
2976
2977         memset(mac, 0, ETH_ALEN);
2978
2979         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2980         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2981                                      GFP_KERNEL);
2982         if (!cmd.va)
2983                 return -ENOMEM;
2984
2985         if (enable) {
2986                 status = pci_write_config_dword(adapter->pdev,
2987                                                 PCICFG_PM_CONTROL_OFFSET,
2988                                                 PCICFG_PM_CONTROL_MASK);
2989                 if (status) {
2990                         dev_err(&adapter->pdev->dev,
2991                                 "Could not enable Wake-on-lan\n");
2992                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2993                                           cmd.dma);
2994                         return status;
2995                 }
2996                 status = be_cmd_enable_magic_wol(adapter,
2997                                                  adapter->netdev->dev_addr,
2998                                                  &cmd);
2999                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3000                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3001         } else {
3002                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3003                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3004                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3005         }
3006
3007         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3008         return status;
3009 }
3010
3011 /*
3012  * Generate a seed MAC address from the PF MAC Address using jhash.
3013  * MAC Address for VFs are assigned incrementally starting from the seed.
3014  * These addresses are programmed in the ASIC by the PF and the VF driver
3015  * queries for the MAC address during its probe.
3016  */
3017 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3018 {
3019         u32 vf;
3020         int status = 0;
3021         u8 mac[ETH_ALEN];
3022         struct be_vf_cfg *vf_cfg;
3023
3024         be_vf_eth_addr_generate(adapter, mac);
3025
3026         for_all_vfs(adapter, vf_cfg, vf) {
3027                 if (BEx_chip(adapter))
3028                         status = be_cmd_pmac_add(adapter, mac,
3029                                                  vf_cfg->if_handle,
3030                                                  &vf_cfg->pmac_id, vf + 1);
3031                 else
3032                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3033                                                 vf + 1);
3034
3035                 if (status)
3036                         dev_err(&adapter->pdev->dev,
3037                                 "Mac address assignment failed for VF %d\n",
3038                                 vf);
3039                 else
3040                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3041
3042                 mac[5] += 1;
3043         }
3044         return status;
3045 }
3046
3047 static int be_vfs_mac_query(struct be_adapter *adapter)
3048 {
3049         int status, vf;
3050         u8 mac[ETH_ALEN];
3051         struct be_vf_cfg *vf_cfg;
3052
3053         for_all_vfs(adapter, vf_cfg, vf) {
3054                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3055                                                mac, vf_cfg->if_handle,
3056                                                false, vf+1);
3057                 if (status)
3058                         return status;
3059                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3060         }
3061         return 0;
3062 }
3063
3064 static void be_vf_clear(struct be_adapter *adapter)
3065 {
3066         struct be_vf_cfg *vf_cfg;
3067         u32 vf;
3068
3069         if (pci_vfs_assigned(adapter->pdev)) {
3070                 dev_warn(&adapter->pdev->dev,
3071                          "VFs are assigned to VMs: not disabling VFs\n");
3072                 goto done;
3073         }
3074
3075         pci_disable_sriov(adapter->pdev);
3076
3077         for_all_vfs(adapter, vf_cfg, vf) {
3078                 if (BEx_chip(adapter))
3079                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3080                                         vf_cfg->pmac_id, vf + 1);
3081                 else
3082                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3083                                        vf + 1);
3084
3085                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3086         }
3087 done:
3088         kfree(adapter->vf_cfg);
3089         adapter->num_vfs = 0;
3090         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3091 }
3092
3093 static void be_clear_queues(struct be_adapter *adapter)
3094 {
3095         be_mcc_queues_destroy(adapter);
3096         be_rx_cqs_destroy(adapter);
3097         be_tx_queues_destroy(adapter);
3098         be_evt_queues_destroy(adapter);
3099 }
3100
3101 static void be_cancel_worker(struct be_adapter *adapter)
3102 {
3103         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3104                 cancel_delayed_work_sync(&adapter->work);
3105                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3106         }
3107 }
3108
3109 static void be_mac_clear(struct be_adapter *adapter)
3110 {
3111         int i;
3112
3113         if (adapter->pmac_id) {
3114                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3115                         be_cmd_pmac_del(adapter, adapter->if_handle,
3116                                         adapter->pmac_id[i], 0);
3117                 adapter->uc_macs = 0;
3118
3119                 kfree(adapter->pmac_id);
3120                 adapter->pmac_id = NULL;
3121         }
3122 }
3123
3124 #ifdef CONFIG_BE2NET_VXLAN
3125 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3126 {
3127         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3128                 be_cmd_manage_iface(adapter, adapter->if_handle,
3129                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3130
3131         if (adapter->vxlan_port)
3132                 be_cmd_set_vxlan_port(adapter, 0);
3133
3134         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3135         adapter->vxlan_port = 0;
3136 }
3137 #endif
3138
3139 static int be_clear(struct be_adapter *adapter)
3140 {
3141         be_cancel_worker(adapter);
3142
3143         if (sriov_enabled(adapter))
3144                 be_vf_clear(adapter);
3145
3146         /* Re-configure FW to distribute resources evenly across max-supported
3147          * number of VFs, only when VFs are not already enabled.
3148          */
3149         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3150                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3151                                         pci_sriov_get_totalvfs(adapter->pdev));
3152
3153 #ifdef CONFIG_BE2NET_VXLAN
3154         be_disable_vxlan_offloads(adapter);
3155 #endif
3156         /* delete the primary mac along with the uc-mac list */
3157         be_mac_clear(adapter);
3158
3159         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3160
3161         be_clear_queues(adapter);
3162
3163         be_msix_disable(adapter);
3164         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3165         return 0;
3166 }
3167
3168 static int be_vfs_if_create(struct be_adapter *adapter)
3169 {
3170         struct be_resources res = {0};
3171         struct be_vf_cfg *vf_cfg;
3172         u32 cap_flags, en_flags, vf;
3173         int status = 0;
3174
3175         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3176                     BE_IF_FLAGS_MULTICAST;
3177
3178         for_all_vfs(adapter, vf_cfg, vf) {
3179                 if (!BE3_chip(adapter)) {
3180                         status = be_cmd_get_profile_config(adapter, &res,
3181                                                            vf + 1);
3182                         if (!status)
3183                                 cap_flags = res.if_cap_flags;
3184                 }
3185
3186                 /* If a FW profile exists, then cap_flags are updated */
3187                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3188                                         BE_IF_FLAGS_BROADCAST |
3189                                         BE_IF_FLAGS_MULTICAST);
3190                 status =
3191                     be_cmd_if_create(adapter, cap_flags, en_flags,
3192                                      &vf_cfg->if_handle, vf + 1);
3193                 if (status)
3194                         goto err;
3195         }
3196 err:
3197         return status;
3198 }
3199
3200 static int be_vf_setup_init(struct be_adapter *adapter)
3201 {
3202         struct be_vf_cfg *vf_cfg;
3203         int vf;
3204
3205         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3206                                   GFP_KERNEL);
3207         if (!adapter->vf_cfg)
3208                 return -ENOMEM;
3209
3210         for_all_vfs(adapter, vf_cfg, vf) {
3211                 vf_cfg->if_handle = -1;
3212                 vf_cfg->pmac_id = -1;
3213         }
3214         return 0;
3215 }
3216
3217 static int be_vf_setup(struct be_adapter *adapter)
3218 {
3219         struct device *dev = &adapter->pdev->dev;
3220         struct be_vf_cfg *vf_cfg;
3221         int status, old_vfs, vf;
3222         u32 privileges;
3223
3224         old_vfs = pci_num_vf(adapter->pdev);
3225
3226         status = be_vf_setup_init(adapter);
3227         if (status)
3228                 goto err;
3229
3230         if (old_vfs) {
3231                 for_all_vfs(adapter, vf_cfg, vf) {
3232                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3233                         if (status)
3234                                 goto err;
3235                 }
3236
3237                 status = be_vfs_mac_query(adapter);
3238                 if (status)
3239                         goto err;
3240         } else {
3241                 status = be_vfs_if_create(adapter);
3242                 if (status)
3243                         goto err;
3244
3245                 status = be_vf_eth_addr_config(adapter);
3246                 if (status)
3247                         goto err;
3248         }
3249
3250         for_all_vfs(adapter, vf_cfg, vf) {
3251                 /* Allow VFs to programs MAC/VLAN filters */
3252                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3253                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3254                         status = be_cmd_set_fn_privileges(adapter,
3255                                                           privileges |
3256                                                           BE_PRIV_FILTMGMT,
3257                                                           vf + 1);
3258                         if (!status)
3259                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3260                                          vf);
3261                 }
3262
3263                 /* Allow full available bandwidth */
3264                 if (!old_vfs)
3265                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3266
3267                 if (!old_vfs) {
3268                         be_cmd_enable_vf(adapter, vf + 1);
3269                         be_cmd_set_logical_link_config(adapter,
3270                                                        IFLA_VF_LINK_STATE_AUTO,
3271                                                        vf+1);
3272                 }
3273         }
3274
3275         if (!old_vfs) {
3276                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3277                 if (status) {
3278                         dev_err(dev, "SRIOV enable failed\n");
3279                         adapter->num_vfs = 0;
3280                         goto err;
3281                 }
3282         }
3283
3284         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3285         return 0;
3286 err:
3287         dev_err(dev, "VF setup failed\n");
3288         be_vf_clear(adapter);
3289         return status;
3290 }
3291
3292 /* Converting function_mode bits on BE3 to SH mc_type enums */
3293
3294 static u8 be_convert_mc_type(u32 function_mode)
3295 {
3296         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3297                 return vNIC1;
3298         else if (function_mode & QNQ_MODE)
3299                 return FLEX10;
3300         else if (function_mode & VNIC_MODE)
3301                 return vNIC2;
3302         else if (function_mode & UMC_ENABLED)
3303                 return UMC;
3304         else
3305                 return MC_NONE;
3306 }
3307
3308 /* On BE2/BE3 FW does not suggest the supported limits */
3309 static void BEx_get_resources(struct be_adapter *adapter,
3310                               struct be_resources *res)
3311 {
3312         bool use_sriov = adapter->num_vfs ? 1 : 0;
3313
3314         if (be_physfn(adapter))
3315                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3316         else
3317                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3318
3319         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3320
3321         if (be_is_mc(adapter)) {
3322                 /* Assuming that there are 4 channels per port,
3323                  * when multi-channel is enabled
3324                  */
3325                 if (be_is_qnq_mode(adapter))
3326                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3327                 else
3328                         /* In a non-qnq multichannel mode, the pvid
3329                          * takes up one vlan entry
3330                          */
3331                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3332         } else {
3333                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3334         }
3335
3336         res->max_mcast_mac = BE_MAX_MC;
3337
3338         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3339          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3340          *    *only* if it is RSS-capable.
3341          */
3342         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3343             !be_physfn(adapter) || (be_is_mc(adapter) &&
3344             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3345                 res->max_tx_qs = 1;
3346         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3347                 struct be_resources super_nic_res = {0};
3348
3349                 /* On a SuperNIC profile, the driver needs to use the
3350                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3351                  */
3352                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3353                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3354                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3355         } else {
3356                 res->max_tx_qs = BE3_MAX_TX_QS;
3357         }
3358
3359         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3360             !use_sriov && be_physfn(adapter))
3361                 res->max_rss_qs = (adapter->be3_native) ?
3362                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3363         res->max_rx_qs = res->max_rss_qs + 1;
3364
3365         if (be_physfn(adapter))
3366                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3367                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3368         else
3369                 res->max_evt_qs = 1;
3370
3371         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3372         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3373                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3374 }
3375
3376 static void be_setup_init(struct be_adapter *adapter)
3377 {
3378         adapter->vlan_prio_bmap = 0xff;
3379         adapter->phy.link_speed = -1;
3380         adapter->if_handle = -1;
3381         adapter->be3_native = false;
3382         adapter->promiscuous = false;
3383         if (be_physfn(adapter))
3384                 adapter->cmd_privileges = MAX_PRIVILEGES;
3385         else
3386                 adapter->cmd_privileges = MIN_PRIVILEGES;
3387 }
3388
3389 static int be_get_sriov_config(struct be_adapter *adapter)
3390 {
3391         struct device *dev = &adapter->pdev->dev;
3392         struct be_resources res = {0};
3393         int max_vfs, old_vfs;
3394
3395         /* Some old versions of BE3 FW don't report max_vfs value */
3396         be_cmd_get_profile_config(adapter, &res, 0);
3397
3398         if (BE3_chip(adapter) && !res.max_vfs) {
3399                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3400                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3401         }
3402
3403         adapter->pool_res = res;
3404
3405         if (!be_max_vfs(adapter)) {
3406                 if (num_vfs)
3407                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3408                 adapter->num_vfs = 0;
3409                 return 0;
3410         }
3411
3412         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3413
3414         /* validate num_vfs module param */
3415         old_vfs = pci_num_vf(adapter->pdev);
3416         if (old_vfs) {
3417                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3418                 if (old_vfs != num_vfs)
3419                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3420                 adapter->num_vfs = old_vfs;
3421         } else {
3422                 if (num_vfs > be_max_vfs(adapter)) {
3423                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3424                                  num_vfs);
3425                         dev_info(dev, "Limiting to %d VFs\n",
3426                                  be_max_vfs(adapter));
3427                 }
3428                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3429         }
3430
3431         return 0;
3432 }
3433
3434 static int be_get_resources(struct be_adapter *adapter)
3435 {
3436         struct device *dev = &adapter->pdev->dev;
3437         struct be_resources res = {0};
3438         int status;
3439
3440         if (BEx_chip(adapter)) {
3441                 BEx_get_resources(adapter, &res);
3442                 adapter->res = res;
3443         }
3444
3445         /* For Lancer, SH etc read per-function resource limits from FW.
3446          * GET_FUNC_CONFIG returns per function guaranteed limits.
3447          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3448          */
3449         if (!BEx_chip(adapter)) {
3450                 status = be_cmd_get_func_config(adapter, &res);
3451                 if (status)
3452                         return status;
3453
3454                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3455                 if (be_roce_supported(adapter))
3456                         res.max_evt_qs /= 2;
3457                 adapter->res = res;
3458         }
3459
3460         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3461                  be_max_txqs(adapter), be_max_rxqs(adapter),
3462                  be_max_rss(adapter), be_max_eqs(adapter),
3463                  be_max_vfs(adapter));
3464         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3465                  be_max_uc(adapter), be_max_mc(adapter),
3466                  be_max_vlans(adapter));
3467
3468         return 0;
3469 }
3470
3471 static void be_sriov_config(struct be_adapter *adapter)
3472 {
3473         struct device *dev = &adapter->pdev->dev;
3474         int status;
3475
3476         status = be_get_sriov_config(adapter);
3477         if (status) {
3478                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3479                 dev_err(dev, "SR-IOV cannot be enabled\n");
3480                 return;
3481         }
3482
3483         /* When the HW is in SRIOV capable configuration, the PF-pool
3484          * resources are equally distributed across the max-number of
3485          * VFs. The user may request only a subset of the max-vfs to be
3486          * enabled. Based on num_vfs, redistribute the resources across
3487          * num_vfs so that each VF will have access to more number of
3488          * resources. This facility is not available in BE3 FW.
3489          * Also, this is done by FW in Lancer chip.
3490          */
3491         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3492                 status = be_cmd_set_sriov_config(adapter,
3493                                                  adapter->pool_res,
3494                                                  adapter->num_vfs);
3495                 if (status)
3496                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3497         }
3498 }
3499
3500 static int be_get_config(struct be_adapter *adapter)
3501 {
3502         u16 profile_id;
3503         int status;
3504
3505         status = be_cmd_query_fw_cfg(adapter);
3506         if (status)
3507                 return status;
3508
3509          if (be_physfn(adapter)) {
3510                 status = be_cmd_get_active_profile(adapter, &profile_id);
3511                 if (!status)
3512                         dev_info(&adapter->pdev->dev,
3513                                  "Using profile 0x%x\n", profile_id);
3514         }
3515
3516         if (!BE2_chip(adapter) && be_physfn(adapter))
3517                 be_sriov_config(adapter);
3518
3519         status = be_get_resources(adapter);
3520         if (status)
3521                 return status;
3522
3523         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3524                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3525         if (!adapter->pmac_id)
3526                 return -ENOMEM;
3527
3528         /* Sanitize cfg_num_qs based on HW and platform limits */
3529         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3530
3531         return 0;
3532 }
3533
3534 static int be_mac_setup(struct be_adapter *adapter)
3535 {
3536         u8 mac[ETH_ALEN];
3537         int status;
3538
3539         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3540                 status = be_cmd_get_perm_mac(adapter, mac);
3541                 if (status)
3542                         return status;
3543
3544                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3545                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3546         } else {
3547                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3548                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3549         }
3550
3551         /* For BE3-R VFs, the PF programs the initial MAC address */
3552         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3553                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3554                                 &adapter->pmac_id[0], 0);
3555         return 0;
3556 }
3557
3558 static void be_schedule_worker(struct be_adapter *adapter)
3559 {
3560         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3561         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3562 }
3563
3564 static int be_setup_queues(struct be_adapter *adapter)
3565 {
3566         struct net_device *netdev = adapter->netdev;
3567         int status;
3568
3569         status = be_evt_queues_create(adapter);
3570         if (status)
3571                 goto err;
3572
3573         status = be_tx_qs_create(adapter);
3574         if (status)
3575                 goto err;
3576
3577         status = be_rx_cqs_create(adapter);
3578         if (status)
3579                 goto err;
3580
3581         status = be_mcc_queues_create(adapter);
3582         if (status)
3583                 goto err;
3584
3585         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3586         if (status)
3587                 goto err;
3588
3589         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3590         if (status)
3591                 goto err;
3592
3593         return 0;
3594 err:
3595         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3596         return status;
3597 }
3598
3599 int be_update_queues(struct be_adapter *adapter)
3600 {
3601         struct net_device *netdev = adapter->netdev;
3602         int status;
3603
3604         if (netif_running(netdev))
3605                 be_close(netdev);
3606
3607         be_cancel_worker(adapter);
3608
3609         /* If any vectors have been shared with RoCE we cannot re-program
3610          * the MSIx table.
3611          */
3612         if (!adapter->num_msix_roce_vec)
3613                 be_msix_disable(adapter);
3614
3615         be_clear_queues(adapter);
3616
3617         if (!msix_enabled(adapter)) {
3618                 status = be_msix_enable(adapter);
3619                 if (status)
3620                         return status;
3621         }
3622
3623         status = be_setup_queues(adapter);
3624         if (status)
3625                 return status;
3626
3627         be_schedule_worker(adapter);
3628
3629         if (netif_running(netdev))
3630                 status = be_open(netdev);
3631
3632         return status;
3633 }
3634
3635 static int be_setup(struct be_adapter *adapter)
3636 {
3637         struct device *dev = &adapter->pdev->dev;
3638         u32 tx_fc, rx_fc, en_flags;
3639         int status;
3640
3641         be_setup_init(adapter);
3642
3643         if (!lancer_chip(adapter))
3644                 be_cmd_req_native_mode(adapter);
3645
3646         status = be_get_config(adapter);
3647         if (status)
3648                 goto err;
3649
3650         status = be_msix_enable(adapter);
3651         if (status)
3652                 goto err;
3653
3654         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3655                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3656         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3657                 en_flags |= BE_IF_FLAGS_RSS;
3658         en_flags = en_flags & be_if_cap_flags(adapter);
3659         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3660                                   &adapter->if_handle, 0);
3661         if (status)
3662                 goto err;
3663
3664         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3665         rtnl_lock();
3666         status = be_setup_queues(adapter);
3667         rtnl_unlock();
3668         if (status)
3669                 goto err;
3670
3671         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3672
3673         status = be_mac_setup(adapter);
3674         if (status)
3675                 goto err;
3676
3677         be_cmd_get_fw_ver(adapter);
3678         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3679
3680         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3681                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3682                         adapter->fw_ver);
3683                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3684         }
3685
3686         if (adapter->vlans_added)
3687                 be_vid_config(adapter);
3688
3689         be_set_rx_mode(adapter->netdev);
3690
3691         be_cmd_get_acpi_wol_cap(adapter);
3692
3693         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3694
3695         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3696                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3697                                         adapter->rx_fc);
3698
3699         if (be_physfn(adapter))
3700                 be_cmd_set_logical_link_config(adapter,
3701                                                IFLA_VF_LINK_STATE_AUTO, 0);
3702
3703         if (adapter->num_vfs)
3704                 be_vf_setup(adapter);
3705
3706         status = be_cmd_get_phy_info(adapter);
3707         if (!status && be_pause_supported(adapter))
3708                 adapter->phy.fc_autoneg = 1;
3709
3710         be_schedule_worker(adapter);
3711         adapter->flags |= BE_FLAGS_SETUP_DONE;
3712         return 0;
3713 err:
3714         be_clear(adapter);
3715         return status;
3716 }
3717
3718 #ifdef CONFIG_NET_POLL_CONTROLLER
3719 static void be_netpoll(struct net_device *netdev)
3720 {
3721         struct be_adapter *adapter = netdev_priv(netdev);
3722         struct be_eq_obj *eqo;
3723         int i;
3724
3725         for_all_evt_queues(adapter, eqo, i) {
3726                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3727                 napi_schedule(&eqo->napi);
3728         }
3729 }
3730 #endif
3731
3732 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3733
3734 static bool phy_flashing_required(struct be_adapter *adapter)
3735 {
3736         return (adapter->phy.phy_type == TN_8022 &&
3737                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3738 }
3739
3740 static bool is_comp_in_ufi(struct be_adapter *adapter,
3741                            struct flash_section_info *fsec, int type)
3742 {
3743         int i = 0, img_type = 0;
3744         struct flash_section_info_g2 *fsec_g2 = NULL;
3745
3746         if (BE2_chip(adapter))
3747                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3748
3749         for (i = 0; i < MAX_FLASH_COMP; i++) {
3750                 if (fsec_g2)
3751                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3752                 else
3753                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3754
3755                 if (img_type == type)
3756                         return true;
3757         }
3758         return false;
3759
3760 }
3761
3762 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3763                                                 int header_size,
3764                                                 const struct firmware *fw)
3765 {
3766         struct flash_section_info *fsec = NULL;
3767         const u8 *p = fw->data;
3768
3769         p += header_size;
3770         while (p < (fw->data + fw->size)) {
3771                 fsec = (struct flash_section_info *)p;
3772                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3773                         return fsec;
3774                 p += 32;
3775         }
3776         return NULL;
3777 }
3778
3779 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3780                               u32 img_offset, u32 img_size, int hdr_size,
3781                               u16 img_optype, bool *crc_match)
3782 {
3783         u32 crc_offset;
3784         int status;
3785         u8 crc[4];
3786
3787         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3788         if (status)
3789                 return status;
3790
3791         crc_offset = hdr_size + img_offset + img_size - 4;
3792
3793         /* Skip flashing, if crc of flashed region matches */
3794         if (!memcmp(crc, p + crc_offset, 4))
3795                 *crc_match = true;
3796         else
3797                 *crc_match = false;
3798
3799         return status;
3800 }
3801
3802 static int be_flash(struct be_adapter *adapter, const u8 *img,
3803                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3804 {
3805         struct be_cmd_write_flashrom *req = flash_cmd->va;
3806         u32 total_bytes, flash_op, num_bytes;
3807         int status;
3808
3809         total_bytes = img_size;
3810         while (total_bytes) {
3811                 num_bytes = min_t(u32, 32*1024, total_bytes);
3812
3813                 total_bytes -= num_bytes;
3814
3815                 if (!total_bytes) {
3816                         if (optype == OPTYPE_PHY_FW)
3817                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3818                         else
3819                                 flash_op = FLASHROM_OPER_FLASH;
3820                 } else {
3821                         if (optype == OPTYPE_PHY_FW)
3822                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3823                         else
3824                                 flash_op = FLASHROM_OPER_SAVE;
3825                 }
3826
3827                 memcpy(req->data_buf, img, num_bytes);
3828                 img += num_bytes;
3829                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3830                                                flash_op, num_bytes);
3831                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3832                     optype == OPTYPE_PHY_FW)
3833                         break;
3834                 else if (status)
3835                         return status;
3836         }
3837         return 0;
3838 }
3839
3840 /* For BE2, BE3 and BE3-R */
3841 static int be_flash_BEx(struct be_adapter *adapter,
3842                         const struct firmware *fw,
3843                         struct be_dma_mem *flash_cmd, int num_of_images)
3844 {
3845         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3846         struct device *dev = &adapter->pdev->dev;
3847         struct flash_section_info *fsec = NULL;
3848         int status, i, filehdr_size, num_comp;
3849         const struct flash_comp *pflashcomp;
3850         bool crc_match;
3851         const u8 *p;
3852
3853         struct flash_comp gen3_flash_types[] = {
3854                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3855                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3856                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3857                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3858                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3859                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3860                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3861                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3862                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3863                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3864                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3865                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3866                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3867                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3868                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3869                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3870                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3871                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3872                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3873                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3874         };
3875
3876         struct flash_comp gen2_flash_types[] = {
3877                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3878                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3879                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3880                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3881                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3882                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3883                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3884                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3885                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3886                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3887                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3888                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3889                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3890                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3891                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3892                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3893         };
3894
3895         if (BE3_chip(adapter)) {
3896                 pflashcomp = gen3_flash_types;
3897                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3898                 num_comp = ARRAY_SIZE(gen3_flash_types);
3899         } else {
3900                 pflashcomp = gen2_flash_types;
3901                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3902                 num_comp = ARRAY_SIZE(gen2_flash_types);
3903         }
3904
3905         /* Get flash section info*/
3906         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3907         if (!fsec) {
3908                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3909                 return -1;
3910         }
3911         for (i = 0; i < num_comp; i++) {
3912                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3913                         continue;
3914
3915                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3916                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3917                         continue;
3918
3919                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3920                     !phy_flashing_required(adapter))
3921                                 continue;
3922
3923                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3924                         status = be_check_flash_crc(adapter, fw->data,
3925                                                     pflashcomp[i].offset,
3926                                                     pflashcomp[i].size,
3927                                                     filehdr_size +
3928                                                     img_hdrs_size,
3929                                                     OPTYPE_REDBOOT, &crc_match);
3930                         if (status) {
3931                                 dev_err(dev,
3932                                         "Could not get CRC for 0x%x region\n",
3933                                         pflashcomp[i].optype);
3934                                 continue;
3935                         }
3936
3937                         if (crc_match)
3938                                 continue;
3939                 }
3940
3941                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3942                         img_hdrs_size;
3943                 if (p + pflashcomp[i].size > fw->data + fw->size)
3944                         return -1;
3945
3946                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3947                                   pflashcomp[i].size);
3948                 if (status) {
3949                         dev_err(dev, "Flashing section type 0x%x failed\n",
3950                                 pflashcomp[i].img_type);
3951                         return status;
3952                 }
3953         }
3954         return 0;
3955 }
3956
3957 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3958 {
3959         u32 img_type = le32_to_cpu(fsec_entry.type);
3960         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3961
3962         if (img_optype != 0xFFFF)
3963                 return img_optype;
3964
3965         switch (img_type) {
3966         case IMAGE_FIRMWARE_iSCSI:
3967                 img_optype = OPTYPE_ISCSI_ACTIVE;
3968                 break;
3969         case IMAGE_BOOT_CODE:
3970                 img_optype = OPTYPE_REDBOOT;
3971                 break;
3972         case IMAGE_OPTION_ROM_ISCSI:
3973                 img_optype = OPTYPE_BIOS;
3974                 break;
3975         case IMAGE_OPTION_ROM_PXE:
3976                 img_optype = OPTYPE_PXE_BIOS;
3977                 break;
3978         case IMAGE_OPTION_ROM_FCoE:
3979                 img_optype = OPTYPE_FCOE_BIOS;
3980                 break;
3981         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3982                 img_optype = OPTYPE_ISCSI_BACKUP;
3983                 break;
3984         case IMAGE_NCSI:
3985                 img_optype = OPTYPE_NCSI_FW;
3986                 break;
3987         case IMAGE_FLASHISM_JUMPVECTOR:
3988                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3989                 break;
3990         case IMAGE_FIRMWARE_PHY:
3991                 img_optype = OPTYPE_SH_PHY_FW;
3992                 break;
3993         case IMAGE_REDBOOT_DIR:
3994                 img_optype = OPTYPE_REDBOOT_DIR;
3995                 break;
3996         case IMAGE_REDBOOT_CONFIG:
3997                 img_optype = OPTYPE_REDBOOT_CONFIG;
3998                 break;
3999         case IMAGE_UFI_DIR:
4000                 img_optype = OPTYPE_UFI_DIR;
4001                 break;
4002         default:
4003                 break;
4004         }
4005
4006         return img_optype;
4007 }
4008
4009 static int be_flash_skyhawk(struct be_adapter *adapter,
4010                             const struct firmware *fw,
4011                             struct be_dma_mem *flash_cmd, int num_of_images)
4012 {
4013         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4014         struct device *dev = &adapter->pdev->dev;
4015         struct flash_section_info *fsec = NULL;
4016         u32 img_offset, img_size, img_type;
4017         int status, i, filehdr_size;
4018         bool crc_match, old_fw_img;
4019         u16 img_optype;
4020         const u8 *p;
4021
4022         filehdr_size = sizeof(struct flash_file_hdr_g3);
4023         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4024         if (!fsec) {
4025                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4026                 return -EINVAL;
4027         }
4028
4029         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4030                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4031                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4032                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4033                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4034                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4035
4036                 if (img_optype == 0xFFFF)
4037                         continue;
4038                 /* Don't bother verifying CRC if an old FW image is being
4039                  * flashed
4040                  */
4041                 if (old_fw_img)
4042                         goto flash;
4043
4044                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4045                                             img_size, filehdr_size +
4046                                             img_hdrs_size, img_optype,
4047                                             &crc_match);
4048                 /* The current FW image on the card does not recognize the new
4049                  * FLASH op_type. The FW download is partially complete.
4050                  * Reboot the server now to enable FW image to recognize the
4051                  * new FLASH op_type. To complete the remaining process,
4052                  * download the same FW again after the reboot.
4053                  */
4054                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4055                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4056                         dev_err(dev, "Flash incomplete. Reset the server\n");
4057                         dev_err(dev, "Download FW image again after reset\n");
4058                         return -EAGAIN;
4059                 } else if (status) {
4060                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4061                                 img_optype);
4062                         return -EFAULT;
4063                 }
4064
4065                 if (crc_match)
4066                         continue;
4067
4068 flash:
4069                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4070                 if (p + img_size > fw->data + fw->size)
4071                         return -1;
4072
4073                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4074                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4075                  * UFI_DIR region
4076                  */
4077                 if (old_fw_img &&
4078                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4079                      (img_optype == OPTYPE_UFI_DIR &&
4080                       base_status(status) == MCC_STATUS_FAILED))) {
4081                         continue;
4082                 } else if (status) {
4083                         dev_err(dev, "Flashing section type 0x%x failed\n",
4084                                 img_type);
4085                         return -EFAULT;
4086                 }
4087         }
4088         return 0;
4089 }
4090
4091 static int lancer_fw_download(struct be_adapter *adapter,
4092                               const struct firmware *fw)
4093 {
4094 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4095 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4096         struct device *dev = &adapter->pdev->dev;
4097         struct be_dma_mem flash_cmd;
4098         const u8 *data_ptr = NULL;
4099         u8 *dest_image_ptr = NULL;
4100         size_t image_size = 0;
4101         u32 chunk_size = 0;
4102         u32 data_written = 0;
4103         u32 offset = 0;
4104         int status = 0;
4105         u8 add_status = 0;
4106         u8 change_status;
4107
4108         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4109                 dev_err(dev, "FW image size should be multiple of 4\n");
4110                 return -EINVAL;
4111         }
4112
4113         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4114                                 + LANCER_FW_DOWNLOAD_CHUNK;
4115         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4116                                           &flash_cmd.dma, GFP_KERNEL);
4117         if (!flash_cmd.va)
4118                 return -ENOMEM;
4119
4120         dest_image_ptr = flash_cmd.va +
4121                                 sizeof(struct lancer_cmd_req_write_object);
4122         image_size = fw->size;
4123         data_ptr = fw->data;
4124
4125         while (image_size) {
4126                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4127
4128                 /* Copy the image chunk content. */
4129                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4130
4131                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4132                                                  chunk_size, offset,
4133                                                  LANCER_FW_DOWNLOAD_LOCATION,
4134                                                  &data_written, &change_status,
4135                                                  &add_status);
4136                 if (status)
4137                         break;
4138
4139                 offset += data_written;
4140                 data_ptr += data_written;
4141                 image_size -= data_written;
4142         }
4143
4144         if (!status) {
4145                 /* Commit the FW written */
4146                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4147                                                  0, offset,
4148                                                  LANCER_FW_DOWNLOAD_LOCATION,
4149                                                  &data_written, &change_status,
4150                                                  &add_status);
4151         }
4152
4153         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4154         if (status) {
4155                 dev_err(dev, "Firmware load error\n");
4156                 return be_cmd_status(status);
4157         }
4158
4159         dev_info(dev, "Firmware flashed successfully\n");
4160
4161         if (change_status == LANCER_FW_RESET_NEEDED) {
4162                 dev_info(dev, "Resetting adapter to activate new FW\n");
4163                 status = lancer_physdev_ctrl(adapter,
4164                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4165                 if (status) {
4166                         dev_err(dev, "Adapter busy, could not reset FW\n");
4167                         dev_err(dev, "Reboot server to activate new FW\n");
4168                 }
4169         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4170                 dev_info(dev, "Reboot server to activate new FW\n");
4171         }
4172
4173         return 0;
4174 }
4175
4176 #define UFI_TYPE2               2
4177 #define UFI_TYPE3               3
4178 #define UFI_TYPE3R              10
4179 #define UFI_TYPE4               4
4180 static int be_get_ufi_type(struct be_adapter *adapter,
4181                            struct flash_file_hdr_g3 *fhdr)
4182 {
4183         if (!fhdr)
4184                 goto be_get_ufi_exit;
4185
4186         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4187                 return UFI_TYPE4;
4188         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4189                 if (fhdr->asic_type_rev == 0x10)
4190                         return UFI_TYPE3R;
4191                 else
4192                         return UFI_TYPE3;
4193         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4194                 return UFI_TYPE2;
4195
4196 be_get_ufi_exit:
4197         dev_err(&adapter->pdev->dev,
4198                 "UFI and Interface are not compatible for flashing\n");
4199         return -1;
4200 }
4201
4202 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4203 {
4204         struct flash_file_hdr_g3 *fhdr3;
4205         struct image_hdr *img_hdr_ptr = NULL;
4206         struct be_dma_mem flash_cmd;
4207         const u8 *p;
4208         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4209
4210         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4211         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4212                                           &flash_cmd.dma, GFP_KERNEL);
4213         if (!flash_cmd.va) {
4214                 status = -ENOMEM;
4215                 goto be_fw_exit;
4216         }
4217
4218         p = fw->data;
4219         fhdr3 = (struct flash_file_hdr_g3 *)p;
4220
4221         ufi_type = be_get_ufi_type(adapter, fhdr3);
4222
4223         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4224         for (i = 0; i < num_imgs; i++) {
4225                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4226                                 (sizeof(struct flash_file_hdr_g3) +
4227                                  i * sizeof(struct image_hdr)));
4228                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4229                         switch (ufi_type) {
4230                         case UFI_TYPE4:
4231                                 status = be_flash_skyhawk(adapter, fw,
4232                                                           &flash_cmd, num_imgs);
4233                                 break;
4234                         case UFI_TYPE3R:
4235                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4236                                                       num_imgs);
4237                                 break;
4238                         case UFI_TYPE3:
4239                                 /* Do not flash this ufi on BE3-R cards */
4240                                 if (adapter->asic_rev < 0x10)
4241                                         status = be_flash_BEx(adapter, fw,
4242                                                               &flash_cmd,
4243                                                               num_imgs);
4244                                 else {
4245                                         status = -EINVAL;
4246                                         dev_err(&adapter->pdev->dev,
4247                                                 "Can't load BE3 UFI on BE3R\n");
4248                                 }
4249                         }
4250                 }
4251         }
4252
4253         if (ufi_type == UFI_TYPE2)
4254                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4255         else if (ufi_type == -1)
4256                 status = -EINVAL;
4257
4258         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4259                           flash_cmd.dma);
4260         if (status) {
4261                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4262                 goto be_fw_exit;
4263         }
4264
4265         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4266
4267 be_fw_exit:
4268         return status;
4269 }
4270
4271 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4272 {
4273         const struct firmware *fw;
4274         int status;
4275
4276         if (!netif_running(adapter->netdev)) {
4277                 dev_err(&adapter->pdev->dev,
4278                         "Firmware load not allowed (interface is down)\n");
4279                 return -ENETDOWN;
4280         }
4281
4282         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4283         if (status)
4284                 goto fw_exit;
4285
4286         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4287
4288         if (lancer_chip(adapter))
4289                 status = lancer_fw_download(adapter, fw);
4290         else
4291                 status = be_fw_download(adapter, fw);
4292
4293         if (!status)
4294                 be_cmd_get_fw_ver(adapter);
4295
4296 fw_exit:
4297         release_firmware(fw);
4298         return status;
4299 }
4300
4301 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4302 {
4303         struct be_adapter *adapter = netdev_priv(dev);
4304         struct nlattr *attr, *br_spec;
4305         int rem;
4306         int status = 0;
4307         u16 mode = 0;
4308
4309         if (!sriov_enabled(adapter))
4310                 return -EOPNOTSUPP;
4311
4312         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4313         if (!br_spec)
4314                 return -EINVAL;
4315
4316         nla_for_each_nested(attr, br_spec, rem) {
4317                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4318                         continue;
4319
4320                 if (nla_len(attr) < sizeof(mode))
4321                         return -EINVAL;
4322
4323                 mode = nla_get_u16(attr);
4324                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4325                         return -EINVAL;
4326
4327                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4328                                                adapter->if_handle,
4329                                                mode == BRIDGE_MODE_VEPA ?
4330                                                PORT_FWD_TYPE_VEPA :
4331                                                PORT_FWD_TYPE_VEB);
4332                 if (status)
4333                         goto err;
4334
4335                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4336                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4337
4338                 return status;
4339         }
4340 err:
4341         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4342                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4343
4344         return status;
4345 }
4346
4347 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4348                                  struct net_device *dev, u32 filter_mask)
4349 {
4350         struct be_adapter *adapter = netdev_priv(dev);
4351         int status = 0;
4352         u8 hsw_mode;
4353
4354         if (!sriov_enabled(adapter))
4355                 return 0;
4356
4357         /* BE and Lancer chips support VEB mode only */
4358         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4359                 hsw_mode = PORT_FWD_TYPE_VEB;
4360         } else {
4361                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4362                                                adapter->if_handle, &hsw_mode);
4363                 if (status)
4364                         return 0;
4365         }
4366
4367         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4368                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4369                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4370                                        0, 0);
4371 }
4372
4373 #ifdef CONFIG_BE2NET_VXLAN
4374 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4375                               __be16 port)
4376 {
4377         struct be_adapter *adapter = netdev_priv(netdev);
4378         struct device *dev = &adapter->pdev->dev;
4379         int status;
4380
4381         if (lancer_chip(adapter) || BEx_chip(adapter))
4382                 return;
4383
4384         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4385                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4386                          be16_to_cpu(port));
4387                 dev_info(dev,
4388                          "Only one UDP port supported for VxLAN offloads\n");
4389                 return;
4390         }
4391
4392         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4393                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4394         if (status) {
4395                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4396                 goto err;
4397         }
4398
4399         status = be_cmd_set_vxlan_port(adapter, port);
4400         if (status) {
4401                 dev_warn(dev, "Failed to add VxLAN port\n");
4402                 goto err;
4403         }
4404         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4405         adapter->vxlan_port = port;
4406
4407         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4408                  be16_to_cpu(port));
4409         return;
4410 err:
4411         be_disable_vxlan_offloads(adapter);
4412 }
4413
4414 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4415                               __be16 port)
4416 {
4417         struct be_adapter *adapter = netdev_priv(netdev);
4418
4419         if (lancer_chip(adapter) || BEx_chip(adapter))
4420                 return;
4421
4422         if (adapter->vxlan_port != port)
4423                 return;
4424
4425         be_disable_vxlan_offloads(adapter);
4426
4427         dev_info(&adapter->pdev->dev,
4428                  "Disabled VxLAN offloads for UDP port %d\n",
4429                  be16_to_cpu(port));
4430 }
4431
4432 static bool be_gso_check(struct sk_buff *skb, struct net_device *dev)
4433 {
4434         return vxlan_gso_check(skb);
4435 }
4436 #endif
4437
4438 static const struct net_device_ops be_netdev_ops = {
4439         .ndo_open               = be_open,
4440         .ndo_stop               = be_close,
4441         .ndo_start_xmit         = be_xmit,
4442         .ndo_set_rx_mode        = be_set_rx_mode,
4443         .ndo_set_mac_address    = be_mac_addr_set,
4444         .ndo_change_mtu         = be_change_mtu,
4445         .ndo_get_stats64        = be_get_stats64,
4446         .ndo_validate_addr      = eth_validate_addr,
4447         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4448         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4449         .ndo_set_vf_mac         = be_set_vf_mac,
4450         .ndo_set_vf_vlan        = be_set_vf_vlan,
4451         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4452         .ndo_get_vf_config      = be_get_vf_config,
4453         .ndo_set_vf_link_state  = be_set_vf_link_state,
4454 #ifdef CONFIG_NET_POLL_CONTROLLER
4455         .ndo_poll_controller    = be_netpoll,
4456 #endif
4457         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4458         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4459 #ifdef CONFIG_NET_RX_BUSY_POLL
4460         .ndo_busy_poll          = be_busy_poll,
4461 #endif
4462 #ifdef CONFIG_BE2NET_VXLAN
4463         .ndo_add_vxlan_port     = be_add_vxlan_port,
4464         .ndo_del_vxlan_port     = be_del_vxlan_port,
4465         .ndo_gso_check          = be_gso_check,
4466 #endif
4467 };
4468
4469 static void be_netdev_init(struct net_device *netdev)
4470 {
4471         struct be_adapter *adapter = netdev_priv(netdev);
4472
4473         if (skyhawk_chip(adapter)) {
4474                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4475                                            NETIF_F_TSO | NETIF_F_TSO6 |
4476                                            NETIF_F_GSO_UDP_TUNNEL;
4477                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4478         }
4479         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4480                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4481                 NETIF_F_HW_VLAN_CTAG_TX;
4482         if (be_multi_rxq(adapter))
4483                 netdev->hw_features |= NETIF_F_RXHASH;
4484
4485         netdev->features |= netdev->hw_features |
4486                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4487
4488         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4489                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4490
4491         netdev->priv_flags |= IFF_UNICAST_FLT;
4492
4493         netdev->flags |= IFF_MULTICAST;
4494
4495         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4496
4497         netdev->netdev_ops = &be_netdev_ops;
4498
4499         netdev->ethtool_ops = &be_ethtool_ops;
4500 }
4501
4502 static void be_unmap_pci_bars(struct be_adapter *adapter)
4503 {
4504         if (adapter->csr)
4505                 pci_iounmap(adapter->pdev, adapter->csr);
4506         if (adapter->db)
4507                 pci_iounmap(adapter->pdev, adapter->db);
4508 }
4509
4510 static int db_bar(struct be_adapter *adapter)
4511 {
4512         if (lancer_chip(adapter) || !be_physfn(adapter))
4513                 return 0;
4514         else
4515                 return 4;
4516 }
4517
4518 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4519 {
4520         if (skyhawk_chip(adapter)) {
4521                 adapter->roce_db.size = 4096;
4522                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4523                                                               db_bar(adapter));
4524                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4525                                                                db_bar(adapter));
4526         }
4527         return 0;
4528 }
4529
4530 static int be_map_pci_bars(struct be_adapter *adapter)
4531 {
4532         u8 __iomem *addr;
4533
4534         if (BEx_chip(adapter) && be_physfn(adapter)) {
4535                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4536                 if (!adapter->csr)
4537                         return -ENOMEM;
4538         }
4539
4540         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4541         if (!addr)
4542                 goto pci_map_err;
4543         adapter->db = addr;
4544
4545         be_roce_map_pci_bars(adapter);
4546         return 0;
4547
4548 pci_map_err:
4549         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4550         be_unmap_pci_bars(adapter);
4551         return -ENOMEM;
4552 }
4553
4554 static void be_ctrl_cleanup(struct be_adapter *adapter)
4555 {
4556         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4557
4558         be_unmap_pci_bars(adapter);
4559
4560         if (mem->va)
4561                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4562                                   mem->dma);
4563
4564         mem = &adapter->rx_filter;
4565         if (mem->va)
4566                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4567                                   mem->dma);
4568 }
4569
4570 static int be_ctrl_init(struct be_adapter *adapter)
4571 {
4572         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4573         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4574         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4575         u32 sli_intf;
4576         int status;
4577
4578         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4579         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4580                                  SLI_INTF_FAMILY_SHIFT;
4581         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4582
4583         status = be_map_pci_bars(adapter);
4584         if (status)
4585                 goto done;
4586
4587         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4588         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4589                                                 mbox_mem_alloc->size,
4590                                                 &mbox_mem_alloc->dma,
4591                                                 GFP_KERNEL);
4592         if (!mbox_mem_alloc->va) {
4593                 status = -ENOMEM;
4594                 goto unmap_pci_bars;
4595         }
4596         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4597         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4598         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4599         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4600
4601         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4602         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4603                                             rx_filter->size, &rx_filter->dma,
4604                                             GFP_KERNEL);
4605         if (!rx_filter->va) {
4606                 status = -ENOMEM;
4607                 goto free_mbox;
4608         }
4609
4610         mutex_init(&adapter->mbox_lock);
4611         spin_lock_init(&adapter->mcc_lock);
4612         spin_lock_init(&adapter->mcc_cq_lock);
4613
4614         init_completion(&adapter->et_cmd_compl);
4615         pci_save_state(adapter->pdev);
4616         return 0;
4617
4618 free_mbox:
4619         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4620                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4621
4622 unmap_pci_bars:
4623         be_unmap_pci_bars(adapter);
4624
4625 done:
4626         return status;
4627 }
4628
4629 static void be_stats_cleanup(struct be_adapter *adapter)
4630 {
4631         struct be_dma_mem *cmd = &adapter->stats_cmd;
4632
4633         if (cmd->va)
4634                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4635                                   cmd->va, cmd->dma);
4636 }
4637
4638 static int be_stats_init(struct be_adapter *adapter)
4639 {
4640         struct be_dma_mem *cmd = &adapter->stats_cmd;
4641
4642         if (lancer_chip(adapter))
4643                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4644         else if (BE2_chip(adapter))
4645                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4646         else if (BE3_chip(adapter))
4647                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4648         else
4649                 /* ALL non-BE ASICs */
4650                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4651
4652         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4653                                       GFP_KERNEL);
4654         if (!cmd->va)
4655                 return -ENOMEM;
4656         return 0;
4657 }
4658
4659 static void be_remove(struct pci_dev *pdev)
4660 {
4661         struct be_adapter *adapter = pci_get_drvdata(pdev);
4662
4663         if (!adapter)
4664                 return;
4665
4666         be_roce_dev_remove(adapter);
4667         be_intr_set(adapter, false);
4668
4669         cancel_delayed_work_sync(&adapter->func_recovery_work);
4670
4671         unregister_netdev(adapter->netdev);
4672
4673         be_clear(adapter);
4674
4675         /* tell fw we're done with firing cmds */
4676         be_cmd_fw_clean(adapter);
4677
4678         be_stats_cleanup(adapter);
4679
4680         be_ctrl_cleanup(adapter);
4681
4682         pci_disable_pcie_error_reporting(pdev);
4683
4684         pci_release_regions(pdev);
4685         pci_disable_device(pdev);
4686
4687         free_netdev(adapter->netdev);
4688 }
4689
4690 static int be_get_initial_config(struct be_adapter *adapter)
4691 {
4692         int status, level;
4693
4694         status = be_cmd_get_cntl_attributes(adapter);
4695         if (status)
4696                 return status;
4697
4698         /* Must be a power of 2 or else MODULO will BUG_ON */
4699         adapter->be_get_temp_freq = 64;
4700
4701         if (BEx_chip(adapter)) {
4702                 level = be_cmd_get_fw_log_level(adapter);
4703                 adapter->msg_enable =
4704                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4705         }
4706
4707         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4708         return 0;
4709 }
4710
4711 static int lancer_recover_func(struct be_adapter *adapter)
4712 {
4713         struct device *dev = &adapter->pdev->dev;
4714         int status;
4715
4716         status = lancer_test_and_set_rdy_state(adapter);
4717         if (status)
4718                 goto err;
4719
4720         if (netif_running(adapter->netdev))
4721                 be_close(adapter->netdev);
4722
4723         be_clear(adapter);
4724
4725         be_clear_all_error(adapter);
4726
4727         status = be_setup(adapter);
4728         if (status)
4729                 goto err;
4730
4731         if (netif_running(adapter->netdev)) {
4732                 status = be_open(adapter->netdev);
4733                 if (status)
4734                         goto err;
4735         }
4736
4737         dev_err(dev, "Adapter recovery successful\n");
4738         return 0;
4739 err:
4740         if (status == -EAGAIN)
4741                 dev_err(dev, "Waiting for resource provisioning\n");
4742         else
4743                 dev_err(dev, "Adapter recovery failed\n");
4744
4745         return status;
4746 }
4747
4748 static void be_func_recovery_task(struct work_struct *work)
4749 {
4750         struct be_adapter *adapter =
4751                 container_of(work, struct be_adapter,  func_recovery_work.work);
4752         int status = 0;
4753
4754         be_detect_error(adapter);
4755
4756         if (adapter->hw_error && lancer_chip(adapter)) {
4757                 rtnl_lock();
4758                 netif_device_detach(adapter->netdev);
4759                 rtnl_unlock();
4760
4761                 status = lancer_recover_func(adapter);
4762                 if (!status)
4763                         netif_device_attach(adapter->netdev);
4764         }
4765
4766         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4767          * no need to attempt further recovery.
4768          */
4769         if (!status || status == -EAGAIN)
4770                 schedule_delayed_work(&adapter->func_recovery_work,
4771                                       msecs_to_jiffies(1000));
4772 }
4773
4774 static void be_worker(struct work_struct *work)
4775 {
4776         struct be_adapter *adapter =
4777                 container_of(work, struct be_adapter, work.work);
4778         struct be_rx_obj *rxo;
4779         int i;
4780
4781         /* when interrupts are not yet enabled, just reap any pending
4782         * mcc completions */
4783         if (!netif_running(adapter->netdev)) {
4784                 local_bh_disable();
4785                 be_process_mcc(adapter);
4786                 local_bh_enable();
4787                 goto reschedule;
4788         }
4789
4790         if (!adapter->stats_cmd_sent) {
4791                 if (lancer_chip(adapter))
4792                         lancer_cmd_get_pport_stats(adapter,
4793                                                    &adapter->stats_cmd);
4794                 else
4795                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4796         }
4797
4798         if (be_physfn(adapter) &&
4799             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4800                 be_cmd_get_die_temperature(adapter);
4801
4802         for_all_rx_queues(adapter, rxo, i) {
4803                 /* Replenish RX-queues starved due to memory
4804                  * allocation failures.
4805                  */
4806                 if (rxo->rx_post_starved)
4807                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4808         }
4809
4810         be_eqd_update(adapter);
4811
4812 reschedule:
4813         adapter->work_counter++;
4814         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4815 }
4816
4817 /* If any VFs are already enabled don't FLR the PF */
4818 static bool be_reset_required(struct be_adapter *adapter)
4819 {
4820         return pci_num_vf(adapter->pdev) ? false : true;
4821 }
4822
4823 static char *mc_name(struct be_adapter *adapter)
4824 {
4825         char *str = ""; /* default */
4826
4827         switch (adapter->mc_type) {
4828         case UMC:
4829                 str = "UMC";
4830                 break;
4831         case FLEX10:
4832                 str = "FLEX10";
4833                 break;
4834         case vNIC1:
4835                 str = "vNIC-1";
4836                 break;
4837         case nPAR:
4838                 str = "nPAR";
4839                 break;
4840         case UFP:
4841                 str = "UFP";
4842                 break;
4843         case vNIC2:
4844                 str = "vNIC-2";
4845                 break;
4846         default:
4847                 str = "";
4848         }
4849
4850         return str;
4851 }
4852
4853 static inline char *func_name(struct be_adapter *adapter)
4854 {
4855         return be_physfn(adapter) ? "PF" : "VF";
4856 }
4857
4858 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4859 {
4860         int status = 0;
4861         struct be_adapter *adapter;
4862         struct net_device *netdev;
4863         char port_name;
4864
4865         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4866
4867         status = pci_enable_device(pdev);
4868         if (status)
4869                 goto do_none;
4870
4871         status = pci_request_regions(pdev, DRV_NAME);
4872         if (status)
4873                 goto disable_dev;
4874         pci_set_master(pdev);
4875
4876         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4877         if (!netdev) {
4878                 status = -ENOMEM;
4879                 goto rel_reg;
4880         }
4881         adapter = netdev_priv(netdev);
4882         adapter->pdev = pdev;
4883         pci_set_drvdata(pdev, adapter);
4884         adapter->netdev = netdev;
4885         SET_NETDEV_DEV(netdev, &pdev->dev);
4886
4887         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4888         if (!status) {
4889                 netdev->features |= NETIF_F_HIGHDMA;
4890         } else {
4891                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4892                 if (status) {
4893                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4894                         goto free_netdev;
4895                 }
4896         }
4897
4898         status = pci_enable_pcie_error_reporting(pdev);
4899         if (!status)
4900                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4901
4902         status = be_ctrl_init(adapter);
4903         if (status)
4904                 goto free_netdev;
4905
4906         /* sync up with fw's ready state */
4907         if (be_physfn(adapter)) {
4908                 status = be_fw_wait_ready(adapter);
4909                 if (status)
4910                         goto ctrl_clean;
4911         }
4912
4913         if (be_reset_required(adapter)) {
4914                 status = be_cmd_reset_function(adapter);
4915                 if (status)
4916                         goto ctrl_clean;
4917
4918                 /* Wait for interrupts to quiesce after an FLR */
4919                 msleep(100);
4920         }
4921
4922         /* Allow interrupts for other ULPs running on NIC function */
4923         be_intr_set(adapter, true);
4924
4925         /* tell fw we're ready to fire cmds */
4926         status = be_cmd_fw_init(adapter);
4927         if (status)
4928                 goto ctrl_clean;
4929
4930         status = be_stats_init(adapter);
4931         if (status)
4932                 goto ctrl_clean;
4933
4934         status = be_get_initial_config(adapter);
4935         if (status)
4936                 goto stats_clean;
4937
4938         INIT_DELAYED_WORK(&adapter->work, be_worker);
4939         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4940         adapter->rx_fc = true;
4941         adapter->tx_fc = true;
4942
4943         status = be_setup(adapter);
4944         if (status)
4945                 goto stats_clean;
4946
4947         be_netdev_init(netdev);
4948         status = register_netdev(netdev);
4949         if (status != 0)
4950                 goto unsetup;
4951
4952         be_roce_dev_add(adapter);
4953
4954         schedule_delayed_work(&adapter->func_recovery_work,
4955                               msecs_to_jiffies(1000));
4956
4957         be_cmd_query_port_name(adapter, &port_name);
4958
4959         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4960                  func_name(adapter), mc_name(adapter), port_name);
4961
4962         return 0;
4963
4964 unsetup:
4965         be_clear(adapter);
4966 stats_clean:
4967         be_stats_cleanup(adapter);
4968 ctrl_clean:
4969         be_ctrl_cleanup(adapter);
4970 free_netdev:
4971         free_netdev(netdev);
4972 rel_reg:
4973         pci_release_regions(pdev);
4974 disable_dev:
4975         pci_disable_device(pdev);
4976 do_none:
4977         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4978         return status;
4979 }
4980
4981 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4982 {
4983         struct be_adapter *adapter = pci_get_drvdata(pdev);
4984         struct net_device *netdev =  adapter->netdev;
4985
4986         if (adapter->wol_en)
4987                 be_setup_wol(adapter, true);
4988
4989         be_intr_set(adapter, false);
4990         cancel_delayed_work_sync(&adapter->func_recovery_work);
4991
4992         netif_device_detach(netdev);
4993         if (netif_running(netdev)) {
4994                 rtnl_lock();
4995                 be_close(netdev);
4996                 rtnl_unlock();
4997         }
4998         be_clear(adapter);
4999
5000         pci_save_state(pdev);
5001         pci_disable_device(pdev);
5002         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5003         return 0;
5004 }
5005
5006 static int be_resume(struct pci_dev *pdev)
5007 {
5008         int status = 0;
5009         struct be_adapter *adapter = pci_get_drvdata(pdev);
5010         struct net_device *netdev =  adapter->netdev;
5011
5012         netif_device_detach(netdev);
5013
5014         status = pci_enable_device(pdev);
5015         if (status)
5016                 return status;
5017
5018         pci_set_power_state(pdev, PCI_D0);
5019         pci_restore_state(pdev);
5020
5021         status = be_fw_wait_ready(adapter);
5022         if (status)
5023                 return status;
5024
5025         be_intr_set(adapter, true);
5026         /* tell fw we're ready to fire cmds */
5027         status = be_cmd_fw_init(adapter);
5028         if (status)
5029                 return status;
5030
5031         be_setup(adapter);
5032         if (netif_running(netdev)) {
5033                 rtnl_lock();
5034                 be_open(netdev);
5035                 rtnl_unlock();
5036         }
5037
5038         schedule_delayed_work(&adapter->func_recovery_work,
5039                               msecs_to_jiffies(1000));
5040         netif_device_attach(netdev);
5041
5042         if (adapter->wol_en)
5043                 be_setup_wol(adapter, false);
5044
5045         return 0;
5046 }
5047
5048 /*
5049  * An FLR will stop BE from DMAing any data.
5050  */
5051 static void be_shutdown(struct pci_dev *pdev)
5052 {
5053         struct be_adapter *adapter = pci_get_drvdata(pdev);
5054
5055         if (!adapter)
5056                 return;
5057
5058         be_roce_dev_shutdown(adapter);
5059         cancel_delayed_work_sync(&adapter->work);
5060         cancel_delayed_work_sync(&adapter->func_recovery_work);
5061
5062         netif_device_detach(adapter->netdev);
5063
5064         be_cmd_reset_function(adapter);
5065
5066         pci_disable_device(pdev);
5067 }
5068
5069 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5070                                             pci_channel_state_t state)
5071 {
5072         struct be_adapter *adapter = pci_get_drvdata(pdev);
5073         struct net_device *netdev =  adapter->netdev;
5074
5075         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5076
5077         if (!adapter->eeh_error) {
5078                 adapter->eeh_error = true;
5079
5080                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5081
5082                 rtnl_lock();
5083                 netif_device_detach(netdev);
5084                 if (netif_running(netdev))
5085                         be_close(netdev);
5086                 rtnl_unlock();
5087
5088                 be_clear(adapter);
5089         }
5090
5091         if (state == pci_channel_io_perm_failure)
5092                 return PCI_ERS_RESULT_DISCONNECT;
5093
5094         pci_disable_device(pdev);
5095
5096         /* The error could cause the FW to trigger a flash debug dump.
5097          * Resetting the card while flash dump is in progress
5098          * can cause it not to recover; wait for it to finish.
5099          * Wait only for first function as it is needed only once per
5100          * adapter.
5101          */
5102         if (pdev->devfn == 0)
5103                 ssleep(30);
5104
5105         return PCI_ERS_RESULT_NEED_RESET;
5106 }
5107
5108 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5109 {
5110         struct be_adapter *adapter = pci_get_drvdata(pdev);
5111         int status;
5112
5113         dev_info(&adapter->pdev->dev, "EEH reset\n");
5114
5115         status = pci_enable_device(pdev);
5116         if (status)
5117                 return PCI_ERS_RESULT_DISCONNECT;
5118
5119         pci_set_master(pdev);
5120         pci_set_power_state(pdev, PCI_D0);
5121         pci_restore_state(pdev);
5122
5123         /* Check if card is ok and fw is ready */
5124         dev_info(&adapter->pdev->dev,
5125                  "Waiting for FW to be ready after EEH reset\n");
5126         status = be_fw_wait_ready(adapter);
5127         if (status)
5128                 return PCI_ERS_RESULT_DISCONNECT;
5129
5130         pci_cleanup_aer_uncorrect_error_status(pdev);
5131         be_clear_all_error(adapter);
5132         return PCI_ERS_RESULT_RECOVERED;
5133 }
5134
5135 static void be_eeh_resume(struct pci_dev *pdev)
5136 {
5137         int status = 0;
5138         struct be_adapter *adapter = pci_get_drvdata(pdev);
5139         struct net_device *netdev =  adapter->netdev;
5140
5141         dev_info(&adapter->pdev->dev, "EEH resume\n");
5142
5143         pci_save_state(pdev);
5144
5145         status = be_cmd_reset_function(adapter);
5146         if (status)
5147                 goto err;
5148
5149         /* On some BE3 FW versions, after a HW reset,
5150          * interrupts will remain disabled for each function.
5151          * So, explicitly enable interrupts
5152          */
5153         be_intr_set(adapter, true);
5154
5155         /* tell fw we're ready to fire cmds */
5156         status = be_cmd_fw_init(adapter);
5157         if (status)
5158                 goto err;
5159
5160         status = be_setup(adapter);
5161         if (status)
5162                 goto err;
5163
5164         if (netif_running(netdev)) {
5165                 status = be_open(netdev);
5166                 if (status)
5167                         goto err;
5168         }
5169
5170         schedule_delayed_work(&adapter->func_recovery_work,
5171                               msecs_to_jiffies(1000));
5172         netif_device_attach(netdev);
5173         return;
5174 err:
5175         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5176 }
5177
5178 static const struct pci_error_handlers be_eeh_handlers = {
5179         .error_detected = be_eeh_err_detected,
5180         .slot_reset = be_eeh_reset,
5181         .resume = be_eeh_resume,
5182 };
5183
5184 static struct pci_driver be_driver = {
5185         .name = DRV_NAME,
5186         .id_table = be_dev_ids,
5187         .probe = be_probe,
5188         .remove = be_remove,
5189         .suspend = be_suspend,
5190         .resume = be_resume,
5191         .shutdown = be_shutdown,
5192         .err_handler = &be_eeh_handlers
5193 };
5194
5195 static int __init be_init_module(void)
5196 {
5197         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5198             rx_frag_size != 2048) {
5199                 printk(KERN_WARNING DRV_NAME
5200                         " : Module param rx_frag_size must be 2048/4096/8192."
5201                         " Using 2048\n");
5202                 rx_frag_size = 2048;
5203         }
5204
5205         return pci_register_driver(&be_driver);
5206 }
5207 module_init(be_init_module);
5208
5209 static void __exit be_exit_module(void)
5210 {
5211         pci_unregister_driver(&be_driver);
5212 }
5213 module_exit(be_exit_module);