drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/busy_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  57
  58         /* Copy the NAPI object as it has been already initialized */
  59         from_fp->napi = to_fp->napi;
  60
  61         /* Move bnx2x_fastpath contents */
  62         memcpy(to_fp, from_fp, sizeof(*to_fp));
  63         to_fp->index = to;
  64
  65         /* Retain the tpa_info of the original `to' version as we don't want
  66          * 2 FPs to contain the same tpa_info pointer.
  67          */
  68         to_fp->tpa_info = old_tpa_info;
  69
  70         /* move sp_objs contents as well, as their indices match fp ones */
  71         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  72
  73         /* move fp_stats contents as well, as their indices match fp ones */
  74         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  75
  76         /* Update txdata pointers in fp and move txdata content accordingly:
  77          * Each fp consumes 'max_cos' txdata structures, so the index should be
  78          * decremented by max_cos x delta.
  79          */
  80
  81         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  82         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  83                                 (bp)->max_cos;
  84         if (from == FCOE_IDX(bp)) {
  85                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  86                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  87         }
  88
  89         memcpy(&bp->bnx2x_txq[new_txdata_index],
  90                &bp->bnx2x_txq[old_txdata_index],
  91                sizeof(struct bnx2x_fp_txdata));
  92         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  93 }
  94
  95 /**
  96  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  97  *
  98  * @bp:        driver handle
  99  * @buf:       character buffer to fill with the fw name
 100  * @buf_len:   length of the above buffer
 101  *
 102  */
 103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 104 {
 105         if (IS_PF(bp)) {
 106                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 107
 108                 phy_fw_ver[0] = '\0';
 109                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 110                                              phy_fw_ver, PHY_FW_VER_LEN);
 111                 strlcpy(buf, bp->fw_ver, buf_len);
 112                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 113                          "bc %d.%d.%d%s%s",
 114                          (bp->common.bc_ver & 0xff0000) >> 16,
 115                          (bp->common.bc_ver & 0xff00) >> 8,
 116                          (bp->common.bc_ver & 0xff),
 117                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 118         } else {
 119                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 120         }
 121 }
 122
 123 /**
 124  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 125  *
 126  * @bp: driver handle
 127  * @delta:      number of eth queues which were not allocated
 128  */
 129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 130 {
 131         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 132
 133         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 134          * backward along the array could cause memory to be overridden
 135          */
 136         for (cos = 1; cos < bp->max_cos; cos++) {
 137                 for (i = 0; i < old_eth_num - delta; i++) {
 138                         struct bnx2x_fastpath *fp = &bp->fp[i];
 139                         int new_idx = cos * (old_eth_num - delta) + i;
 140
 141                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 142                                sizeof(struct bnx2x_fp_txdata));
 143                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 144                 }
 145         }
 146 }
 147
 148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 149
 150 /* free skb in the packet ring at pos idx
 151  * return idx of last bd freed
 152  */
 153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 154                              u16 idx, unsigned int *pkts_compl,
 155                              unsigned int *bytes_compl)
 156 {
 157         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 158         struct eth_tx_start_bd *tx_start_bd;
 159         struct eth_tx_bd *tx_data_bd;
 160         struct sk_buff *skb = tx_buf->skb;
 161         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 162         int nbd;
 163         u16 split_bd_len = 0;
 164
 165         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 166         prefetch(&skb->end);
 167
 168         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 169            txdata->txq_index, idx, tx_buf, skb);
 170
 171         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 172
 173         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 174 #ifdef BNX2X_STOP_ON_ERROR
 175         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 176                 BNX2X_ERR("BAD nbd!\n");
 177                 bnx2x_panic();
 178         }
 179 #endif
 180         new_cons = nbd + tx_buf->first_bd;
 181
 182         /* Get the next bd */
 183         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 184
 185         /* Skip a parse bd... */
 186         --nbd;
 187         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 188
 189         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 190         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 191                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 192                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 193                 --nbd;
 194                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 195         }
 196
 197         /* unmap first bd */
 198         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 199                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 200                          DMA_TO_DEVICE);
 201
 202         /* now free frags */
 203         while (nbd > 0) {
 204
 205                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 206                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 207                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 208                 if (--nbd)
 209                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 210         }
 211
 212         /* release skb */
 213         WARN_ON(!skb);
 214         if (likely(skb)) {
 215                 (*pkts_compl)++;
 216                 (*bytes_compl) += skb->len;
 217         }
 218
 219         dev_kfree_skb_any(skb);
 220         tx_buf->first_bd = 0;
 221         tx_buf->skb = NULL;
 222
 223         return new_cons;
 224 }
 225
 226 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 227 {
 228         struct netdev_queue *txq;
 229         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 230         unsigned int pkts_compl = 0, bytes_compl = 0;
 231
 232 #ifdef BNX2X_STOP_ON_ERROR
 233         if (unlikely(bp->panic))
 234                 return -1;
 235 #endif
 236
 237         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 238         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 239         sw_cons = txdata->tx_pkt_cons;
 240
 241         while (sw_cons != hw_cons) {
 242                 u16 pkt_cons;
 243
 244                 pkt_cons = TX_BD(sw_cons);
 245
 246                 DP(NETIF_MSG_TX_DONE,
 247                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 248                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 249
 250                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 251                                             &pkts_compl, &bytes_compl);
 252
 253                 sw_cons++;
 254         }
 255
 256         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 257
 258         txdata->tx_pkt_cons = sw_cons;
 259         txdata->tx_bd_cons = bd_cons;
 260
 261         /* Need to make the tx_bd_cons update visible to start_xmit()
 262          * before checking for netif_tx_queue_stopped().  Without the
 263          * memory barrier, there is a small possibility that
 264          * start_xmit() will miss it and cause the queue to be stopped
 265          * forever.
 266          * On the other hand we need an rmb() here to ensure the proper
 267          * ordering of bit testing in the following
 268          * netif_tx_queue_stopped(txq) call.
 269          */
 270         smp_mb();
 271
 272         if (unlikely(netif_tx_queue_stopped(txq))) {
 273                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 274                  * while it's empty. This could have happen if rx_action() gets
 275                  * suspended in bnx2x_tx_int() after the condition before
 276                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 277                  *
 278                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 279                  * sends some packets consuming the whole queue again->
 280                  * stops the queue
 281                  */
 282
 283                 __netif_tx_lock(txq, smp_processor_id());
 284
 285                 if ((netif_tx_queue_stopped(txq)) &&
 286                     (bp->state == BNX2X_STATE_OPEN) &&
 287                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 288                         netif_tx_wake_queue(txq);
 289
 290                 __netif_tx_unlock(txq);
 291         }
 292         return 0;
 293 }
 294
 295 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 296                                              u16 idx)
 297 {
 298         u16 last_max = fp->last_max_sge;
 299
 300         if (SUB_S16(idx, last_max) > 0)
 301                 fp->last_max_sge = idx;
 302 }
 303
 304 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 305                                          u16 sge_len,
 306                                          struct eth_end_agg_rx_cqe *cqe)
 307 {
 308         struct bnx2x *bp = fp->bp;
 309         u16 last_max, last_elem, first_elem;
 310         u16 delta = 0;
 311         u16 i;
 312
 313         if (!sge_len)
 314                 return;
 315
 316         /* First mark all used pages */
 317         for (i = 0; i < sge_len; i++)
 318                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 319                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 320
 321         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 322            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 323
 324         /* Here we assume that the last SGE index is the biggest */
 325         prefetch((void *)(fp->sge_mask));
 326         bnx2x_update_last_max_sge(fp,
 327                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 328
 329         last_max = RX_SGE(fp->last_max_sge);
 330         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 331         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 332
 333         /* If ring is not full */
 334         if (last_elem + 1 != first_elem)
 335                 last_elem++;
 336
 337         /* Now update the prod */
 338         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 339                 if (likely(fp->sge_mask[i]))
 340                         break;
 341
 342                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 343                 delta += BIT_VEC64_ELEM_SZ;
 344         }
 345
 346         if (delta > 0) {
 347                 fp->rx_sge_prod += delta;
 348                 /* clear page-end entries */
 349                 bnx2x_clear_sge_mask_next_elems(fp);
 350         }
 351
 352         DP(NETIF_MSG_RX_STATUS,
 353            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 354            fp->last_max_sge, fp->rx_sge_prod);
 355 }
 356
 357 /* Get Toeplitz hash value in the skb using the value from the
 358  * CQE (calculated by HW).
 359  */
 360 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 361                             const struct eth_fast_path_rx_cqe *cqe,
 362                             bool *l4_rxhash)
 363 {
 364         /* Get Toeplitz hash from CQE */
 365         if ((bp->dev->features & NETIF_F_RXHASH) &&
 366             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 367                 enum eth_rss_hash_type htype;
 368
 369                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 370                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 371                              (htype == TCP_IPV6_HASH_TYPE);
 372                 return le32_to_cpu(cqe->rss_hash_result);
 373         }
 374         *l4_rxhash = false;
 375         return 0;
 376 }
 377
 378 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 379                             u16 cons, u16 prod,
 380                             struct eth_fast_path_rx_cqe *cqe)
 381 {
 382         struct bnx2x *bp = fp->bp;
 383         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 384         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 385         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 386         dma_addr_t mapping;
 387         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 388         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 389
 390         /* print error if current state != stop */
 391         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 392                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 393
 394         /* Try to map an empty data buffer from the aggregation info  */
 395         mapping = dma_map_single(&bp->pdev->dev,
 396                                  first_buf->data + NET_SKB_PAD,
 397                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 398         /*
 399          *  ...if it fails - move the skb from the consumer to the producer
 400          *  and set the current aggregation state as ERROR to drop it
 401          *  when TPA_STOP arrives.
 402          */
 403
 404         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 405                 /* Move the BD from the consumer to the producer */
 406                 bnx2x_reuse_rx_data(fp, cons, prod);
 407                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 408                 return;
 409         }
 410
 411         /* move empty data from pool to prod */
 412         prod_rx_buf->data = first_buf->data;
 413         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 414         /* point prod_bd to new data */
 415         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 416         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 417
 418         /* move partial skb from cons to pool (don't unmap yet) */
 419         *first_buf = *cons_rx_buf;
 420
 421         /* mark bin state as START */
 422         tpa_info->parsing_flags =
 423                 le16_to_cpu(cqe->pars_flags.flags);
 424         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 425         tpa_info->tpa_state = BNX2X_TPA_START;
 426         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 427         tpa_info->placement_offset = cqe->placement_offset;
 428         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 429         if (fp->mode == TPA_MODE_GRO) {
 430                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 431                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 432                 tpa_info->gro_size = gro_size;
 433         }
 434
 435 #ifdef BNX2X_STOP_ON_ERROR
 436         fp->tpa_queue_used |= (1 << queue);
 437 #ifdef _ASM_GENERIC_INT_L64_H
 438         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 439 #else
 440         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 441 #endif
 442            fp->tpa_queue_used);
 443 #endif
 444 }
 445
 446 /* Timestamp option length allowed for TPA aggregation:
 447  *
 448  *              nop nop kind length echo val
 449  */
 450 #define TPA_TSTAMP_OPT_LEN      12
 451 /**
 452  * bnx2x_set_gro_params - compute GRO values
 453  *
 454  * @skb:                packet skb
 455  * @parsing_flags:      parsing flags from the START CQE
 456  * @len_on_bd:          total length of the first packet for the
 457  *                      aggregation.
 458  * @pkt_len:            length of all segments
 459  *
 460  * Approximate value of the MSS for this aggregation calculated using
 461  * the first packet of it.
 462  * Compute number of aggregated segments, and gso_type.
 463  */
 464 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 465                                  u16 len_on_bd, unsigned int pkt_len,
 466                                  u16 num_of_coalesced_segs)
 467 {
 468         /* TPA aggregation won't have either IP options or TCP options
 469          * other than timestamp or IPv6 extension headers.
 470          */
 471         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 472
 473         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 474             PRS_FLAG_OVERETH_IPV6) {
 475                 hdrs_len += sizeof(struct ipv6hdr);
 476                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 477         } else {
 478                 hdrs_len += sizeof(struct iphdr);
 479                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 480         }
 481
 482         /* Check if there was a TCP timestamp, if there is it's will
 483          * always be 12 bytes length: nop nop kind length echo val.
 484          *
 485          * Otherwise FW would close the aggregation.
 486          */
 487         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 488                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 489
 490         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 491
 492         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 493          * to skb_shinfo(skb)->gso_segs
 494          */
 495         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 496 }
 497
 498 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 499                               u16 index, gfp_t gfp_mask)
 500 {
 501         struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 502         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 503         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 504         dma_addr_t mapping;
 505
 506         if (unlikely(page == NULL)) {
 507                 BNX2X_ERR("Can't alloc sge\n");
 508                 return -ENOMEM;
 509         }
 510
 511         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 512                                SGE_PAGES, DMA_FROM_DEVICE);
 513         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 514                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 515                 BNX2X_ERR("Can't map sge\n");
 516                 return -ENOMEM;
 517         }
 518
 519         sw_buf->page = page;
 520         dma_unmap_addr_set(sw_buf, mapping, mapping);
 521
 522         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 523         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 524
 525         return 0;
 526 }
 527
 528 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 529                                struct bnx2x_agg_info *tpa_info,
 530                                u16 pages,
 531                                struct sk_buff *skb,
 532                                struct eth_end_agg_rx_cqe *cqe,
 533                                u16 cqe_idx)
 534 {
 535         struct sw_rx_page *rx_pg, old_rx_pg;
 536         u32 i, frag_len, frag_size;
 537         int err, j, frag_id = 0;
 538         u16 len_on_bd = tpa_info->len_on_bd;
 539         u16 full_page = 0, gro_size = 0;
 540
 541         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 542
 543         if (fp->mode == TPA_MODE_GRO) {
 544                 gro_size = tpa_info->gro_size;
 545                 full_page = tpa_info->full_page;
 546         }
 547
 548         /* This is needed in order to enable forwarding support */
 549         if (frag_size)
 550                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 551                                      le16_to_cpu(cqe->pkt_len),
 552                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 553
 554 #ifdef BNX2X_STOP_ON_ERROR
 555         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 556                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 557                           pages, cqe_idx);
 558                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 559                 bnx2x_panic();
 560                 return -EINVAL;
 561         }
 562 #endif
 563
 564         /* Run through the SGL and compose the fragmented skb */
 565         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 566                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 567
 568                 /* FW gives the indices of the SGE as if the ring is an array
 569                    (meaning that "next" element will consume 2 indices) */
 570                 if (fp->mode == TPA_MODE_GRO)
 571                         frag_len = min_t(u32, frag_size, (u32)full_page);
 572                 else /* LRO */
 573                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 574
 575                 rx_pg = &fp->rx_page_ring[sge_idx];
 576                 old_rx_pg = *rx_pg;
 577
 578                 /* If we fail to allocate a substitute page, we simply stop
 579                    where we are and drop the whole packet */
 580                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 581                 if (unlikely(err)) {
 582                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 583                         return err;
 584                 }
 585
 586                 /* Unmap the page as we're going to pass it to the stack */
 587                 dma_unmap_page(&bp->pdev->dev,
 588                                dma_unmap_addr(&old_rx_pg, mapping),
 589                                SGE_PAGES, DMA_FROM_DEVICE);
 590                 /* Add one frag and update the appropriate fields in the skb */
 591                 if (fp->mode == TPA_MODE_LRO)
 592                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 593                 else { /* GRO */
 594                         int rem;
 595                         int offset = 0;
 596                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 597                                 int len = rem > gro_size ? gro_size : rem;
 598                                 skb_fill_page_desc(skb, frag_id++,
 599                                                    old_rx_pg.page, offset, len);
 600                                 if (offset)
 601                                         get_page(old_rx_pg.page);
 602                                 offset += len;
 603                         }
 604                 }
 605
 606                 skb->data_len += frag_len;
 607                 skb->truesize += SGE_PAGES;
 608                 skb->len += frag_len;
 609
 610                 frag_size -= frag_len;
 611         }
 612
 613         return 0;
 614 }
 615
 616 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 617 {
 618         if (fp->rx_frag_size)
 619                 put_page(virt_to_head_page(data));
 620         else
 621                 kfree(data);
 622 }
 623
 624 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 625 {
 626         if (fp->rx_frag_size) {
 627                 /* GFP_KERNEL allocations are used only during initialization */
 628                 if (unlikely(gfp_mask & __GFP_WAIT))
 629                         return (void *)__get_free_page(gfp_mask);
 630
 631                 return netdev_alloc_frag(fp->rx_frag_size);
 632         }
 633
 634         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 635 }
 636
 637 #ifdef CONFIG_INET
 638 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 639 {
 640         const struct iphdr *iph = ip_hdr(skb);
 641         struct tcphdr *th;
 642
 643         skb_set_transport_header(skb, sizeof(struct iphdr));
 644         th = tcp_hdr(skb);
 645
 646         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 647                                   iph->saddr, iph->daddr, 0);
 648 }
 649
 650 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 651 {
 652         struct ipv6hdr *iph = ipv6_hdr(skb);
 653         struct tcphdr *th;
 654
 655         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 656         th = tcp_hdr(skb);
 657
 658         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 659                                   &iph->saddr, &iph->daddr, 0);
 660 }
 661
 662 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 663                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 664 {
 665         skb_set_network_header(skb, 0);
 666         gro_func(bp, skb);
 667         tcp_gro_complete(skb);
 668 }
 669 #endif
 670
 671 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 672                                struct sk_buff *skb)
 673 {
 674 #ifdef CONFIG_INET
 675         if (skb_shinfo(skb)->gso_size) {
 676                 switch (be16_to_cpu(skb->protocol)) {
 677                 case ETH_P_IP:
 678                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 679                         break;
 680                 case ETH_P_IPV6:
 681                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 682                         break;
 683                 default:
 684                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 685                                   be16_to_cpu(skb->protocol));
 686                 }
 687         }
 688 #endif
 689         skb_record_rx_queue(skb, fp->rx_queue);
 690         napi_gro_receive(&fp->napi, skb);
 691 }
 692
 693 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 694                            struct bnx2x_agg_info *tpa_info,
 695                            u16 pages,
 696                            struct eth_end_agg_rx_cqe *cqe,
 697                            u16 cqe_idx)
 698 {
 699         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 700         u8 pad = tpa_info->placement_offset;
 701         u16 len = tpa_info->len_on_bd;
 702         struct sk_buff *skb = NULL;
 703         u8 *new_data, *data = rx_buf->data;
 704         u8 old_tpa_state = tpa_info->tpa_state;
 705
 706         tpa_info->tpa_state = BNX2X_TPA_STOP;
 707
 708         /* If we there was an error during the handling of the TPA_START -
 709          * drop this aggregation.
 710          */
 711         if (old_tpa_state == BNX2X_TPA_ERROR)
 712                 goto drop;
 713
 714         /* Try to allocate the new data */
 715         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 716         /* Unmap skb in the pool anyway, as we are going to change
 717            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 718            fails. */
 719         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 720                          fp->rx_buf_size, DMA_FROM_DEVICE);
 721         if (likely(new_data))
 722                 skb = build_skb(data, fp->rx_frag_size);
 723
 724         if (likely(skb)) {
 725 #ifdef BNX2X_STOP_ON_ERROR
 726                 if (pad + len > fp->rx_buf_size) {
 727                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 728                                   pad, len, fp->rx_buf_size);
 729                         bnx2x_panic();
 730                         return;
 731                 }
 732 #endif
 733
 734                 skb_reserve(skb, pad + NET_SKB_PAD);
 735                 skb_put(skb, len);
 736                 skb->rxhash = tpa_info->rxhash;
 737                 skb->l4_rxhash = tpa_info->l4_rxhash;
 738
 739                 skb->protocol = eth_type_trans(skb, bp->dev);
 740                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 741
 742                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 743                                          skb, cqe, cqe_idx)) {
 744                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 745                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 746                         bnx2x_gro_receive(bp, fp, skb);
 747                 } else {
 748                         DP(NETIF_MSG_RX_STATUS,
 749                            "Failed to allocate new pages - dropping packet!\n");
 750                         dev_kfree_skb_any(skb);
 751                 }
 752
 753                 /* put new data in bin */
 754                 rx_buf->data = new_data;
 755
 756                 return;
 757         }
 758         bnx2x_frag_free(fp, new_data);
 759 drop:
 760         /* drop the packet and keep the buffer in the bin */
 761         DP(NETIF_MSG_RX_STATUS,
 762            "Failed to allocate or map a new skb - dropping packet!\n");
 763         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 764 }
 765
 766 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 767                                u16 index, gfp_t gfp_mask)
 768 {
 769         u8 *data;
 770         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 771         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 772         dma_addr_t mapping;
 773
 774         data = bnx2x_frag_alloc(fp, gfp_mask);
 775         if (unlikely(data == NULL))
 776                 return -ENOMEM;
 777
 778         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 779                                  fp->rx_buf_size,
 780                                  DMA_FROM_DEVICE);
 781         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 782                 bnx2x_frag_free(fp, data);
 783                 BNX2X_ERR("Can't map rx data\n");
 784                 return -ENOMEM;
 785         }
 786
 787         rx_buf->data = data;
 788         dma_unmap_addr_set(rx_buf, mapping, mapping);
 789
 790         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 791         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 792
 793         return 0;
 794 }
 795
 796 static
 797 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 798                                  struct bnx2x_fastpath *fp,
 799                                  struct bnx2x_eth_q_stats *qstats)
 800 {
 801         /* Do nothing if no L4 csum validation was done.
 802          * We do not check whether IP csum was validated. For IPv4 we assume
 803          * that if the card got as far as validating the L4 csum, it also
 804          * validated the IP csum. IPv6 has no IP csum.
 805          */
 806         if (cqe->fast_path_cqe.status_flags &
 807             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 808                 return;
 809
 810         /* If L4 validation was done, check if an error was found. */
 811
 812         if (cqe->fast_path_cqe.type_error_flags &
 813             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 814              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 815                 qstats->hw_csum_err++;
 816         else
 817                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 818 }
 819
 820 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 821 {
 822         struct bnx2x *bp = fp->bp;
 823         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 824         u16 sw_comp_cons, sw_comp_prod;
 825         int rx_pkt = 0;
 826         union eth_rx_cqe *cqe;
 827         struct eth_fast_path_rx_cqe *cqe_fp;
 828
 829 #ifdef BNX2X_STOP_ON_ERROR
 830         if (unlikely(bp->panic))
 831                 return 0;
 832 #endif
 833
 834         bd_cons = fp->rx_bd_cons;
 835         bd_prod = fp->rx_bd_prod;
 836         bd_prod_fw = bd_prod;
 837         sw_comp_cons = fp->rx_comp_cons;
 838         sw_comp_prod = fp->rx_comp_prod;
 839
 840         comp_ring_cons = RCQ_BD(sw_comp_cons);
 841         cqe = &fp->rx_comp_ring[comp_ring_cons];
 842         cqe_fp = &cqe->fast_path_cqe;
 843
 844         DP(NETIF_MSG_RX_STATUS,
 845            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 846
 847         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 848                 struct sw_rx_bd *rx_buf = NULL;
 849                 struct sk_buff *skb;
 850                 u8 cqe_fp_flags;
 851                 enum eth_rx_cqe_type cqe_fp_type;
 852                 u16 len, pad, queue;
 853                 u8 *data;
 854                 bool l4_rxhash;
 855
 856 #ifdef BNX2X_STOP_ON_ERROR
 857                 if (unlikely(bp->panic))
 858                         return 0;
 859 #endif
 860
 861                 bd_prod = RX_BD(bd_prod);
 862                 bd_cons = RX_BD(bd_cons);
 863
 864                 cqe_fp_flags = cqe_fp->type_error_flags;
 865                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 866
 867                 DP(NETIF_MSG_RX_STATUS,
 868                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 869                    CQE_TYPE(cqe_fp_flags),
 870                    cqe_fp_flags, cqe_fp->status_flags,
 871                    le32_to_cpu(cqe_fp->rss_hash_result),
 872                    le16_to_cpu(cqe_fp->vlan_tag),
 873                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 874
 875                 /* is this a slowpath msg? */
 876                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 877                         bnx2x_sp_event(fp, cqe);
 878                         goto next_cqe;
 879                 }
 880
 881                 rx_buf = &fp->rx_buf_ring[bd_cons];
 882                 data = rx_buf->data;
 883
 884                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 885                         struct bnx2x_agg_info *tpa_info;
 886                         u16 frag_size, pages;
 887 #ifdef BNX2X_STOP_ON_ERROR
 888                         /* sanity check */
 889                         if (fp->disable_tpa &&
 890                             (CQE_TYPE_START(cqe_fp_type) ||
 891                              CQE_TYPE_STOP(cqe_fp_type)))
 892                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 893                                           CQE_TYPE(cqe_fp_type));
 894 #endif
 895
 896                         if (CQE_TYPE_START(cqe_fp_type)) {
 897                                 u16 queue = cqe_fp->queue_index;
 898                                 DP(NETIF_MSG_RX_STATUS,
 899                                    "calling tpa_start on queue %d\n",
 900                                    queue);
 901
 902                                 bnx2x_tpa_start(fp, queue,
 903                                                 bd_cons, bd_prod,
 904                                                 cqe_fp);
 905
 906                                 goto next_rx;
 907                         }
 908                         queue = cqe->end_agg_cqe.queue_index;
 909                         tpa_info = &fp->tpa_info[queue];
 910                         DP(NETIF_MSG_RX_STATUS,
 911                            "calling tpa_stop on queue %d\n",
 912                            queue);
 913
 914                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 915                                     tpa_info->len_on_bd;
 916
 917                         if (fp->mode == TPA_MODE_GRO)
 918                                 pages = (frag_size + tpa_info->full_page - 1) /
 919                                          tpa_info->full_page;
 920                         else
 921                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 922                                         SGE_PAGE_SHIFT;
 923
 924                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 925                                        &cqe->end_agg_cqe, comp_ring_cons);
 926 #ifdef BNX2X_STOP_ON_ERROR
 927                         if (bp->panic)
 928                                 return 0;
 929 #endif
 930
 931                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 932                         goto next_cqe;
 933                 }
 934                 /* non TPA */
 935                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 936                 pad = cqe_fp->placement_offset;
 937                 dma_sync_single_for_cpu(&bp->pdev->dev,
 938                                         dma_unmap_addr(rx_buf, mapping),
 939                                         pad + RX_COPY_THRESH,
 940                                         DMA_FROM_DEVICE);
 941                 pad += NET_SKB_PAD;
 942                 prefetch(data + pad); /* speedup eth_type_trans() */
 943                 /* is this an error packet? */
 944                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 945                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 946                            "ERROR  flags %x  rx packet %u\n",
 947                            cqe_fp_flags, sw_comp_cons);
 948                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 949                         goto reuse_rx;
 950                 }
 951
 952                 /* Since we don't have a jumbo ring
 953                  * copy small packets if mtu > 1500
 954                  */
 955                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 956                     (len <= RX_COPY_THRESH)) {
 957                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 958                         if (skb == NULL) {
 959                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 960                                    "ERROR  packet dropped because of alloc failure\n");
 961                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 962                                 goto reuse_rx;
 963                         }
 964                         memcpy(skb->data, data + pad, len);
 965                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 966                 } else {
 967                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
 968                                                        GFP_ATOMIC) == 0)) {
 969                                 dma_unmap_single(&bp->pdev->dev,
 970                                                  dma_unmap_addr(rx_buf, mapping),
 971                                                  fp->rx_buf_size,
 972                                                  DMA_FROM_DEVICE);
 973                                 skb = build_skb(data, fp->rx_frag_size);
 974                                 if (unlikely(!skb)) {
 975                                         bnx2x_frag_free(fp, data);
 976                                         bnx2x_fp_qstats(bp, fp)->
 977                                                         rx_skb_alloc_failed++;
 978                                         goto next_rx;
 979                                 }
 980                                 skb_reserve(skb, pad);
 981                         } else {
 982                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 983                                    "ERROR  packet dropped because of alloc failure\n");
 984                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 985 reuse_rx:
 986                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 987                                 goto next_rx;
 988                         }
 989                 }
 990
 991                 skb_put(skb, len);
 992                 skb->protocol = eth_type_trans(skb, bp->dev);
 993
 994                 /* Set Toeplitz hash for a none-LRO skb */
 995                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 996                 skb->l4_rxhash = l4_rxhash;
 997
 998                 skb_checksum_none_assert(skb);
 999
1000                 if (bp->dev->features & NETIF_F_RXCSUM)
1001                         bnx2x_csum_validate(skb, cqe, fp,
1002                                             bnx2x_fp_qstats(bp, fp));
1003
1004                 skb_record_rx_queue(skb, fp->rx_queue);
1005
1006                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1007                     PARSING_FLAGS_VLAN)
1008                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1009                                                le16_to_cpu(cqe_fp->vlan_tag));
1010
1011                 skb_mark_napi_id(skb, &fp->napi);
1012
1013                 if (bnx2x_fp_ll_polling(fp))
1014                         netif_receive_skb(skb);
1015                 else
1016                         napi_gro_receive(&fp->napi, skb);
1017 next_rx:
1018                 rx_buf->data = NULL;
1019
1020                 bd_cons = NEXT_RX_IDX(bd_cons);
1021                 bd_prod = NEXT_RX_IDX(bd_prod);
1022                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1023                 rx_pkt++;
1024 next_cqe:
1025                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1026                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1027
1028                 /* mark CQE as free */
1029                 BNX2X_SEED_CQE(cqe_fp);
1030
1031                 if (rx_pkt == budget)
1032                         break;
1033
1034                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1035                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1036                 cqe_fp = &cqe->fast_path_cqe;
1037         } /* while */
1038
1039         fp->rx_bd_cons = bd_cons;
1040         fp->rx_bd_prod = bd_prod_fw;
1041         fp->rx_comp_cons = sw_comp_cons;
1042         fp->rx_comp_prod = sw_comp_prod;
1043
1044         /* Update producers */
1045         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1046                              fp->rx_sge_prod);
1047
1048         fp->rx_pkt += rx_pkt;
1049         fp->rx_calls++;
1050
1051         return rx_pkt;
1052 }
1053
1054 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1055 {
1056         struct bnx2x_fastpath *fp = fp_cookie;
1057         struct bnx2x *bp = fp->bp;
1058         u8 cos;
1059
1060         DP(NETIF_MSG_INTR,
1061            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1062            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1063
1064         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1065
1066 #ifdef BNX2X_STOP_ON_ERROR
1067         if (unlikely(bp->panic))
1068                 return IRQ_HANDLED;
1069 #endif
1070
1071         /* Handle Rx and Tx according to MSI-X vector */
1072         for_each_cos_in_tx_queue(fp, cos)
1073                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1074
1075         prefetch(&fp->sb_running_index[SM_RX_ID]);
1076         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1077
1078         return IRQ_HANDLED;
1079 }
1080
1081 /* HW Lock for shared dual port PHYs */
1082 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1083 {
1084         mutex_lock(&bp->port.phy_mutex);
1085
1086         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1087 }
1088
1089 void bnx2x_release_phy_lock(struct bnx2x *bp)
1090 {
1091         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1092
1093         mutex_unlock(&bp->port.phy_mutex);
1094 }
1095
1096 /* calculates MF speed according to current linespeed and MF configuration */
1097 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1098 {
1099         u16 line_speed = bp->link_vars.line_speed;
1100         if (IS_MF(bp)) {
1101                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1102                                                    bp->mf_config[BP_VN(bp)]);
1103
1104                 /* Calculate the current MAX line speed limit for the MF
1105                  * devices
1106                  */
1107                 if (IS_MF_SI(bp))
1108                         line_speed = (line_speed * maxCfg) / 100;
1109                 else { /* SD mode */
1110                         u16 vn_max_rate = maxCfg * 100;
1111
1112                         if (vn_max_rate < line_speed)
1113                                 line_speed = vn_max_rate;
1114                 }
1115         }
1116
1117         return line_speed;
1118 }
1119
1120 /**
1121  * bnx2x_fill_report_data - fill link report data to report
1122  *
1123  * @bp:         driver handle
1124  * @data:       link state to update
1125  *
1126  * It uses a none-atomic bit operations because is called under the mutex.
1127  */
1128 static void bnx2x_fill_report_data(struct bnx2x *bp,
1129                                    struct bnx2x_link_report_data *data)
1130 {
1131         u16 line_speed = bnx2x_get_mf_speed(bp);
1132
1133         memset(data, 0, sizeof(*data));
1134
1135         /* Fill the report data: effective line speed */
1136         data->line_speed = line_speed;
1137
1138         /* Link is down */
1139         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1140                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1141                           &data->link_report_flags);
1142
1143         /* Full DUPLEX */
1144         if (bp->link_vars.duplex == DUPLEX_FULL)
1145                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1146
1147         /* Rx Flow Control is ON */
1148         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1149                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1150
1151         /* Tx Flow Control is ON */
1152         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1153                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1154 }
1155
1156 /**
1157  * bnx2x_link_report - report link status to OS.
1158  *
1159  * @bp:         driver handle
1160  *
1161  * Calls the __bnx2x_link_report() under the same locking scheme
1162  * as a link/PHY state managing code to ensure a consistent link
1163  * reporting.
1164  */
1165
1166 void bnx2x_link_report(struct bnx2x *bp)
1167 {
1168         bnx2x_acquire_phy_lock(bp);
1169         __bnx2x_link_report(bp);
1170         bnx2x_release_phy_lock(bp);
1171 }
1172
1173 /**
1174  * __bnx2x_link_report - report link status to OS.
1175  *
1176  * @bp:         driver handle
1177  *
1178  * None atomic implementation.
1179  * Should be called under the phy_lock.
1180  */
1181 void __bnx2x_link_report(struct bnx2x *bp)
1182 {
1183         struct bnx2x_link_report_data cur_data;
1184
1185         /* reread mf_cfg */
1186         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1187                 bnx2x_read_mf_cfg(bp);
1188
1189         /* Read the current link report info */
1190         bnx2x_fill_report_data(bp, &cur_data);
1191
1192         /* Don't report link down or exactly the same link status twice */
1193         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1194             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1195                       &bp->last_reported_link.link_report_flags) &&
1196              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1197                       &cur_data.link_report_flags)))
1198                 return;
1199
1200         bp->link_cnt++;
1201
1202         /* We are going to report a new link parameters now -
1203          * remember the current data for the next time.
1204          */
1205         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1206
1207         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1208                      &cur_data.link_report_flags)) {
1209                 netif_carrier_off(bp->dev);
1210                 netdev_err(bp->dev, "NIC Link is Down\n");
1211                 return;
1212         } else {
1213                 const char *duplex;
1214                 const char *flow;
1215
1216                 netif_carrier_on(bp->dev);
1217
1218                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1219                                        &cur_data.link_report_flags))
1220                         duplex = "full";
1221                 else
1222                         duplex = "half";
1223
1224                 /* Handle the FC at the end so that only these flags would be
1225                  * possibly set. This way we may easily check if there is no FC
1226                  * enabled.
1227                  */
1228                 if (cur_data.link_report_flags) {
1229                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1230                                      &cur_data.link_report_flags)) {
1231                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1232                                      &cur_data.link_report_flags))
1233                                         flow = "ON - receive & transmit";
1234                                 else
1235                                         flow = "ON - receive";
1236                         } else {
1237                                 flow = "ON - transmit";
1238                         }
1239                 } else {
1240                         flow = "none";
1241                 }
1242                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1243                             cur_data.line_speed, duplex, flow);
1244         }
1245 }
1246
1247 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1248 {
1249         int i;
1250
1251         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1252                 struct eth_rx_sge *sge;
1253
1254                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1255                 sge->addr_hi =
1256                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1257                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1258
1259                 sge->addr_lo =
1260                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1261                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1262         }
1263 }
1264
1265 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1266                                 struct bnx2x_fastpath *fp, int last)
1267 {
1268         int i;
1269
1270         for (i = 0; i < last; i++) {
1271                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1272                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1273                 u8 *data = first_buf->data;
1274
1275                 if (data == NULL) {
1276                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1277                         continue;
1278                 }
1279                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1280                         dma_unmap_single(&bp->pdev->dev,
1281                                          dma_unmap_addr(first_buf, mapping),
1282                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1283                 bnx2x_frag_free(fp, data);
1284                 first_buf->data = NULL;
1285         }
1286 }
1287
1288 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1289 {
1290         int j;
1291
1292         for_each_rx_queue_cnic(bp, j) {
1293                 struct bnx2x_fastpath *fp = &bp->fp[j];
1294
1295                 fp->rx_bd_cons = 0;
1296
1297                 /* Activate BD ring */
1298                 /* Warning!
1299                  * this will generate an interrupt (to the TSTORM)
1300                  * must only be done after chip is initialized
1301                  */
1302                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1303                                      fp->rx_sge_prod);
1304         }
1305 }
1306
1307 void bnx2x_init_rx_rings(struct bnx2x *bp)
1308 {
1309         int func = BP_FUNC(bp);
1310         u16 ring_prod;
1311         int i, j;
1312
1313         /* Allocate TPA resources */
1314         for_each_eth_queue(bp, j) {
1315                 struct bnx2x_fastpath *fp = &bp->fp[j];
1316
1317                 DP(NETIF_MSG_IFUP,
1318                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1319
1320                 if (!fp->disable_tpa) {
1321                         /* Fill the per-aggregation pool */
1322                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1323                                 struct bnx2x_agg_info *tpa_info =
1324                                         &fp->tpa_info[i];
1325                                 struct sw_rx_bd *first_buf =
1326                                         &tpa_info->first_buf;
1327
1328                                 first_buf->data =
1329                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1330                                 if (!first_buf->data) {
1331                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1332                                                   j);
1333                                         bnx2x_free_tpa_pool(bp, fp, i);
1334                                         fp->disable_tpa = 1;
1335                                         break;
1336                                 }
1337                                 dma_unmap_addr_set(first_buf, mapping, 0);
1338                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1339                         }
1340
1341                         /* "next page" elements initialization */
1342                         bnx2x_set_next_page_sgl(fp);
1343
1344                         /* set SGEs bit mask */
1345                         bnx2x_init_sge_ring_bit_mask(fp);
1346
1347                         /* Allocate SGEs and initialize the ring elements */
1348                         for (i = 0, ring_prod = 0;
1349                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1350
1351                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1352                                                        GFP_KERNEL) < 0) {
1353                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1354                                                   i);
1355                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1356                                                   j);
1357                                         /* Cleanup already allocated elements */
1358                                         bnx2x_free_rx_sge_range(bp, fp,
1359                                                                 ring_prod);
1360                                         bnx2x_free_tpa_pool(bp, fp,
1361                                                             MAX_AGG_QS(bp));
1362                                         fp->disable_tpa = 1;
1363                                         ring_prod = 0;
1364                                         break;
1365                                 }
1366                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1367                         }
1368
1369                         fp->rx_sge_prod = ring_prod;
1370                 }
1371         }
1372
1373         for_each_eth_queue(bp, j) {
1374                 struct bnx2x_fastpath *fp = &bp->fp[j];
1375
1376                 fp->rx_bd_cons = 0;
1377
1378                 /* Activate BD ring */
1379                 /* Warning!
1380                  * this will generate an interrupt (to the TSTORM)
1381                  * must only be done after chip is initialized
1382                  */
1383                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1384                                      fp->rx_sge_prod);
1385
1386                 if (j != 0)
1387                         continue;
1388
1389                 if (CHIP_IS_E1(bp)) {
1390                         REG_WR(bp, BAR_USTRORM_INTMEM +
1391                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1392                                U64_LO(fp->rx_comp_mapping));
1393                         REG_WR(bp, BAR_USTRORM_INTMEM +
1394                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1395                                U64_HI(fp->rx_comp_mapping));
1396                 }
1397         }
1398 }
1399
1400 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1401 {
1402         u8 cos;
1403         struct bnx2x *bp = fp->bp;
1404
1405         for_each_cos_in_tx_queue(fp, cos) {
1406                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1407                 unsigned pkts_compl = 0, bytes_compl = 0;
1408
1409                 u16 sw_prod = txdata->tx_pkt_prod;
1410                 u16 sw_cons = txdata->tx_pkt_cons;
1411
1412                 while (sw_cons != sw_prod) {
1413                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1414                                           &pkts_compl, &bytes_compl);
1415                         sw_cons++;
1416                 }
1417
1418                 netdev_tx_reset_queue(
1419                         netdev_get_tx_queue(bp->dev,
1420                                             txdata->txq_index));
1421         }
1422 }
1423
1424 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1425 {
1426         int i;
1427
1428         for_each_tx_queue_cnic(bp, i) {
1429                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1430         }
1431 }
1432
1433 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1434 {
1435         int i;
1436
1437         for_each_eth_queue(bp, i) {
1438                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1439         }
1440 }
1441
1442 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1443 {
1444         struct bnx2x *bp = fp->bp;
1445         int i;
1446
1447         /* ring wasn't allocated */
1448         if (fp->rx_buf_ring == NULL)
1449                 return;
1450
1451         for (i = 0; i < NUM_RX_BD; i++) {
1452                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1453                 u8 *data = rx_buf->data;
1454
1455                 if (data == NULL)
1456                         continue;
1457                 dma_unmap_single(&bp->pdev->dev,
1458                                  dma_unmap_addr(rx_buf, mapping),
1459                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1460
1461                 rx_buf->data = NULL;
1462                 bnx2x_frag_free(fp, data);
1463         }
1464 }
1465
1466 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1467 {
1468         int j;
1469
1470         for_each_rx_queue_cnic(bp, j) {
1471                 bnx2x_free_rx_bds(&bp->fp[j]);
1472         }
1473 }
1474
1475 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1476 {
1477         int j;
1478
1479         for_each_eth_queue(bp, j) {
1480                 struct bnx2x_fastpath *fp = &bp->fp[j];
1481
1482                 bnx2x_free_rx_bds(fp);
1483
1484                 if (!fp->disable_tpa)
1485                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1486         }
1487 }
1488
1489 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1490 {
1491         bnx2x_free_tx_skbs_cnic(bp);
1492         bnx2x_free_rx_skbs_cnic(bp);
1493 }
1494
1495 void bnx2x_free_skbs(struct bnx2x *bp)
1496 {
1497         bnx2x_free_tx_skbs(bp);
1498         bnx2x_free_rx_skbs(bp);
1499 }
1500
1501 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1502 {
1503         /* load old values */
1504         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1505
1506         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1507                 /* leave all but MAX value */
1508                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1509
1510                 /* set new MAX value */
1511                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1512                                 & FUNC_MF_CFG_MAX_BW_MASK;
1513
1514                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1515         }
1516 }
1517
1518 /**
1519  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1520  *
1521  * @bp:         driver handle
1522  * @nvecs:      number of vectors to be released
1523  */
1524 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1525 {
1526         int i, offset = 0;
1527
1528         if (nvecs == offset)
1529                 return;
1530
1531         /* VFs don't have a default SB */
1532         if (IS_PF(bp)) {
1533                 free_irq(bp->msix_table[offset].vector, bp->dev);
1534                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1535                    bp->msix_table[offset].vector);
1536                 offset++;
1537         }
1538
1539         if (CNIC_SUPPORT(bp)) {
1540                 if (nvecs == offset)
1541                         return;
1542                 offset++;
1543         }
1544
1545         for_each_eth_queue(bp, i) {
1546                 if (nvecs == offset)
1547                         return;
1548                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1549                    i, bp->msix_table[offset].vector);
1550
1551                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1552         }
1553 }
1554
1555 void bnx2x_free_irq(struct bnx2x *bp)
1556 {
1557         if (bp->flags & USING_MSIX_FLAG &&
1558             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1559                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1560
1561                 /* vfs don't have a default status block */
1562                 if (IS_PF(bp))
1563                         nvecs++;
1564
1565                 bnx2x_free_msix_irqs(bp, nvecs);
1566         } else {
1567                 free_irq(bp->dev->irq, bp->dev);
1568         }
1569 }
1570
1571 int bnx2x_enable_msix(struct bnx2x *bp)
1572 {
1573         int msix_vec = 0, i, rc;
1574
1575         /* VFs don't have a default status block */
1576         if (IS_PF(bp)) {
1577                 bp->msix_table[msix_vec].entry = msix_vec;
1578                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1579                                bp->msix_table[0].entry);
1580                 msix_vec++;
1581         }
1582
1583         /* Cnic requires an msix vector for itself */
1584         if (CNIC_SUPPORT(bp)) {
1585                 bp->msix_table[msix_vec].entry = msix_vec;
1586                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1587                                msix_vec, bp->msix_table[msix_vec].entry);
1588                 msix_vec++;
1589         }
1590
1591         /* We need separate vectors for ETH queues only (not FCoE) */
1592         for_each_eth_queue(bp, i) {
1593                 bp->msix_table[msix_vec].entry = msix_vec;
1594                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1595                                msix_vec, msix_vec, i);
1596                 msix_vec++;
1597         }
1598
1599         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1600            msix_vec);
1601
1602         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1603
1604         /*
1605          * reconfigure number of tx/rx queues according to available
1606          * MSI-X vectors
1607          */
1608         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1609                 /* how less vectors we will have? */
1610                 int diff = msix_vec - rc;
1611
1612                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1613
1614                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1615
1616                 if (rc) {
1617                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1618                         goto no_msix;
1619                 }
1620                 /*
1621                  * decrease number of queues by number of unallocated entries
1622                  */
1623                 bp->num_ethernet_queues -= diff;
1624                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1625
1626                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1627                                bp->num_queues);
1628         } else if (rc > 0) {
1629                 /* Get by with single vector */
1630                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1631                 if (rc) {
1632                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1633                                        rc);
1634                         goto no_msix;
1635                 }
1636
1637                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1638                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1639
1640                 BNX2X_DEV_INFO("set number of queues to 1\n");
1641                 bp->num_ethernet_queues = 1;
1642                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1643         } else if (rc < 0) {
1644                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1645                 goto no_msix;
1646         }
1647
1648         bp->flags |= USING_MSIX_FLAG;
1649
1650         return 0;
1651
1652 no_msix:
1653         /* fall to INTx if not enough memory */
1654         if (rc == -ENOMEM)
1655                 bp->flags |= DISABLE_MSI_FLAG;
1656
1657         return rc;
1658 }
1659
1660 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1661 {
1662         int i, rc, offset = 0;
1663
1664         /* no default status block for vf */
1665         if (IS_PF(bp)) {
1666                 rc = request_irq(bp->msix_table[offset++].vector,
1667                                  bnx2x_msix_sp_int, 0,
1668                                  bp->dev->name, bp->dev);
1669                 if (rc) {
1670                         BNX2X_ERR("request sp irq failed\n");
1671                         return -EBUSY;
1672                 }
1673         }
1674
1675         if (CNIC_SUPPORT(bp))
1676                 offset++;
1677
1678         for_each_eth_queue(bp, i) {
1679                 struct bnx2x_fastpath *fp = &bp->fp[i];
1680                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1681                          bp->dev->name, i);
1682
1683                 rc = request_irq(bp->msix_table[offset].vector,
1684                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1685                 if (rc) {
1686                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1687                               bp->msix_table[offset].vector, rc);
1688                         bnx2x_free_msix_irqs(bp, offset);
1689                         return -EBUSY;
1690                 }
1691
1692                 offset++;
1693         }
1694
1695         i = BNX2X_NUM_ETH_QUEUES(bp);
1696         if (IS_PF(bp)) {
1697                 offset = 1 + CNIC_SUPPORT(bp);
1698                 netdev_info(bp->dev,
1699                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1700                             bp->msix_table[0].vector,
1701                             0, bp->msix_table[offset].vector,
1702                             i - 1, bp->msix_table[offset + i - 1].vector);
1703         } else {
1704                 offset = CNIC_SUPPORT(bp);
1705                 netdev_info(bp->dev,
1706                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1707                             0, bp->msix_table[offset].vector,
1708                             i - 1, bp->msix_table[offset + i - 1].vector);
1709         }
1710         return 0;
1711 }
1712
1713 int bnx2x_enable_msi(struct bnx2x *bp)
1714 {
1715         int rc;
1716
1717         rc = pci_enable_msi(bp->pdev);
1718         if (rc) {
1719                 BNX2X_DEV_INFO("MSI is not attainable\n");
1720                 return -1;
1721         }
1722         bp->flags |= USING_MSI_FLAG;
1723
1724         return 0;
1725 }
1726
1727 static int bnx2x_req_irq(struct bnx2x *bp)
1728 {
1729         unsigned long flags;
1730         unsigned int irq;
1731
1732         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1733                 flags = 0;
1734         else
1735                 flags = IRQF_SHARED;
1736
1737         if (bp->flags & USING_MSIX_FLAG)
1738                 irq = bp->msix_table[0].vector;
1739         else
1740                 irq = bp->pdev->irq;
1741
1742         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1743 }
1744
1745 static int bnx2x_setup_irqs(struct bnx2x *bp)
1746 {
1747         int rc = 0;
1748         if (bp->flags & USING_MSIX_FLAG &&
1749             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1750                 rc = bnx2x_req_msix_irqs(bp);
1751                 if (rc)
1752                         return rc;
1753         } else {
1754                 rc = bnx2x_req_irq(bp);
1755                 if (rc) {
1756                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1757                         return rc;
1758                 }
1759                 if (bp->flags & USING_MSI_FLAG) {
1760                         bp->dev->irq = bp->pdev->irq;
1761                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1762                                     bp->dev->irq);
1763                 }
1764                 if (bp->flags & USING_MSIX_FLAG) {
1765                         bp->dev->irq = bp->msix_table[0].vector;
1766                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1767                                     bp->dev->irq);
1768                 }
1769         }
1770
1771         return 0;
1772 }
1773
1774 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1775 {
1776         int i;
1777
1778         for_each_rx_queue_cnic(bp, i) {
1779                 bnx2x_fp_init_lock(&bp->fp[i]);
1780                 napi_enable(&bnx2x_fp(bp, i, napi));
1781         }
1782 }
1783
1784 static void bnx2x_napi_enable(struct bnx2x *bp)
1785 {
1786         int i;
1787
1788         for_each_eth_queue(bp, i) {
1789                 bnx2x_fp_init_lock(&bp->fp[i]);
1790                 napi_enable(&bnx2x_fp(bp, i, napi));
1791         }
1792 }
1793
1794 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1795 {
1796         int i;
1797
1798         for_each_rx_queue_cnic(bp, i) {
1799                 napi_disable(&bnx2x_fp(bp, i, napi));
1800                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1801                         usleep_range(1000, 2000);
1802         }
1803 }
1804
1805 static void bnx2x_napi_disable(struct bnx2x *bp)
1806 {
1807         int i;
1808
1809         for_each_eth_queue(bp, i) {
1810                 napi_disable(&bnx2x_fp(bp, i, napi));
1811                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1812                         usleep_range(1000, 2000);
1813         }
1814 }
1815
1816 void bnx2x_netif_start(struct bnx2x *bp)
1817 {
1818         if (netif_running(bp->dev)) {
1819                 bnx2x_napi_enable(bp);
1820                 if (CNIC_LOADED(bp))
1821                         bnx2x_napi_enable_cnic(bp);
1822                 bnx2x_int_enable(bp);
1823                 if (bp->state == BNX2X_STATE_OPEN)
1824                         netif_tx_wake_all_queues(bp->dev);
1825         }
1826 }
1827
1828 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1829 {
1830         bnx2x_int_disable_sync(bp, disable_hw);
1831         bnx2x_napi_disable(bp);
1832         if (CNIC_LOADED(bp))
1833                 bnx2x_napi_disable_cnic(bp);
1834 }
1835
1836 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1837                        void *accel_priv)
1838 {
1839         struct bnx2x *bp = netdev_priv(dev);
1840
1841         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1842                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1843                 u16 ether_type = ntohs(hdr->h_proto);
1844
1845                 /* Skip VLAN tag if present */
1846                 if (ether_type == ETH_P_8021Q) {
1847                         struct vlan_ethhdr *vhdr =
1848                                 (struct vlan_ethhdr *)skb->data;
1849
1850                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1851                 }
1852
1853                 /* If ethertype is FCoE or FIP - use FCoE ring */
1854                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1855                         return bnx2x_fcoe_tx(bp, txq_index);
1856         }
1857
1858         /* select a non-FCoE queue */
1859         return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1860 }
1861
1862 void bnx2x_set_num_queues(struct bnx2x *bp)
1863 {
1864         /* RSS queues */
1865         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1866
1867         /* override in STORAGE SD modes */
1868         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1869                 bp->num_ethernet_queues = 1;
1870
1871         /* Add special queues */
1872         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1873         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1874
1875         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1876 }
1877
1878 /**
1879  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1880  *
1881  * @bp:         Driver handle
1882  *
1883  * We currently support for at most 16 Tx queues for each CoS thus we will
1884  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1885  * bp->max_cos.
1886  *
1887  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1888  * index after all ETH L2 indices.
1889  *
1890  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1891  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1892  * 16..31,...) with indices that are not coupled with any real Tx queue.
1893  *
1894  * The proper configuration of skb->queue_mapping is handled by
1895  * bnx2x_select_queue() and __skb_tx_hash().
1896  *
1897  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1898  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1899  */
1900 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1901 {
1902         int rc, tx, rx;
1903
1904         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1905         rx = BNX2X_NUM_ETH_QUEUES(bp);
1906
1907 /* account for fcoe queue */
1908         if (include_cnic && !NO_FCOE(bp)) {
1909                 rx++;
1910                 tx++;
1911         }
1912
1913         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1914         if (rc) {
1915                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1916                 return rc;
1917         }
1918         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1919         if (rc) {
1920                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1921                 return rc;
1922         }
1923
1924         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1925                           tx, rx);
1926
1927         return rc;
1928 }
1929
1930 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1931 {
1932         int i;
1933
1934         for_each_queue(bp, i) {
1935                 struct bnx2x_fastpath *fp = &bp->fp[i];
1936                 u32 mtu;
1937
1938                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1939                 if (IS_FCOE_IDX(i))
1940                         /*
1941                          * Although there are no IP frames expected to arrive to
1942                          * this ring we still want to add an
1943                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1944                          * overrun attack.
1945                          */
1946                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1947                 else
1948                         mtu = bp->dev->mtu;
1949                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1950                                   IP_HEADER_ALIGNMENT_PADDING +
1951                                   ETH_OVREHEAD +
1952                                   mtu +
1953                                   BNX2X_FW_RX_ALIGN_END;
1954                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1955                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1956                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1957                 else
1958                         fp->rx_frag_size = 0;
1959         }
1960 }
1961
1962 static int bnx2x_init_rss(struct bnx2x *bp)
1963 {
1964         int i;
1965         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1966
1967         /* Prepare the initial contents for the indirection table if RSS is
1968          * enabled
1969          */
1970         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1971                 bp->rss_conf_obj.ind_table[i] =
1972                         bp->fp->cl_id +
1973                         ethtool_rxfh_indir_default(i, num_eth_queues);
1974
1975         /*
1976          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1977          * per-port, so if explicit configuration is needed , do it only
1978          * for a PMF.
1979          *
1980          * For 57712 and newer on the other hand it's a per-function
1981          * configuration.
1982          */
1983         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1984 }
1985
1986 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1987               bool config_hash, bool enable)
1988 {
1989         struct bnx2x_config_rss_params params = {NULL};
1990
1991         /* Although RSS is meaningless when there is a single HW queue we
1992          * still need it enabled in order to have HW Rx hash generated.
1993          *
1994          * if (!is_eth_multi(bp))
1995          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1996          */
1997
1998         params.rss_obj = rss_obj;
1999
2000         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2001
2002         if (enable) {
2003                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2004
2005                 /* RSS configuration */
2006                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2007                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2008                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2009                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2010                 if (rss_obj->udp_rss_v4)
2011                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2012                 if (rss_obj->udp_rss_v6)
2013                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2014         } else {
2015                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2016         }
2017
2018         /* Hash bits */
2019         params.rss_result_mask = MULTI_MASK;
2020
2021         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2022
2023         if (config_hash) {
2024                 /* RSS keys */
2025                 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2026                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2027         }
2028
2029         if (IS_PF(bp))
2030                 return bnx2x_config_rss(bp, &params);
2031         else
2032                 return bnx2x_vfpf_config_rss(bp, &params);
2033 }
2034
2035 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2036 {
2037         struct bnx2x_func_state_params func_params = {NULL};
2038
2039         /* Prepare parameters for function state transitions */
2040         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2041
2042         func_params.f_obj = &bp->func_obj;
2043         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2044
2045         func_params.params.hw_init.load_phase = load_code;
2046
2047         return bnx2x_func_state_change(bp, &func_params);
2048 }
2049
2050 /*
2051  * Cleans the object that have internal lists without sending
2052  * ramrods. Should be run when interrupts are disabled.
2053  */
2054 void bnx2x_squeeze_objects(struct bnx2x *bp)
2055 {
2056         int rc;
2057         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2058         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2059         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2060
2061         /***************** Cleanup MACs' object first *************************/
2062
2063         /* Wait for completion of requested */
2064         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2065         /* Perform a dry cleanup */
2066         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2067
2068         /* Clean ETH primary MAC */
2069         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2070         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2071                                  &ramrod_flags);
2072         if (rc != 0)
2073                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2074
2075         /* Cleanup UC list */
2076         vlan_mac_flags = 0;
2077         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2078         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2079                                  &ramrod_flags);
2080         if (rc != 0)
2081                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2082
2083         /***************** Now clean mcast object *****************************/
2084         rparam.mcast_obj = &bp->mcast_obj;
2085         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2086
2087         /* Add a DEL command... - Since we're doing a driver cleanup only,
2088          * we take a lock surrounding both the initial send and the CONTs,
2089          * as we don't want a true completion to disrupt us in the middle.
2090          */
2091         netif_addr_lock_bh(bp->dev);
2092         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2093         if (rc < 0)
2094                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2095                           rc);
2096
2097         /* ...and wait until all pending commands are cleared */
2098         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2099         while (rc != 0) {
2100                 if (rc < 0) {
2101                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2102                                   rc);
2103                         netif_addr_unlock_bh(bp->dev);
2104                         return;
2105                 }
2106
2107                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2108         }
2109         netif_addr_unlock_bh(bp->dev);
2110 }
2111
2112 #ifndef BNX2X_STOP_ON_ERROR
2113 #define LOAD_ERROR_EXIT(bp, label) \
2114         do { \
2115                 (bp)->state = BNX2X_STATE_ERROR; \
2116                 goto label; \
2117         } while (0)
2118
2119 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2120         do { \
2121                 bp->cnic_loaded = false; \
2122                 goto label; \
2123         } while (0)
2124 #else /*BNX2X_STOP_ON_ERROR*/
2125 #define LOAD_ERROR_EXIT(bp, label) \
2126         do { \
2127                 (bp)->state = BNX2X_STATE_ERROR; \
2128                 (bp)->panic = 1; \
2129                 return -EBUSY; \
2130         } while (0)
2131 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2132         do { \
2133                 bp->cnic_loaded = false; \
2134                 (bp)->panic = 1; \
2135                 return -EBUSY; \
2136         } while (0)
2137 #endif /*BNX2X_STOP_ON_ERROR*/
2138
2139 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2140 {
2141         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2142                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2143         return;
2144 }
2145
2146 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2147 {
2148         int num_groups, vf_headroom = 0;
2149         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2150
2151         /* number of queues for statistics is number of eth queues + FCoE */
2152         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2153
2154         /* Total number of FW statistics requests =
2155          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2156          * and fcoe l2 queue) stats + num of queues (which includes another 1
2157          * for fcoe l2 queue if applicable)
2158          */
2159         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2160
2161         /* vf stats appear in the request list, but their data is allocated by
2162          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2163          * it is used to determine where to place the vf stats queries in the
2164          * request struct
2165          */
2166         if (IS_SRIOV(bp))
2167                 vf_headroom = bnx2x_vf_headroom(bp);
2168
2169         /* Request is built from stats_query_header and an array of
2170          * stats_query_cmd_group each of which contains
2171          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2172          * configured in the stats_query_header.
2173          */
2174         num_groups =
2175                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2176                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2177                  1 : 0));
2178
2179         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2180            bp->fw_stats_num, vf_headroom, num_groups);
2181         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2182                 num_groups * sizeof(struct stats_query_cmd_group);
2183
2184         /* Data for statistics requests + stats_counter
2185          * stats_counter holds per-STORM counters that are incremented
2186          * when STORM has finished with the current request.
2187          * memory for FCoE offloaded statistics are counted anyway,
2188          * even if they will not be sent.
2189          * VF stats are not accounted for here as the data of VF stats is stored
2190          * in memory allocated by the VF, not here.
2191          */
2192         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2193                 sizeof(struct per_pf_stats) +
2194                 sizeof(struct fcoe_statistics_params) +
2195                 sizeof(struct per_queue_stats) * num_queue_stats +
2196                 sizeof(struct stats_counter);
2197
2198         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2199                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2200
2201         /* Set shortcuts */
2202         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2203         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2204         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2205                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2206         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2207                 bp->fw_stats_req_sz;
2208
2209         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2210            U64_HI(bp->fw_stats_req_mapping),
2211            U64_LO(bp->fw_stats_req_mapping));
2212         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2213            U64_HI(bp->fw_stats_data_mapping),
2214            U64_LO(bp->fw_stats_data_mapping));
2215         return 0;
2216
2217 alloc_mem_err:
2218         bnx2x_free_fw_stats_mem(bp);
2219         BNX2X_ERR("Can't allocate FW stats memory\n");
2220         return -ENOMEM;
2221 }
2222
2223 /* send load request to mcp and analyze response */
2224 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2225 {
2226         u32 param;
2227
2228         /* init fw_seq */
2229         bp->fw_seq =
2230                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2231                  DRV_MSG_SEQ_NUMBER_MASK);
2232         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2233
2234         /* Get current FW pulse sequence */
2235         bp->fw_drv_pulse_wr_seq =
2236                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2237                  DRV_PULSE_SEQ_MASK);
2238         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2239
2240         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2241
2242         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2243                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2244
2245         /* load request */
2246         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2247
2248         /* if mcp fails to respond we must abort */
2249         if (!(*load_code)) {
2250                 BNX2X_ERR("MCP response failure, aborting\n");
2251                 return -EBUSY;
2252         }
2253
2254         /* If mcp refused (e.g. other port is in diagnostic mode) we
2255          * must abort
2256          */
2257         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2258                 BNX2X_ERR("MCP refused load request, aborting\n");
2259                 return -EBUSY;
2260         }
2261         return 0;
2262 }
2263
2264 /* check whether another PF has already loaded FW to chip. In
2265  * virtualized environments a pf from another VM may have already
2266  * initialized the device including loading FW
2267  */
2268 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2269 {
2270         /* is another pf loaded on this engine? */
2271         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2272             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2273                 /* build my FW version dword */
2274                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2275                         (BCM_5710_FW_MINOR_VERSION << 8) +
2276                         (BCM_5710_FW_REVISION_VERSION << 16) +
2277                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2278
2279                 /* read loaded FW from chip */
2280                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2281
2282                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2283                    loaded_fw, my_fw);
2284
2285                 /* abort nic load if version mismatch */
2286                 if (my_fw != loaded_fw) {
2287                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2288                                   loaded_fw, my_fw);
2289                         return -EBUSY;
2290                 }
2291         }
2292         return 0;
2293 }
2294
2295 /* returns the "mcp load_code" according to global load_count array */
2296 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2297 {
2298         int path = BP_PATH(bp);
2299
2300         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2301            path, load_count[path][0], load_count[path][1],
2302            load_count[path][2]);
2303         load_count[path][0]++;
2304         load_count[path][1 + port]++;
2305         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2306            path, load_count[path][0], load_count[path][1],
2307            load_count[path][2]);
2308         if (load_count[path][0] == 1)
2309                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2310         else if (load_count[path][1 + port] == 1)
2311                 return FW_MSG_CODE_DRV_LOAD_PORT;
2312         else
2313                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2314 }
2315
2316 /* mark PMF if applicable */
2317 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2318 {
2319         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2320             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2321             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2322                 bp->port.pmf = 1;
2323                 /* We need the barrier to ensure the ordering between the
2324                  * writing to bp->port.pmf here and reading it from the
2325                  * bnx2x_periodic_task().
2326                  */
2327                 smp_mb();
2328         } else {
2329                 bp->port.pmf = 0;
2330         }
2331
2332         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2333 }
2334
2335 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2336 {
2337         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2338              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2339             (bp->common.shmem2_base)) {
2340                 if (SHMEM2_HAS(bp, dcc_support))
2341                         SHMEM2_WR(bp, dcc_support,
2342                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2343                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2344                 if (SHMEM2_HAS(bp, afex_driver_support))
2345                         SHMEM2_WR(bp, afex_driver_support,
2346                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2347         }
2348
2349         /* Set AFEX default VLAN tag to an invalid value */
2350         bp->afex_def_vlan_tag = -1;
2351 }
2352
2353 /**
2354  * bnx2x_bz_fp - zero content of the fastpath structure.
2355  *
2356  * @bp:         driver handle
2357  * @index:      fastpath index to be zeroed
2358  *
2359  * Makes sure the contents of the bp->fp[index].napi is kept
2360  * intact.
2361  */
2362 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2363 {
2364         struct bnx2x_fastpath *fp = &bp->fp[index];
2365         int cos;
2366         struct napi_struct orig_napi = fp->napi;
2367         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2368
2369         /* bzero bnx2x_fastpath contents */
2370         if (fp->tpa_info)
2371                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2372                        sizeof(struct bnx2x_agg_info));
2373         memset(fp, 0, sizeof(*fp));
2374
2375         /* Restore the NAPI object as it has been already initialized */
2376         fp->napi = orig_napi;
2377         fp->tpa_info = orig_tpa_info;
2378         fp->bp = bp;
2379         fp->index = index;
2380         if (IS_ETH_FP(fp))
2381                 fp->max_cos = bp->max_cos;
2382         else
2383                 /* Special queues support only one CoS */
2384                 fp->max_cos = 1;
2385
2386         /* Init txdata pointers */
2387         if (IS_FCOE_FP(fp))
2388                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2389         if (IS_ETH_FP(fp))
2390                 for_each_cos_in_tx_queue(fp, cos)
2391                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2392                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2393
2394         /* set the tpa flag for each queue. The tpa flag determines the queue
2395          * minimal size so it must be set prior to queue memory allocation
2396          */
2397         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2398                                   (bp->flags & GRO_ENABLE_FLAG &&
2399                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2400         if (bp->flags & TPA_ENABLE_FLAG)
2401                 fp->mode = TPA_MODE_LRO;
2402         else if (bp->flags & GRO_ENABLE_FLAG)
2403                 fp->mode = TPA_MODE_GRO;
2404
2405         /* We don't want TPA on an FCoE L2 ring */
2406         if (IS_FCOE_FP(fp))
2407                 fp->disable_tpa = 1;
2408 }
2409
2410 int bnx2x_load_cnic(struct bnx2x *bp)
2411 {
2412         int i, rc, port = BP_PORT(bp);
2413
2414         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2415
2416         mutex_init(&bp->cnic_mutex);
2417
2418         if (IS_PF(bp)) {
2419                 rc = bnx2x_alloc_mem_cnic(bp);
2420                 if (rc) {
2421                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2422                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2423                 }
2424         }
2425
2426         rc = bnx2x_alloc_fp_mem_cnic(bp);
2427         if (rc) {
2428                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2429                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2430         }
2431
2432         /* Update the number of queues with the cnic queues */
2433         rc = bnx2x_set_real_num_queues(bp, 1);
2434         if (rc) {
2435                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2436                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2437         }
2438
2439         /* Add all CNIC NAPI objects */
2440         bnx2x_add_all_napi_cnic(bp);
2441         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2442         bnx2x_napi_enable_cnic(bp);
2443
2444         rc = bnx2x_init_hw_func_cnic(bp);
2445         if (rc)
2446                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2447
2448         bnx2x_nic_init_cnic(bp);
2449
2450         if (IS_PF(bp)) {
2451                 /* Enable Timer scan */
2452                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2453
2454                 /* setup cnic queues */
2455                 for_each_cnic_queue(bp, i) {
2456                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2457                         if (rc) {
2458                                 BNX2X_ERR("Queue setup failed\n");
2459                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2460                         }
2461                 }
2462         }
2463
2464         /* Initialize Rx filter. */
2465         bnx2x_set_rx_mode_inner(bp);
2466
2467         /* re-read iscsi info */
2468         bnx2x_get_iscsi_info(bp);
2469         bnx2x_setup_cnic_irq_info(bp);
2470         bnx2x_setup_cnic_info(bp);
2471         bp->cnic_loaded = true;
2472         if (bp->state == BNX2X_STATE_OPEN)
2473                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2474
2475         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2476
2477         return 0;
2478
2479 #ifndef BNX2X_STOP_ON_ERROR
2480 load_error_cnic2:
2481         /* Disable Timer scan */
2482         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2483
2484 load_error_cnic1:
2485         bnx2x_napi_disable_cnic(bp);
2486         /* Update the number of queues without the cnic queues */
2487         if (bnx2x_set_real_num_queues(bp, 0))
2488                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2489 load_error_cnic0:
2490         BNX2X_ERR("CNIC-related load failed\n");
2491         bnx2x_free_fp_mem_cnic(bp);
2492         bnx2x_free_mem_cnic(bp);
2493         return rc;
2494 #endif /* ! BNX2X_STOP_ON_ERROR */
2495 }
2496
2497 /* must be called with rtnl_lock */
2498 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2499 {
2500         int port = BP_PORT(bp);
2501         int i, rc = 0, load_code = 0;
2502
2503         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2504         DP(NETIF_MSG_IFUP,
2505            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2506
2507 #ifdef BNX2X_STOP_ON_ERROR
2508         if (unlikely(bp->panic)) {
2509                 BNX2X_ERR("Can't load NIC when there is panic\n");
2510                 return -EPERM;
2511         }
2512 #endif
2513
2514         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2515
2516         /* zero the structure w/o any lock, before SP handler is initialized */
2517         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2518         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2519                 &bp->last_reported_link.link_report_flags);
2520
2521         if (IS_PF(bp))
2522                 /* must be called before memory allocation and HW init */
2523                 bnx2x_ilt_set_info(bp);
2524
2525         /*
2526          * Zero fastpath structures preserving invariants like napi, which are
2527          * allocated only once, fp index, max_cos, bp pointer.
2528          * Also set fp->disable_tpa and txdata_ptr.
2529          */
2530         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2531         for_each_queue(bp, i)
2532                 bnx2x_bz_fp(bp, i);
2533         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2534                                   bp->num_cnic_queues) *
2535                                   sizeof(struct bnx2x_fp_txdata));
2536
2537         bp->fcoe_init = false;
2538
2539         /* Set the receive queues buffer size */
2540         bnx2x_set_rx_buf_size(bp);
2541
2542         if (IS_PF(bp)) {
2543                 rc = bnx2x_alloc_mem(bp);
2544                 if (rc) {
2545                         BNX2X_ERR("Unable to allocate bp memory\n");
2546                         return rc;
2547                 }
2548         }
2549
2550         /* need to be done after alloc mem, since it's self adjusting to amount
2551          * of memory available for RSS queues
2552          */
2553         rc = bnx2x_alloc_fp_mem(bp);
2554         if (rc) {
2555                 BNX2X_ERR("Unable to allocate memory for fps\n");
2556                 LOAD_ERROR_EXIT(bp, load_error0);
2557         }
2558
2559         /* Allocated memory for FW statistics  */
2560         if (bnx2x_alloc_fw_stats_mem(bp))
2561                 LOAD_ERROR_EXIT(bp, load_error0);
2562
2563         /* request pf to initialize status blocks */
2564         if (IS_VF(bp)) {
2565                 rc = bnx2x_vfpf_init(bp);
2566                 if (rc)
2567                         LOAD_ERROR_EXIT(bp, load_error0);
2568         }
2569
2570         /* As long as bnx2x_alloc_mem() may possibly update
2571          * bp->num_queues, bnx2x_set_real_num_queues() should always
2572          * come after it. At this stage cnic queues are not counted.
2573          */
2574         rc = bnx2x_set_real_num_queues(bp, 0);
2575         if (rc) {
2576                 BNX2X_ERR("Unable to set real_num_queues\n");
2577                 LOAD_ERROR_EXIT(bp, load_error0);
2578         }
2579
2580         /* configure multi cos mappings in kernel.
2581          * this configuration may be overridden by a multi class queue
2582          * discipline or by a dcbx negotiation result.
2583          */
2584         bnx2x_setup_tc(bp->dev, bp->max_cos);
2585
2586         /* Add all NAPI objects */
2587         bnx2x_add_all_napi(bp);
2588         DP(NETIF_MSG_IFUP, "napi added\n");
2589         bnx2x_napi_enable(bp);
2590
2591         if (IS_PF(bp)) {
2592                 /* set pf load just before approaching the MCP */
2593                 bnx2x_set_pf_load(bp);
2594
2595                 /* if mcp exists send load request and analyze response */
2596                 if (!BP_NOMCP(bp)) {
2597                         /* attempt to load pf */
2598                         rc = bnx2x_nic_load_request(bp, &load_code);
2599                         if (rc)
2600                                 LOAD_ERROR_EXIT(bp, load_error1);
2601
2602                         /* what did mcp say? */
2603                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2604                         if (rc) {
2605                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2606                                 LOAD_ERROR_EXIT(bp, load_error2);
2607                         }
2608                 } else {
2609                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2610                 }
2611
2612                 /* mark pmf if applicable */
2613                 bnx2x_nic_load_pmf(bp, load_code);
2614
2615                 /* Init Function state controlling object */
2616                 bnx2x__init_func_obj(bp);
2617
2618                 /* Initialize HW */
2619                 rc = bnx2x_init_hw(bp, load_code);
2620                 if (rc) {
2621                         BNX2X_ERR("HW init failed, aborting\n");
2622                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2623                         LOAD_ERROR_EXIT(bp, load_error2);
2624                 }
2625         }
2626
2627         bnx2x_pre_irq_nic_init(bp);
2628
2629         /* Connect to IRQs */
2630         rc = bnx2x_setup_irqs(bp);
2631         if (rc) {
2632                 BNX2X_ERR("setup irqs failed\n");
2633                 if (IS_PF(bp))
2634                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2635                 LOAD_ERROR_EXIT(bp, load_error2);
2636         }
2637
2638         /* Init per-function objects */
2639         if (IS_PF(bp)) {
2640                 /* Setup NIC internals and enable interrupts */
2641                 bnx2x_post_irq_nic_init(bp, load_code);
2642
2643                 bnx2x_init_bp_objs(bp);
2644                 bnx2x_iov_nic_init(bp);
2645
2646                 /* Set AFEX default VLAN tag to an invalid value */
2647                 bp->afex_def_vlan_tag = -1;
2648                 bnx2x_nic_load_afex_dcc(bp, load_code);
2649                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2650                 rc = bnx2x_func_start(bp);
2651                 if (rc) {
2652                         BNX2X_ERR("Function start failed!\n");
2653                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2654
2655                         LOAD_ERROR_EXIT(bp, load_error3);
2656                 }
2657
2658                 /* Send LOAD_DONE command to MCP */
2659                 if (!BP_NOMCP(bp)) {
2660                         load_code = bnx2x_fw_command(bp,
2661                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2662                         if (!load_code) {
2663                                 BNX2X_ERR("MCP response failure, aborting\n");
2664                                 rc = -EBUSY;
2665                                 LOAD_ERROR_EXIT(bp, load_error3);
2666                         }
2667                 }
2668
2669                 /* initialize FW coalescing state machines in RAM */
2670                 bnx2x_update_coalesce(bp);
2671         }
2672
2673         /* setup the leading queue */
2674         rc = bnx2x_setup_leading(bp);
2675         if (rc) {
2676                 BNX2X_ERR("Setup leading failed!\n");
2677                 LOAD_ERROR_EXIT(bp, load_error3);
2678         }
2679
2680         /* set up the rest of the queues */
2681         for_each_nondefault_eth_queue(bp, i) {
2682                 if (IS_PF(bp))
2683                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2684                 else /* VF */
2685                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2686                 if (rc) {
2687                         BNX2X_ERR("Queue %d setup failed\n", i);
2688                         LOAD_ERROR_EXIT(bp, load_error3);
2689                 }
2690         }
2691
2692         /* setup rss */
2693         rc = bnx2x_init_rss(bp);
2694         if (rc) {
2695                 BNX2X_ERR("PF RSS init failed\n");
2696                 LOAD_ERROR_EXIT(bp, load_error3);
2697         }
2698
2699         /* Now when Clients are configured we are ready to work */
2700         bp->state = BNX2X_STATE_OPEN;
2701
2702         /* Configure a ucast MAC */
2703         if (IS_PF(bp))
2704                 rc = bnx2x_set_eth_mac(bp, true);
2705         else /* vf */
2706                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2707                                            true);
2708         if (rc) {
2709                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2710                 LOAD_ERROR_EXIT(bp, load_error3);
2711         }
2712
2713         if (IS_PF(bp) && bp->pending_max) {
2714                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2715                 bp->pending_max = 0;
2716         }
2717
2718         if (bp->port.pmf) {
2719                 rc = bnx2x_initial_phy_init(bp, load_mode);
2720                 if (rc)
2721                         LOAD_ERROR_EXIT(bp, load_error3);
2722         }
2723         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2724
2725         /* Start fast path */
2726
2727         /* Initialize Rx filter. */
2728         bnx2x_set_rx_mode_inner(bp);
2729
2730         /* Start the Tx */
2731         switch (load_mode) {
2732         case LOAD_NORMAL:
2733                 /* Tx queue should be only re-enabled */
2734                 netif_tx_wake_all_queues(bp->dev);
2735                 break;
2736
2737         case LOAD_OPEN:
2738                 netif_tx_start_all_queues(bp->dev);
2739                 smp_mb__after_clear_bit();
2740                 break;
2741
2742         case LOAD_DIAG:
2743         case LOAD_LOOPBACK_EXT:
2744                 bp->state = BNX2X_STATE_DIAG;
2745                 break;
2746
2747         default:
2748                 break;
2749         }
2750
2751         if (bp->port.pmf)
2752                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2753         else
2754                 bnx2x__link_status_update(bp);
2755
2756         /* start the timer */
2757         mod_timer(&bp->timer, jiffies + bp->current_interval);
2758
2759         if (CNIC_ENABLED(bp))
2760                 bnx2x_load_cnic(bp);
2761
2762         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2763                 /* mark driver is loaded in shmem2 */
2764                 u32 val;
2765                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2766                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2767                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2768                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2769         }
2770
2771         /* Wait for all pending SP commands to complete */
2772         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2773                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2774                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2775                 return -EBUSY;
2776         }
2777
2778         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2779         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2780                 bnx2x_dcbx_init(bp, false);
2781
2782         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2783
2784         return 0;
2785
2786 #ifndef BNX2X_STOP_ON_ERROR
2787 load_error3:
2788         if (IS_PF(bp)) {
2789                 bnx2x_int_disable_sync(bp, 1);
2790
2791                 /* Clean queueable objects */
2792                 bnx2x_squeeze_objects(bp);
2793         }
2794
2795         /* Free SKBs, SGEs, TPA pool and driver internals */
2796         bnx2x_free_skbs(bp);
2797         for_each_rx_queue(bp, i)
2798                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2799
2800         /* Release IRQs */
2801         bnx2x_free_irq(bp);
2802 load_error2:
2803         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2804                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2805                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2806         }
2807
2808         bp->port.pmf = 0;
2809 load_error1:
2810         bnx2x_napi_disable(bp);
2811         bnx2x_del_all_napi(bp);
2812
2813         /* clear pf_load status, as it was already set */
2814         if (IS_PF(bp))
2815                 bnx2x_clear_pf_load(bp);
2816 load_error0:
2817         bnx2x_free_fw_stats_mem(bp);
2818         bnx2x_free_fp_mem(bp);
2819         bnx2x_free_mem(bp);
2820
2821         return rc;
2822 #endif /* ! BNX2X_STOP_ON_ERROR */
2823 }
2824
2825 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2826 {
2827         u8 rc = 0, cos, i;
2828
2829         /* Wait until tx fastpath tasks complete */
2830         for_each_tx_queue(bp, i) {
2831                 struct bnx2x_fastpath *fp = &bp->fp[i];
2832
2833                 for_each_cos_in_tx_queue(fp, cos)
2834                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2835                 if (rc)
2836                         return rc;
2837         }
2838         return 0;
2839 }
2840
2841 /* must be called with rtnl_lock */
2842 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2843 {
2844         int i;
2845         bool global = false;
2846
2847         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2848
2849         /* mark driver is unloaded in shmem2 */
2850         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2851                 u32 val;
2852                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2853                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2854                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2855         }
2856
2857         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2858             (bp->state == BNX2X_STATE_CLOSED ||
2859              bp->state == BNX2X_STATE_ERROR)) {
2860                 /* We can get here if the driver has been unloaded
2861                  * during parity error recovery and is either waiting for a
2862                  * leader to complete or for other functions to unload and
2863                  * then ifdown has been issued. In this case we want to
2864                  * unload and let other functions to complete a recovery
2865                  * process.
2866                  */
2867                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2868                 bp->is_leader = 0;
2869                 bnx2x_release_leader_lock(bp);
2870                 smp_mb();
2871
2872                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2873                 BNX2X_ERR("Can't unload in closed or error state\n");
2874                 return -EINVAL;
2875         }
2876
2877         /* Nothing to do during unload if previous bnx2x_nic_load()
2878          * have not completed successfully - all resources are released.
2879          *
2880          * we can get here only after unsuccessful ndo_* callback, during which
2881          * dev->IFF_UP flag is still on.
2882          */
2883         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2884                 return 0;
2885
2886         /* It's important to set the bp->state to the value different from
2887          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2888          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2889          */
2890         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2891         smp_mb();
2892
2893         /* indicate to VFs that the PF is going down */
2894         bnx2x_iov_channel_down(bp);
2895
2896         if (CNIC_LOADED(bp))
2897                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2898
2899         /* Stop Tx */
2900         bnx2x_tx_disable(bp);
2901         netdev_reset_tc(bp->dev);
2902
2903         bp->rx_mode = BNX2X_RX_MODE_NONE;
2904
2905         del_timer_sync(&bp->timer);
2906
2907         if (IS_PF(bp)) {
2908                 /* Set ALWAYS_ALIVE bit in shmem */
2909                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2910                 bnx2x_drv_pulse(bp);
2911                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2912                 bnx2x_save_statistics(bp);
2913         }
2914
2915         /* wait till consumers catch up with producers in all queues */
2916         bnx2x_drain_tx_queues(bp);
2917
2918         /* if VF indicate to PF this function is going down (PF will delete sp
2919          * elements and clear initializations
2920          */
2921         if (IS_VF(bp))
2922                 bnx2x_vfpf_close_vf(bp);
2923         else if (unload_mode != UNLOAD_RECOVERY)
2924                 /* if this is a normal/close unload need to clean up chip*/
2925                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2926         else {
2927                 /* Send the UNLOAD_REQUEST to the MCP */
2928                 bnx2x_send_unload_req(bp, unload_mode);
2929
2930                 /* Prevent transactions to host from the functions on the
2931                  * engine that doesn't reset global blocks in case of global
2932                  * attention once global blocks are reset and gates are opened
2933                  * (the engine which leader will perform the recovery
2934                  * last).
2935                  */
2936                 if (!CHIP_IS_E1x(bp))
2937                         bnx2x_pf_disable(bp);
2938
2939                 /* Disable HW interrupts, NAPI */
2940                 bnx2x_netif_stop(bp, 1);
2941                 /* Delete all NAPI objects */
2942                 bnx2x_del_all_napi(bp);
2943                 if (CNIC_LOADED(bp))
2944                         bnx2x_del_all_napi_cnic(bp);
2945                 /* Release IRQs */
2946                 bnx2x_free_irq(bp);
2947
2948                 /* Report UNLOAD_DONE to MCP */
2949                 bnx2x_send_unload_done(bp, false);
2950         }
2951
2952         /*
2953          * At this stage no more interrupts will arrive so we may safely clean
2954          * the queueable objects here in case they failed to get cleaned so far.
2955          */
2956         if (IS_PF(bp))
2957                 bnx2x_squeeze_objects(bp);
2958
2959         /* There should be no more pending SP commands at this stage */
2960         bp->sp_state = 0;
2961
2962         bp->port.pmf = 0;
2963
2964         /* clear pending work in rtnl task */
2965         bp->sp_rtnl_state = 0;
2966         smp_mb();
2967
2968         /* Free SKBs, SGEs, TPA pool and driver internals */
2969         bnx2x_free_skbs(bp);
2970         if (CNIC_LOADED(bp))
2971                 bnx2x_free_skbs_cnic(bp);
2972         for_each_rx_queue(bp, i)
2973                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2974
2975         bnx2x_free_fp_mem(bp);
2976         if (CNIC_LOADED(bp))
2977                 bnx2x_free_fp_mem_cnic(bp);
2978
2979         if (IS_PF(bp)) {
2980                 if (CNIC_LOADED(bp))
2981                         bnx2x_free_mem_cnic(bp);
2982         }
2983         bnx2x_free_mem(bp);
2984
2985         bp->state = BNX2X_STATE_CLOSED;
2986         bp->cnic_loaded = false;
2987
2988         /* Check if there are pending parity attentions. If there are - set
2989          * RECOVERY_IN_PROGRESS.
2990          */
2991         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2992                 bnx2x_set_reset_in_progress(bp);
2993
2994                 /* Set RESET_IS_GLOBAL if needed */
2995                 if (global)
2996                         bnx2x_set_reset_global(bp);
2997         }
2998
2999         /* The last driver must disable a "close the gate" if there is no
3000          * parity attention or "process kill" pending.
3001          */
3002         if (IS_PF(bp) &&
3003             !bnx2x_clear_pf_load(bp) &&
3004             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3005                 bnx2x_disable_close_the_gate(bp);
3006
3007         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3008
3009         return 0;
3010 }
3011
3012 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3013 {
3014         u16 pmcsr;
3015
3016         /* If there is no power capability, silently succeed */
3017         if (!bp->pdev->pm_cap) {
3018                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3019                 return 0;
3020         }
3021
3022         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3023
3024         switch (state) {
3025         case PCI_D0:
3026                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3027                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3028                                        PCI_PM_CTRL_PME_STATUS));
3029
3030                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3031                         /* delay required during transition out of D3hot */
3032                         msleep(20);
3033                 break;
3034
3035         case PCI_D3hot:
3036                 /* If there are other clients above don't
3037                    shut down the power */
3038                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3039                         return 0;
3040                 /* Don't shut down the power for emulation and FPGA */
3041                 if (CHIP_REV_IS_SLOW(bp))
3042                         return 0;
3043
3044                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3045                 pmcsr |= 3;
3046
3047                 if (bp->wol)
3048                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3049
3050                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3051                                       pmcsr);
3052
3053                 /* No more memory access after this point until
3054                 * device is brought back to D0.
3055                 */
3056                 break;
3057
3058         default:
3059                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3060                 return -EINVAL;
3061         }
3062         return 0;
3063 }
3064
3065 /*
3066  * net_device service functions
3067  */
3068 int bnx2x_poll(struct napi_struct *napi, int budget)
3069 {
3070         int work_done = 0;
3071         u8 cos;
3072         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3073                                                  napi);
3074         struct bnx2x *bp = fp->bp;
3075
3076         while (1) {
3077 #ifdef BNX2X_STOP_ON_ERROR
3078                 if (unlikely(bp->panic)) {
3079                         napi_complete(napi);
3080                         return 0;
3081                 }
3082 #endif
3083                 if (!bnx2x_fp_lock_napi(fp))
3084                         return work_done;
3085
3086                 for_each_cos_in_tx_queue(fp, cos)
3087                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3088                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3089
3090                 if (bnx2x_has_rx_work(fp)) {
3091                         work_done += bnx2x_rx_int(fp, budget - work_done);
3092
3093                         /* must not complete if we consumed full budget */
3094                         if (work_done >= budget) {
3095                                 bnx2x_fp_unlock_napi(fp);
3096                                 break;
3097                         }
3098                 }
3099
3100                 /* Fall out from the NAPI loop if needed */
3101                 if (!bnx2x_fp_unlock_napi(fp) &&
3102                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3103
3104                         /* No need to update SB for FCoE L2 ring as long as
3105                          * it's connected to the default SB and the SB
3106                          * has been updated when NAPI was scheduled.
3107                          */
3108                         if (IS_FCOE_FP(fp)) {
3109                                 napi_complete(napi);
3110                                 break;
3111                         }
3112                         bnx2x_update_fpsb_idx(fp);
3113                         /* bnx2x_has_rx_work() reads the status block,
3114                          * thus we need to ensure that status block indices
3115                          * have been actually read (bnx2x_update_fpsb_idx)
3116                          * prior to this check (bnx2x_has_rx_work) so that
3117                          * we won't write the "newer" value of the status block
3118                          * to IGU (if there was a DMA right after
3119                          * bnx2x_has_rx_work and if there is no rmb, the memory
3120                          * reading (bnx2x_update_fpsb_idx) may be postponed
3121                          * to right before bnx2x_ack_sb). In this case there
3122                          * will never be another interrupt until there is
3123                          * another update of the status block, while there
3124                          * is still unhandled work.
3125                          */
3126                         rmb();
3127
3128                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3129                                 napi_complete(napi);
3130                                 /* Re-enable interrupts */
3131                                 DP(NETIF_MSG_RX_STATUS,
3132                                    "Update index to %d\n", fp->fp_hc_idx);
3133                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3134                                              le16_to_cpu(fp->fp_hc_idx),
3135                                              IGU_INT_ENABLE, 1);
3136                                 break;
3137                         }
3138                 }
3139         }
3140
3141         return work_done;
3142 }
3143
3144 #ifdef CONFIG_NET_RX_BUSY_POLL
3145 /* must be called with local_bh_disable()d */
3146 int bnx2x_low_latency_recv(struct napi_struct *napi)
3147 {
3148         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3149                                                  napi);
3150         struct bnx2x *bp = fp->bp;
3151         int found = 0;
3152
3153         if ((bp->state == BNX2X_STATE_CLOSED) ||
3154             (bp->state == BNX2X_STATE_ERROR) ||
3155             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3156                 return LL_FLUSH_FAILED;
3157
3158         if (!bnx2x_fp_lock_poll(fp))
3159                 return LL_FLUSH_BUSY;
3160
3161         if (bnx2x_has_rx_work(fp))
3162                 found = bnx2x_rx_int(fp, 4);
3163
3164         bnx2x_fp_unlock_poll(fp);
3165
3166         return found;
3167 }
3168 #endif
3169
3170 /* we split the first BD into headers and data BDs
3171  * to ease the pain of our fellow microcode engineers
3172  * we use one mapping for both BDs
3173  */
3174 static u16 bnx2x_tx_split(struct bnx2x *bp,
3175                           struct bnx2x_fp_txdata *txdata,
3176                           struct sw_tx_bd *tx_buf,
3177                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3178                           u16 bd_prod)
3179 {
3180         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3181         struct eth_tx_bd *d_tx_bd;
3182         dma_addr_t mapping;
3183         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3184
3185         /* first fix first BD */
3186         h_tx_bd->nbytes = cpu_to_le16(hlen);
3187
3188         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3189            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3190
3191         /* now get a new data BD
3192          * (after the pbd) and fill it */
3193         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3194         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3195
3196         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3197                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3198
3199         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3200         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3201         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3202
3203         /* this marks the BD as one that has no individual mapping */
3204         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3205
3206         DP(NETIF_MSG_TX_QUEUED,
3207            "TSO split data size is %d (%x:%x)\n",
3208            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3209
3210         /* update tx_bd */
3211         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3212
3213         return bd_prod;
3214 }
3215
3216 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3217 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3218 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3219 {
3220         __sum16 tsum = (__force __sum16) csum;
3221
3222         if (fix > 0)
3223                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3224                                   csum_partial(t_header - fix, fix, 0)));
3225
3226         else if (fix < 0)
3227                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3228                                   csum_partial(t_header, -fix, 0)));
3229
3230         return bswab16(tsum);
3231 }
3232
3233 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3234 {
3235         u32 rc;
3236         __u8 prot = 0;
3237         __be16 protocol;
3238
3239         if (skb->ip_summed != CHECKSUM_PARTIAL)
3240                 return XMIT_PLAIN;
3241
3242         protocol = vlan_get_protocol(skb);
3243         if (protocol == htons(ETH_P_IPV6)) {
3244                 rc = XMIT_CSUM_V6;
3245                 prot = ipv6_hdr(skb)->nexthdr;
3246         } else {
3247                 rc = XMIT_CSUM_V4;
3248                 prot = ip_hdr(skb)->protocol;
3249         }
3250
3251         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3252                 if (inner_ip_hdr(skb)->version == 6) {
3253                         rc |= XMIT_CSUM_ENC_V6;
3254                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3255                                 rc |= XMIT_CSUM_TCP;
3256                 } else {
3257                         rc |= XMIT_CSUM_ENC_V4;
3258                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3259                                 rc |= XMIT_CSUM_TCP;
3260                 }
3261         }
3262         if (prot == IPPROTO_TCP)
3263                 rc |= XMIT_CSUM_TCP;
3264
3265         if (skb_is_gso(skb)) {
3266                 if (skb_is_gso_v6(skb)) {
3267                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3268                         if (rc & XMIT_CSUM_ENC)
3269                                 rc |= XMIT_GSO_ENC_V6;
3270                 } else {
3271                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3272                         if (rc & XMIT_CSUM_ENC)
3273                                 rc |= XMIT_GSO_ENC_V4;
3274                 }
3275         }
3276
3277         return rc;
3278 }
3279
3280 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3281 /* check if packet requires linearization (packet is too fragmented)
3282    no need to check fragmentation if page size > 8K (there will be no
3283    violation to FW restrictions) */
3284 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3285                              u32 xmit_type)
3286 {
3287         int to_copy = 0;
3288         int hlen = 0;
3289         int first_bd_sz = 0;
3290
3291         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3292         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3293
3294                 if (xmit_type & XMIT_GSO) {
3295                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3296                         /* Check if LSO packet needs to be copied:
3297                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3298                         int wnd_size = MAX_FETCH_BD - 3;
3299                         /* Number of windows to check */
3300                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3301                         int wnd_idx = 0;
3302                         int frag_idx = 0;
3303                         u32 wnd_sum = 0;
3304
3305                         /* Headers length */
3306                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3307                                 tcp_hdrlen(skb);
3308
3309                         /* Amount of data (w/o headers) on linear part of SKB*/
3310                         first_bd_sz = skb_headlen(skb) - hlen;
3311
3312                         wnd_sum  = first_bd_sz;
3313
3314                         /* Calculate the first sum - it's special */
3315                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3316                                 wnd_sum +=
3317                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3318
3319                         /* If there was data on linear skb data - check it */
3320                         if (first_bd_sz > 0) {
3321                                 if (unlikely(wnd_sum < lso_mss)) {
3322                                         to_copy = 1;
3323                                         goto exit_lbl;
3324                                 }
3325
3326                                 wnd_sum -= first_bd_sz;
3327                         }
3328
3329                         /* Others are easier: run through the frag list and
3330                            check all windows */
3331                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3332                                 wnd_sum +=
3333                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3334
3335                                 if (unlikely(wnd_sum < lso_mss)) {
3336                                         to_copy = 1;
3337                                         break;
3338                                 }
3339                                 wnd_sum -=
3340                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3341                         }
3342                 } else {
3343                         /* in non-LSO too fragmented packet should always
3344                            be linearized */
3345                         to_copy = 1;
3346                 }
3347         }
3348
3349 exit_lbl:
3350         if (unlikely(to_copy))
3351                 DP(NETIF_MSG_TX_QUEUED,
3352                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3353                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3354                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3355
3356         return to_copy;
3357 }
3358 #endif
3359
3360 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3361                                  u32 xmit_type)
3362 {
3363         struct ipv6hdr *ipv6;
3364
3365         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3366                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3367                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3368
3369         if (xmit_type & XMIT_GSO_ENC_V6)
3370                 ipv6 = inner_ipv6_hdr(skb);
3371         else if (xmit_type & XMIT_GSO_V6)
3372                 ipv6 = ipv6_hdr(skb);
3373         else
3374                 ipv6 = NULL;
3375
3376         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3377                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3378 }
3379
3380 /**
3381  * bnx2x_set_pbd_gso - update PBD in GSO case.
3382  *
3383  * @skb:        packet skb
3384  * @pbd:        parse BD
3385  * @xmit_type:  xmit flags
3386  */
3387 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3388                               struct eth_tx_parse_bd_e1x *pbd,
3389                               struct eth_tx_start_bd *tx_start_bd,
3390                               u32 xmit_type)
3391 {
3392         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3393         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3394         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3395
3396         if (xmit_type & XMIT_GSO_V4) {
3397                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3398                 pbd->tcp_pseudo_csum =
3399                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3400                                                    ip_hdr(skb)->daddr,
3401                                                    0, IPPROTO_TCP, 0));
3402
3403                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3404                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3405         } else {
3406                 pbd->tcp_pseudo_csum =
3407                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3408                                                  &ipv6_hdr(skb)->daddr,
3409                                                  0, IPPROTO_TCP, 0));
3410         }
3411
3412         pbd->global_data |=
3413                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3414 }
3415
3416 /**
3417  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3418  *
3419  * @bp:                 driver handle
3420  * @skb:                packet skb
3421  * @parsing_data:       data to be updated
3422  * @xmit_type:          xmit flags
3423  *
3424  * 57712/578xx related, when skb has encapsulation
3425  */
3426 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3427                                  u32 *parsing_data, u32 xmit_type)
3428 {
3429         *parsing_data |=
3430                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3431                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3432                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3433
3434         if (xmit_type & XMIT_CSUM_TCP) {
3435                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3436                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3437                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3438
3439                 return skb_inner_transport_header(skb) +
3440                         inner_tcp_hdrlen(skb) - skb->data;
3441         }
3442
3443         /* We support checksum offload for TCP and UDP only.
3444          * No need to pass the UDP header length - it's a constant.
3445          */
3446         return skb_inner_transport_header(skb) +
3447                 sizeof(struct udphdr) - skb->data;
3448 }
3449
3450 /**
3451  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3452  *
3453  * @bp:                 driver handle
3454  * @skb:                packet skb
3455  * @parsing_data:       data to be updated
3456  * @xmit_type:          xmit flags
3457  *
3458  * 57712/578xx related
3459  */
3460 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3461                                 u32 *parsing_data, u32 xmit_type)
3462 {
3463         *parsing_data |=
3464                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3465                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3466                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3467
3468         if (xmit_type & XMIT_CSUM_TCP) {
3469                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3470                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3471                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3472
3473                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3474         }
3475         /* We support checksum offload for TCP and UDP only.
3476          * No need to pass the UDP header length - it's a constant.
3477          */
3478         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3479 }
3480
3481 /* set FW indication according to inner or outer protocols if tunneled */
3482 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3483                                struct eth_tx_start_bd *tx_start_bd,
3484                                u32 xmit_type)
3485 {
3486         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3487
3488         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3489                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3490
3491         if (!(xmit_type & XMIT_CSUM_TCP))
3492                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3493 }
3494
3495 /**
3496  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3497  *
3498  * @bp:         driver handle
3499  * @skb:        packet skb
3500  * @pbd:        parse BD to be updated
3501  * @xmit_type:  xmit flags
3502  */
3503 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3504                              struct eth_tx_parse_bd_e1x *pbd,
3505                              u32 xmit_type)
3506 {
3507         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3508
3509         /* for now NS flag is not used in Linux */
3510         pbd->global_data =
3511                 cpu_to_le16(hlen |
3512                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3513                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3514
3515         pbd->ip_hlen_w = (skb_transport_header(skb) -
3516                         skb_network_header(skb)) >> 1;
3517
3518         hlen += pbd->ip_hlen_w;
3519
3520         /* We support checksum offload for TCP and UDP only */
3521         if (xmit_type & XMIT_CSUM_TCP)
3522                 hlen += tcp_hdrlen(skb) / 2;
3523         else
3524                 hlen += sizeof(struct udphdr) / 2;
3525
3526         pbd->total_hlen_w = cpu_to_le16(hlen);
3527         hlen = hlen*2;
3528
3529         if (xmit_type & XMIT_CSUM_TCP) {
3530                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3531
3532         } else {
3533                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3534
3535                 DP(NETIF_MSG_TX_QUEUED,
3536                    "hlen %d  fix %d  csum before fix %x\n",
3537                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3538
3539                 /* HW bug: fixup the CSUM */
3540                 pbd->tcp_pseudo_csum =
3541                         bnx2x_csum_fix(skb_transport_header(skb),
3542                                        SKB_CS(skb), fix);
3543
3544                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3545                    pbd->tcp_pseudo_csum);
3546         }
3547
3548         return hlen;
3549 }
3550
3551 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3552                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3553                                       struct eth_tx_parse_2nd_bd *pbd2,
3554                                       u16 *global_data,
3555                                       u32 xmit_type)
3556 {
3557         u16 hlen_w = 0;
3558         u8 outerip_off, outerip_len = 0;
3559
3560         /* from outer IP to transport */
3561         hlen_w = (skb_inner_transport_header(skb) -
3562                   skb_network_header(skb)) >> 1;
3563
3564         /* transport len */
3565         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3566
3567         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3568
3569         /* outer IP header info */
3570         if (xmit_type & XMIT_CSUM_V4) {
3571                 struct iphdr *iph = ip_hdr(skb);
3572                 u32 csum = (__force u32)(~iph->check) -
3573                            (__force u32)iph->tot_len -
3574                            (__force u32)iph->frag_off;
3575
3576                 pbd2->fw_ip_csum_wo_len_flags_frag =
3577                         bswab16(csum_fold((__force __wsum)csum));
3578         } else {
3579                 pbd2->fw_ip_hdr_to_payload_w =
3580                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3581         }
3582
3583         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3584
3585         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3586
3587         if (xmit_type & XMIT_GSO_V4) {
3588                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3589
3590                 pbd_e2->data.tunnel_data.pseudo_csum =
3591                         bswab16(~csum_tcpudp_magic(
3592                                         inner_ip_hdr(skb)->saddr,
3593                                         inner_ip_hdr(skb)->daddr,
3594                                         0, IPPROTO_TCP, 0));
3595
3596                 outerip_len = ip_hdr(skb)->ihl << 1;
3597         } else {
3598                 pbd_e2->data.tunnel_data.pseudo_csum =
3599                         bswab16(~csum_ipv6_magic(
3600                                         &inner_ipv6_hdr(skb)->saddr,
3601                                         &inner_ipv6_hdr(skb)->daddr,
3602                                         0, IPPROTO_TCP, 0));
3603         }
3604
3605         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3606
3607         *global_data |=
3608                 outerip_off |
3609                 (!!(xmit_type & XMIT_CSUM_V6) <<
3610                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3611                 (outerip_len <<
3612                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3613                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3614                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3615
3616         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3617                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3618                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3619         }
3620 }
3621
3622 /* called with netif_tx_lock
3623  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3624  * netif_wake_queue()
3625  */
3626 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3627 {
3628         struct bnx2x *bp = netdev_priv(dev);
3629
3630         struct netdev_queue *txq;
3631         struct bnx2x_fp_txdata *txdata;
3632         struct sw_tx_bd *tx_buf;
3633         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3634         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3635         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3636         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3637         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3638         u32 pbd_e2_parsing_data = 0;
3639         u16 pkt_prod, bd_prod;
3640         int nbd, txq_index;
3641         dma_addr_t mapping;
3642         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3643         int i;
3644         u8 hlen = 0;
3645         __le16 pkt_size = 0;
3646         struct ethhdr *eth;
3647         u8 mac_type = UNICAST_ADDRESS;
3648
3649 #ifdef BNX2X_STOP_ON_ERROR
3650         if (unlikely(bp->panic))
3651                 return NETDEV_TX_BUSY;
3652 #endif
3653
3654         txq_index = skb_get_queue_mapping(skb);
3655         txq = netdev_get_tx_queue(dev, txq_index);
3656
3657         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3658
3659         txdata = &bp->bnx2x_txq[txq_index];
3660
3661         /* enable this debug print to view the transmission queue being used
3662         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3663            txq_index, fp_index, txdata_index); */
3664
3665         /* enable this debug print to view the transmission details
3666         DP(NETIF_MSG_TX_QUEUED,
3667            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3668            txdata->cid, fp_index, txdata_index, txdata, fp); */
3669
3670         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3671                         skb_shinfo(skb)->nr_frags +
3672                         BDS_PER_TX_PKT +
3673                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3674                 /* Handle special storage cases separately */
3675                 if (txdata->tx_ring_size == 0) {
3676                         struct bnx2x_eth_q_stats *q_stats =
3677                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3678                         q_stats->driver_filtered_tx_pkt++;
3679                         dev_kfree_skb(skb);
3680                         return NETDEV_TX_OK;
3681                 }
3682                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3683                 netif_tx_stop_queue(txq);
3684                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3685
3686                 return NETDEV_TX_BUSY;
3687         }
3688
3689         DP(NETIF_MSG_TX_QUEUED,
3690            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3691            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3692            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3693            skb->len);
3694
3695         eth = (struct ethhdr *)skb->data;
3696
3697         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3698         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3699                 if (is_broadcast_ether_addr(eth->h_dest))
3700                         mac_type = BROADCAST_ADDRESS;
3701                 else
3702                         mac_type = MULTICAST_ADDRESS;
3703         }
3704
3705 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3706         /* First, check if we need to linearize the skb (due to FW
3707            restrictions). No need to check fragmentation if page size > 8K
3708            (there will be no violation to FW restrictions) */
3709         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3710                 /* Statistics of linearization */
3711                 bp->lin_cnt++;
3712                 if (skb_linearize(skb) != 0) {
3713                         DP(NETIF_MSG_TX_QUEUED,
3714                            "SKB linearization failed - silently dropping this SKB\n");
3715                         dev_kfree_skb_any(skb);
3716                         return NETDEV_TX_OK;
3717                 }
3718         }
3719 #endif
3720         /* Map skb linear data for DMA */
3721         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3722                                  skb_headlen(skb), DMA_TO_DEVICE);
3723         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3724                 DP(NETIF_MSG_TX_QUEUED,
3725                    "SKB mapping failed - silently dropping this SKB\n");
3726                 dev_kfree_skb_any(skb);
3727                 return NETDEV_TX_OK;
3728         }
3729         /*
3730         Please read carefully. First we use one BD which we mark as start,
3731         then we have a parsing info BD (used for TSO or xsum),
3732         and only then we have the rest of the TSO BDs.
3733         (don't forget to mark the last one as last,
3734         and to unmap only AFTER you write to the BD ...)
3735         And above all, all pdb sizes are in words - NOT DWORDS!
3736         */
3737
3738         /* get current pkt produced now - advance it just before sending packet
3739          * since mapping of pages may fail and cause packet to be dropped
3740          */
3741         pkt_prod = txdata->tx_pkt_prod;
3742         bd_prod = TX_BD(txdata->tx_bd_prod);
3743
3744         /* get a tx_buf and first BD
3745          * tx_start_bd may be changed during SPLIT,
3746          * but first_bd will always stay first
3747          */
3748         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3749         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3750         first_bd = tx_start_bd;
3751
3752         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3753
3754         /* header nbd: indirectly zero other flags! */
3755         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3756
3757         /* remember the first BD of the packet */
3758         tx_buf->first_bd = txdata->tx_bd_prod;
3759         tx_buf->skb = skb;
3760         tx_buf->flags = 0;
3761
3762         DP(NETIF_MSG_TX_QUEUED,
3763            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3764            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3765
3766         if (vlan_tx_tag_present(skb)) {
3767                 tx_start_bd->vlan_or_ethertype =
3768                     cpu_to_le16(vlan_tx_tag_get(skb));
3769                 tx_start_bd->bd_flags.as_bitfield |=
3770                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3771         } else {
3772                 /* when transmitting in a vf, start bd must hold the ethertype
3773                  * for fw to enforce it
3774                  */
3775                 if (IS_VF(bp))
3776                         tx_start_bd->vlan_or_ethertype =
3777                                 cpu_to_le16(ntohs(eth->h_proto));
3778                 else
3779                         /* used by FW for packet accounting */
3780                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3781         }
3782
3783         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3784
3785         /* turn on parsing and get a BD */
3786         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3787
3788         if (xmit_type & XMIT_CSUM)
3789                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3790
3791         if (!CHIP_IS_E1x(bp)) {
3792                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3793                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3794
3795                 if (xmit_type & XMIT_CSUM_ENC) {
3796                         u16 global_data = 0;
3797
3798                         /* Set PBD in enc checksum offload case */
3799                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3800                                                       &pbd_e2_parsing_data,
3801                                                       xmit_type);
3802
3803                         /* turn on 2nd parsing and get a BD */
3804                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3805
3806                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3807
3808                         memset(pbd2, 0, sizeof(*pbd2));
3809
3810                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3811                                 (skb_inner_network_header(skb) -
3812                                  skb->data) >> 1;
3813
3814                         if (xmit_type & XMIT_GSO_ENC)
3815                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3816                                                           &global_data,
3817                                                           xmit_type);
3818
3819                         pbd2->global_data = cpu_to_le16(global_data);
3820
3821                         /* add addition parse BD indication to start BD */
3822                         SET_FLAG(tx_start_bd->general_data,
3823                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3824                         /* set encapsulation flag in start BD */
3825                         SET_FLAG(tx_start_bd->general_data,
3826                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3827                         nbd++;
3828                 } else if (xmit_type & XMIT_CSUM) {
3829                         /* Set PBD in checksum offload case w/o encapsulation */
3830                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3831                                                      &pbd_e2_parsing_data,
3832                                                      xmit_type);
3833                 }
3834
3835                 /* Add the macs to the parsing BD this is a vf */
3836                 if (IS_VF(bp)) {
3837                         /* override GRE parameters in BD */
3838                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3839                                               &pbd_e2->data.mac_addr.src_mid,
3840                                               &pbd_e2->data.mac_addr.src_lo,
3841                                               eth->h_source);
3842
3843                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3844                                               &pbd_e2->data.mac_addr.dst_mid,
3845                                               &pbd_e2->data.mac_addr.dst_lo,
3846                                               eth->h_dest);
3847                 }
3848
3849                 SET_FLAG(pbd_e2_parsing_data,
3850                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3851         } else {
3852                 u16 global_data = 0;
3853                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3854                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3855                 /* Set PBD in checksum offload case */
3856                 if (xmit_type & XMIT_CSUM)
3857                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3858
3859                 SET_FLAG(global_data,
3860                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3861                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3862         }
3863
3864         /* Setup the data pointer of the first BD of the packet */
3865         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3866         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3867         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3868         pkt_size = tx_start_bd->nbytes;
3869
3870         DP(NETIF_MSG_TX_QUEUED,
3871            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3872            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3873            le16_to_cpu(tx_start_bd->nbytes),
3874            tx_start_bd->bd_flags.as_bitfield,
3875            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3876
3877         if (xmit_type & XMIT_GSO) {
3878
3879                 DP(NETIF_MSG_TX_QUEUED,
3880                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3881                    skb->len, hlen, skb_headlen(skb),
3882                    skb_shinfo(skb)->gso_size);
3883
3884                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3885
3886                 if (unlikely(skb_headlen(skb) > hlen)) {
3887                         nbd++;
3888                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3889                                                  &tx_start_bd, hlen,
3890                                                  bd_prod);
3891                 }
3892                 if (!CHIP_IS_E1x(bp))
3893                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3894                                              xmit_type);
3895                 else
3896                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3897         }
3898
3899         /* Set the PBD's parsing_data field if not zero
3900          * (for the chips newer than 57711).
3901          */
3902         if (pbd_e2_parsing_data)
3903                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3904
3905         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3906
3907         /* Handle fragmented skb */
3908         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3909                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3910
3911                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3912                                            skb_frag_size(frag), DMA_TO_DEVICE);
3913                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3914                         unsigned int pkts_compl = 0, bytes_compl = 0;
3915
3916                         DP(NETIF_MSG_TX_QUEUED,
3917                            "Unable to map page - dropping packet...\n");
3918
3919                         /* we need unmap all buffers already mapped
3920                          * for this SKB;
3921                          * first_bd->nbd need to be properly updated
3922                          * before call to bnx2x_free_tx_pkt
3923                          */
3924                         first_bd->nbd = cpu_to_le16(nbd);
3925                         bnx2x_free_tx_pkt(bp, txdata,
3926                                           TX_BD(txdata->tx_pkt_prod),
3927                                           &pkts_compl, &bytes_compl);
3928                         return NETDEV_TX_OK;
3929                 }
3930
3931                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3932                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3933                 if (total_pkt_bd == NULL)
3934                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3935
3936                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3937                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3938                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3939                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3940                 nbd++;
3941
3942                 DP(NETIF_MSG_TX_QUEUED,
3943                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3944                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3945                    le16_to_cpu(tx_data_bd->nbytes));
3946         }
3947
3948         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3949
3950         /* update with actual num BDs */
3951         first_bd->nbd = cpu_to_le16(nbd);
3952
3953         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3954
3955         /* now send a tx doorbell, counting the next BD
3956          * if the packet contains or ends with it
3957          */
3958         if (TX_BD_POFF(bd_prod) < nbd)
3959                 nbd++;
3960
3961         /* total_pkt_bytes should be set on the first data BD if
3962          * it's not an LSO packet and there is more than one
3963          * data BD. In this case pkt_size is limited by an MTU value.
3964          * However we prefer to set it for an LSO packet (while we don't
3965          * have to) in order to save some CPU cycles in a none-LSO
3966          * case, when we much more care about them.
3967          */
3968         if (total_pkt_bd != NULL)
3969                 total_pkt_bd->total_pkt_bytes = pkt_size;
3970
3971         if (pbd_e1x)
3972                 DP(NETIF_MSG_TX_QUEUED,
3973                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3974                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3975                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3976                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3977                     le16_to_cpu(pbd_e1x->total_hlen_w));
3978         if (pbd_e2)
3979                 DP(NETIF_MSG_TX_QUEUED,
3980                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3981                    pbd_e2,
3982                    pbd_e2->data.mac_addr.dst_hi,
3983                    pbd_e2->data.mac_addr.dst_mid,
3984                    pbd_e2->data.mac_addr.dst_lo,
3985                    pbd_e2->data.mac_addr.src_hi,
3986                    pbd_e2->data.mac_addr.src_mid,
3987                    pbd_e2->data.mac_addr.src_lo,
3988                    pbd_e2->parsing_data);
3989         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3990
3991         netdev_tx_sent_queue(txq, skb->len);
3992
3993         skb_tx_timestamp(skb);
3994
3995         txdata->tx_pkt_prod++;
3996         /*
3997          * Make sure that the BD data is updated before updating the producer
3998          * since FW might read the BD right after the producer is updated.
3999          * This is only applicable for weak-ordered memory model archs such
4000          * as IA-64. The following barrier is also mandatory since FW will
4001          * assumes packets must have BDs.
4002          */
4003         wmb();
4004
4005         txdata->tx_db.data.prod += nbd;
4006         barrier();
4007
4008         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4009
4010         mmiowb();
4011
4012         txdata->tx_bd_prod += nbd;
4013
4014         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4015                 netif_tx_stop_queue(txq);
4016
4017                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4018                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4019                  * fp->bd_tx_cons */
4020                 smp_mb();
4021
4022                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4023                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4024                         netif_tx_wake_queue(txq);
4025         }
4026         txdata->tx_pkt++;
4027
4028         return NETDEV_TX_OK;
4029 }
4030
4031 /**
4032  * bnx2x_setup_tc - routine to configure net_device for multi tc
4033  *
4034  * @netdev: net device to configure
4035  * @tc: number of traffic classes to enable
4036  *
4037  * callback connected to the ndo_setup_tc function pointer
4038  */
4039 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4040 {
4041         int cos, prio, count, offset;
4042         struct bnx2x *bp = netdev_priv(dev);
4043
4044         /* setup tc must be called under rtnl lock */
4045         ASSERT_RTNL();
4046
4047         /* no traffic classes requested. Aborting */
4048         if (!num_tc) {
4049                 netdev_reset_tc(dev);
4050                 return 0;
4051         }
4052
4053         /* requested to support too many traffic classes */
4054         if (num_tc > bp->max_cos) {
4055                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4056                           num_tc, bp->max_cos);
4057                 return -EINVAL;
4058         }
4059
4060         /* declare amount of supported traffic classes */
4061         if (netdev_set_num_tc(dev, num_tc)) {
4062                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4063                 return -EINVAL;
4064         }
4065
4066         /* configure priority to traffic class mapping */
4067         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4068                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4069                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4070                    "mapping priority %d to tc %d\n",
4071                    prio, bp->prio_to_cos[prio]);
4072         }
4073
4074         /* Use this configuration to differentiate tc0 from other COSes
4075            This can be used for ets or pfc, and save the effort of setting
4076            up a multio class queue disc or negotiating DCBX with a switch
4077         netdev_set_prio_tc_map(dev, 0, 0);
4078         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4079         for (prio = 1; prio < 16; prio++) {
4080                 netdev_set_prio_tc_map(dev, prio, 1);
4081                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4082         } */
4083
4084         /* configure traffic class to transmission queue mapping */
4085         for (cos = 0; cos < bp->max_cos; cos++) {
4086                 count = BNX2X_NUM_ETH_QUEUES(bp);
4087                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4088                 netdev_set_tc_queue(dev, cos, count, offset);
4089                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4090                    "mapping tc %d to offset %d count %d\n",
4091                    cos, offset, count);
4092         }
4093
4094         return 0;
4095 }
4096
4097 /* called with rtnl_lock */
4098 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4099 {
4100         struct sockaddr *addr = p;
4101         struct bnx2x *bp = netdev_priv(dev);
4102         int rc = 0;
4103
4104         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4105                 BNX2X_ERR("Requested MAC address is not valid\n");
4106                 return -EINVAL;
4107         }
4108
4109         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4110             !is_zero_ether_addr(addr->sa_data)) {
4111                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4112                 return -EINVAL;
4113         }
4114
4115         if (netif_running(dev))  {
4116                 rc = bnx2x_set_eth_mac(bp, false);
4117                 if (rc)
4118                         return rc;
4119         }
4120
4121         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4122
4123         if (netif_running(dev))
4124                 rc = bnx2x_set_eth_mac(bp, true);
4125
4126         return rc;
4127 }
4128
4129 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4130 {
4131         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4132         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4133         u8 cos;
4134
4135         /* Common */
4136
4137         if (IS_FCOE_IDX(fp_index)) {
4138                 memset(sb, 0, sizeof(union host_hc_status_block));
4139                 fp->status_blk_mapping = 0;
4140         } else {
4141                 /* status blocks */
4142                 if (!CHIP_IS_E1x(bp))
4143                         BNX2X_PCI_FREE(sb->e2_sb,
4144                                        bnx2x_fp(bp, fp_index,
4145                                                 status_blk_mapping),
4146                                        sizeof(struct host_hc_status_block_e2));
4147                 else
4148                         BNX2X_PCI_FREE(sb->e1x_sb,
4149                                        bnx2x_fp(bp, fp_index,
4150                                                 status_blk_mapping),
4151                                        sizeof(struct host_hc_status_block_e1x));
4152         }
4153
4154         /* Rx */
4155         if (!skip_rx_queue(bp, fp_index)) {
4156                 bnx2x_free_rx_bds(fp);
4157
4158                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4159                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4160                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4161                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4162                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4163
4164                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4165                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4166                                sizeof(struct eth_fast_path_rx_cqe) *
4167                                NUM_RCQ_BD);
4168
4169                 /* SGE ring */
4170                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4171                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4172                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4173                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4174         }
4175
4176         /* Tx */
4177         if (!skip_tx_queue(bp, fp_index)) {
4178                 /* fastpath tx rings: tx_buf tx_desc */
4179                 for_each_cos_in_tx_queue(fp, cos) {
4180                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4181
4182                         DP(NETIF_MSG_IFDOWN,
4183                            "freeing tx memory of fp %d cos %d cid %d\n",
4184                            fp_index, cos, txdata->cid);
4185
4186                         BNX2X_FREE(txdata->tx_buf_ring);
4187                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4188                                 txdata->tx_desc_mapping,
4189                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4190                 }
4191         }
4192         /* end of fastpath */
4193 }
4194
4195 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4196 {
4197         int i;
4198         for_each_cnic_queue(bp, i)
4199                 bnx2x_free_fp_mem_at(bp, i);
4200 }
4201
4202 void bnx2x_free_fp_mem(struct bnx2x *bp)
4203 {
4204         int i;
4205         for_each_eth_queue(bp, i)
4206                 bnx2x_free_fp_mem_at(bp, i);
4207 }
4208
4209 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4210 {
4211         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4212         if (!CHIP_IS_E1x(bp)) {
4213                 bnx2x_fp(bp, index, sb_index_values) =
4214                         (__le16 *)status_blk.e2_sb->sb.index_values;
4215                 bnx2x_fp(bp, index, sb_running_index) =
4216                         (__le16 *)status_blk.e2_sb->sb.running_index;
4217         } else {
4218                 bnx2x_fp(bp, index, sb_index_values) =
4219                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4220                 bnx2x_fp(bp, index, sb_running_index) =
4221                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4222         }
4223 }
4224
4225 /* Returns the number of actually allocated BDs */
4226 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4227                               int rx_ring_size)
4228 {
4229         struct bnx2x *bp = fp->bp;
4230         u16 ring_prod, cqe_ring_prod;
4231         int i, failure_cnt = 0;
4232
4233         fp->rx_comp_cons = 0;
4234         cqe_ring_prod = ring_prod = 0;
4235
4236         /* This routine is called only during fo init so
4237          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4238          */
4239         for (i = 0; i < rx_ring_size; i++) {
4240                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4241                         failure_cnt++;
4242                         continue;
4243                 }
4244                 ring_prod = NEXT_RX_IDX(ring_prod);
4245                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4246                 WARN_ON(ring_prod <= (i - failure_cnt));
4247         }
4248
4249         if (failure_cnt)
4250                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4251                           i - failure_cnt, fp->index);
4252
4253         fp->rx_bd_prod = ring_prod;
4254         /* Limit the CQE producer by the CQE ring size */
4255         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4256                                cqe_ring_prod);
4257         fp->rx_pkt = fp->rx_calls = 0;
4258
4259         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4260
4261         return i - failure_cnt;
4262 }
4263
4264 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4265 {
4266         int i;
4267
4268         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4269                 struct eth_rx_cqe_next_page *nextpg;
4270
4271                 nextpg = (struct eth_rx_cqe_next_page *)
4272                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4273                 nextpg->addr_hi =
4274                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4275                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4276                 nextpg->addr_lo =
4277                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4278                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4279         }
4280 }
4281
4282 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4283 {
4284         union host_hc_status_block *sb;
4285         struct bnx2x_fastpath *fp = &bp->fp[index];
4286         int ring_size = 0;
4287         u8 cos;
4288         int rx_ring_size = 0;
4289
4290         if (!bp->rx_ring_size &&
4291             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4292                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4293                 bp->rx_ring_size = rx_ring_size;
4294         } else if (!bp->rx_ring_size) {
4295                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4296
4297                 if (CHIP_IS_E3(bp)) {
4298                         u32 cfg = SHMEM_RD(bp,
4299                                            dev_info.port_hw_config[BP_PORT(bp)].
4300                                            default_cfg);
4301
4302                         /* Decrease ring size for 1G functions */
4303                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4304                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4305                                 rx_ring_size /= 10;
4306                 }
4307
4308                 /* allocate at least number of buffers required by FW */
4309                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4310                                      MIN_RX_SIZE_TPA, rx_ring_size);
4311
4312                 bp->rx_ring_size = rx_ring_size;
4313         } else /* if rx_ring_size specified - use it */
4314                 rx_ring_size = bp->rx_ring_size;
4315
4316         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4317
4318         /* Common */
4319         sb = &bnx2x_fp(bp, index, status_blk);
4320
4321         if (!IS_FCOE_IDX(index)) {
4322                 /* status blocks */
4323                 if (!CHIP_IS_E1x(bp))
4324                         BNX2X_PCI_ALLOC(sb->e2_sb,
4325                                 &bnx2x_fp(bp, index, status_blk_mapping),
4326                                 sizeof(struct host_hc_status_block_e2));
4327                 else
4328                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4329                                 &bnx2x_fp(bp, index, status_blk_mapping),
4330                             sizeof(struct host_hc_status_block_e1x));
4331         }
4332
4333         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4334          * set shortcuts for it.
4335          */
4336         if (!IS_FCOE_IDX(index))
4337                 set_sb_shortcuts(bp, index);
4338
4339         /* Tx */
4340         if (!skip_tx_queue(bp, index)) {
4341                 /* fastpath tx rings: tx_buf tx_desc */
4342                 for_each_cos_in_tx_queue(fp, cos) {
4343                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4344
4345                         DP(NETIF_MSG_IFUP,
4346                            "allocating tx memory of fp %d cos %d\n",
4347                            index, cos);
4348
4349                         BNX2X_ALLOC(txdata->tx_buf_ring,
4350                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4351                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4352                                 &txdata->tx_desc_mapping,
4353                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4354                 }
4355         }
4356
4357         /* Rx */
4358         if (!skip_rx_queue(bp, index)) {
4359                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4360                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4361                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4362                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4363                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4364                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4365
4366                 /* Seed all CQEs by 1s */
4367                 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4368                                  &bnx2x_fp(bp, index, rx_comp_mapping),
4369                                  sizeof(struct eth_fast_path_rx_cqe) *
4370                                  NUM_RCQ_BD);
4371
4372                 /* SGE ring */
4373                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4374                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4375                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4376                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4377                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4378                 /* RX BD ring */
4379                 bnx2x_set_next_page_rx_bd(fp);
4380
4381                 /* CQ ring */
4382                 bnx2x_set_next_page_rx_cq(fp);
4383
4384                 /* BDs */
4385                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4386                 if (ring_size < rx_ring_size)
4387                         goto alloc_mem_err;
4388         }
4389
4390         return 0;
4391
4392 /* handles low memory cases */
4393 alloc_mem_err:
4394         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4395                                                 index, ring_size);
4396         /* FW will drop all packets if queue is not big enough,
4397          * In these cases we disable the queue
4398          * Min size is different for OOO, TPA and non-TPA queues
4399          */
4400         if (ring_size < (fp->disable_tpa ?
4401                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4402                         /* release memory allocated for this queue */
4403                         bnx2x_free_fp_mem_at(bp, index);
4404                         return -ENOMEM;
4405         }
4406         return 0;
4407 }
4408
4409 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4410 {
4411         if (!NO_FCOE(bp))
4412                 /* FCoE */
4413                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4414                         /* we will fail load process instead of mark
4415                          * NO_FCOE_FLAG
4416                          */
4417                         return -ENOMEM;
4418
4419         return 0;
4420 }
4421
4422 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4423 {
4424         int i;
4425
4426         /* 1. Allocate FP for leading - fatal if error
4427          * 2. Allocate RSS - fix number of queues if error
4428          */
4429
4430         /* leading */
4431         if (bnx2x_alloc_fp_mem_at(bp, 0))
4432                 return -ENOMEM;
4433
4434         /* RSS */
4435         for_each_nondefault_eth_queue(bp, i)
4436                 if (bnx2x_alloc_fp_mem_at(bp, i))
4437                         break;
4438
4439         /* handle memory failures */
4440         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4441                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4442
4443                 WARN_ON(delta < 0);
4444                 bnx2x_shrink_eth_fp(bp, delta);
4445                 if (CNIC_SUPPORT(bp))
4446                         /* move non eth FPs next to last eth FP
4447                          * must be done in that order
4448                          * FCOE_IDX < FWD_IDX < OOO_IDX
4449                          */
4450
4451                         /* move FCoE fp even NO_FCOE_FLAG is on */
4452                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4453                 bp->num_ethernet_queues -= delta;
4454                 bp->num_queues = bp->num_ethernet_queues +
4455                                  bp->num_cnic_queues;
4456                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4457                           bp->num_queues + delta, bp->num_queues);
4458         }
4459
4460         return 0;
4461 }
4462
4463 void bnx2x_free_mem_bp(struct bnx2x *bp)
4464 {
4465         int i;
4466
4467         for (i = 0; i < bp->fp_array_size; i++)
4468                 kfree(bp->fp[i].tpa_info);
4469         kfree(bp->fp);
4470         kfree(bp->sp_objs);
4471         kfree(bp->fp_stats);
4472         kfree(bp->bnx2x_txq);
4473         kfree(bp->msix_table);
4474         kfree(bp->ilt);
4475 }
4476
4477 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4478 {
4479         struct bnx2x_fastpath *fp;
4480         struct msix_entry *tbl;
4481         struct bnx2x_ilt *ilt;
4482         int msix_table_size = 0;
4483         int fp_array_size, txq_array_size;
4484         int i;
4485
4486         /*
4487          * The biggest MSI-X table we might need is as a maximum number of fast
4488          * path IGU SBs plus default SB (for PF only).
4489          */
4490         msix_table_size = bp->igu_sb_cnt;
4491         if (IS_PF(bp))
4492                 msix_table_size++;
4493         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4494
4495         /* fp array: RSS plus CNIC related L2 queues */
4496         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4497         bp->fp_array_size = fp_array_size;
4498         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4499
4500         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4501         if (!fp)
4502                 goto alloc_err;
4503         for (i = 0; i < bp->fp_array_size; i++) {
4504                 fp[i].tpa_info =
4505                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4506                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4507                 if (!(fp[i].tpa_info))
4508                         goto alloc_err;
4509         }
4510
4511         bp->fp = fp;
4512
4513         /* allocate sp objs */
4514         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4515                               GFP_KERNEL);
4516         if (!bp->sp_objs)
4517                 goto alloc_err;
4518
4519         /* allocate fp_stats */
4520         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4521                                GFP_KERNEL);
4522         if (!bp->fp_stats)
4523                 goto alloc_err;
4524
4525         /* Allocate memory for the transmission queues array */
4526         txq_array_size =
4527                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4528         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4529
4530         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4531                                 GFP_KERNEL);
4532         if (!bp->bnx2x_txq)
4533                 goto alloc_err;
4534
4535         /* msix table */
4536         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4537         if (!tbl)
4538                 goto alloc_err;
4539         bp->msix_table = tbl;
4540
4541         /* ilt */
4542         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4543         if (!ilt)
4544                 goto alloc_err;
4545         bp->ilt = ilt;
4546
4547         return 0;
4548 alloc_err:
4549         bnx2x_free_mem_bp(bp);
4550         return -ENOMEM;
4551 }
4552
4553 int bnx2x_reload_if_running(struct net_device *dev)
4554 {
4555         struct bnx2x *bp = netdev_priv(dev);
4556
4557         if (unlikely(!netif_running(dev)))
4558                 return 0;
4559
4560         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4561         return bnx2x_nic_load(bp, LOAD_NORMAL);
4562 }
4563
4564 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4565 {
4566         u32 sel_phy_idx = 0;
4567         if (bp->link_params.num_phys <= 1)
4568                 return INT_PHY;
4569
4570         if (bp->link_vars.link_up) {
4571                 sel_phy_idx = EXT_PHY1;
4572                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4573                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4574                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4575                         sel_phy_idx = EXT_PHY2;
4576         } else {
4577
4578                 switch (bnx2x_phy_selection(&bp->link_params)) {
4579                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4580                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4581                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4582                        sel_phy_idx = EXT_PHY1;
4583                        break;
4584                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4585                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4586                        sel_phy_idx = EXT_PHY2;
4587                        break;
4588                 }
4589         }
4590
4591         return sel_phy_idx;
4592 }
4593 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4594 {
4595         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4596         /*
4597          * The selected activated PHY is always after swapping (in case PHY
4598          * swapping is enabled). So when swapping is enabled, we need to reverse
4599          * the configuration
4600          */
4601
4602         if (bp->link_params.multi_phy_config &
4603             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4604                 if (sel_phy_idx == EXT_PHY1)
4605                         sel_phy_idx = EXT_PHY2;
4606                 else if (sel_phy_idx == EXT_PHY2)
4607                         sel_phy_idx = EXT_PHY1;
4608         }
4609         return LINK_CONFIG_IDX(sel_phy_idx);
4610 }
4611
4612 #ifdef NETDEV_FCOE_WWNN
4613 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4614 {
4615         struct bnx2x *bp = netdev_priv(dev);
4616         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4617
4618         switch (type) {
4619         case NETDEV_FCOE_WWNN:
4620                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4621                                 cp->fcoe_wwn_node_name_lo);
4622                 break;
4623         case NETDEV_FCOE_WWPN:
4624                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4625                                 cp->fcoe_wwn_port_name_lo);
4626                 break;
4627         default:
4628                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4629                 return -EINVAL;
4630         }
4631
4632         return 0;
4633 }
4634 #endif
4635
4636 /* called with rtnl_lock */
4637 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4638 {
4639         struct bnx2x *bp = netdev_priv(dev);
4640
4641         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4642                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4643                 return -EAGAIN;
4644         }
4645
4646         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4647             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4648                 BNX2X_ERR("Can't support requested MTU size\n");
4649                 return -EINVAL;
4650         }
4651
4652         /* This does not race with packet allocation
4653          * because the actual alloc size is
4654          * only updated as part of load
4655          */
4656         dev->mtu = new_mtu;
4657
4658         return bnx2x_reload_if_running(dev);
4659 }
4660
4661 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4662                                      netdev_features_t features)
4663 {
4664         struct bnx2x *bp = netdev_priv(dev);
4665
4666         /* TPA requires Rx CSUM offloading */
4667         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4668                 features &= ~NETIF_F_LRO;
4669                 features &= ~NETIF_F_GRO;
4670         }
4671
4672         return features;
4673 }
4674
4675 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4676 {
4677         struct bnx2x *bp = netdev_priv(dev);
4678         u32 flags = bp->flags;
4679         u32 changes;
4680         bool bnx2x_reload = false;
4681
4682         if (features & NETIF_F_LRO)
4683                 flags |= TPA_ENABLE_FLAG;
4684         else
4685                 flags &= ~TPA_ENABLE_FLAG;
4686
4687         if (features & NETIF_F_GRO)
4688                 flags |= GRO_ENABLE_FLAG;
4689         else
4690                 flags &= ~GRO_ENABLE_FLAG;
4691
4692         if (features & NETIF_F_LOOPBACK) {
4693                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4694                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4695                         bnx2x_reload = true;
4696                 }
4697         } else {
4698                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4699                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4700                         bnx2x_reload = true;
4701                 }
4702         }
4703
4704         changes = flags ^ bp->flags;
4705
4706         /* if GRO is changed while LRO is enabled, don't force a reload */
4707         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4708                 changes &= ~GRO_ENABLE_FLAG;
4709
4710         if (changes)
4711                 bnx2x_reload = true;
4712
4713         bp->flags = flags;
4714
4715         if (bnx2x_reload) {
4716                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4717                         return bnx2x_reload_if_running(dev);
4718                 /* else: bnx2x_nic_load() will be called at end of recovery */
4719         }
4720
4721         return 0;
4722 }
4723
4724 void bnx2x_tx_timeout(struct net_device *dev)
4725 {
4726         struct bnx2x *bp = netdev_priv(dev);
4727
4728 #ifdef BNX2X_STOP_ON_ERROR
4729         if (!bp->panic)
4730                 bnx2x_panic();
4731 #endif
4732
4733         smp_mb__before_clear_bit();
4734         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4735         smp_mb__after_clear_bit();
4736
4737         /* This allows the netif to be shutdown gracefully before resetting */
4738         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4739 }
4740
4741 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4742 {
4743         struct net_device *dev = pci_get_drvdata(pdev);
4744         struct bnx2x *bp;
4745
4746         if (!dev) {
4747                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4748                 return -ENODEV;
4749         }
4750         bp = netdev_priv(dev);
4751
4752         rtnl_lock();
4753
4754         pci_save_state(pdev);
4755
4756         if (!netif_running(dev)) {
4757                 rtnl_unlock();
4758                 return 0;
4759         }
4760
4761         netif_device_detach(dev);
4762
4763         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4764
4765         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4766
4767         rtnl_unlock();
4768
4769         return 0;
4770 }
4771
4772 int bnx2x_resume(struct pci_dev *pdev)
4773 {
4774         struct net_device *dev = pci_get_drvdata(pdev);
4775         struct bnx2x *bp;
4776         int rc;
4777
4778         if (!dev) {
4779                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4780                 return -ENODEV;
4781         }
4782         bp = netdev_priv(dev);
4783
4784         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4785                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4786                 return -EAGAIN;
4787         }
4788
4789         rtnl_lock();
4790
4791         pci_restore_state(pdev);
4792
4793         if (!netif_running(dev)) {
4794                 rtnl_unlock();
4795                 return 0;
4796         }
4797
4798         bnx2x_set_power_state(bp, PCI_D0);
4799         netif_device_attach(dev);
4800
4801         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4802
4803         rtnl_unlock();
4804
4805         return rc;
4806 }
4807
4808 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4809                               u32 cid)
4810 {
4811         if (!cxt) {
4812                 BNX2X_ERR("bad context pointer %p\n", cxt);
4813                 return;
4814         }
4815
4816         /* ustorm cxt validation */
4817         cxt->ustorm_ag_context.cdu_usage =
4818                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4819                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4820         /* xcontext validation */
4821         cxt->xstorm_ag_context.cdu_reserved =
4822                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4823                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4824 }
4825
4826 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4827                                     u8 fw_sb_id, u8 sb_index,
4828                                     u8 ticks)
4829 {
4830         u32 addr = BAR_CSTRORM_INTMEM +
4831                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4832         REG_WR8(bp, addr, ticks);
4833         DP(NETIF_MSG_IFUP,
4834            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4835            port, fw_sb_id, sb_index, ticks);
4836 }
4837
4838 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4839                                     u16 fw_sb_id, u8 sb_index,
4840                                     u8 disable)
4841 {
4842         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4843         u32 addr = BAR_CSTRORM_INTMEM +
4844                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4845         u8 flags = REG_RD8(bp, addr);
4846         /* clear and set */
4847         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4848         flags |= enable_flag;
4849         REG_WR8(bp, addr, flags);
4850         DP(NETIF_MSG_IFUP,
4851            "port %x fw_sb_id %d sb_index %d disable %d\n",
4852            port, fw_sb_id, sb_index, disable);
4853 }
4854
4855 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4856                                     u8 sb_index, u8 disable, u16 usec)
4857 {
4858         int port = BP_PORT(bp);
4859         u8 ticks = usec / BNX2X_BTR;
4860
4861         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4862
4863         disable = disable ? 1 : (usec ? 0 : 1);
4864         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4865 }