drivers/net/ethernet/intel/i40e/i40e_txrx.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include "i40e.h"
  28 #include "i40e_prototype.h"
  29
  30 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
  31                                 u32 td_tag)
  32 {
  33         return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
  34                            ((u64)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
  35                            ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
  36                            ((u64)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
  37                            ((u64)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
  38 }
  39
  40 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
  41 /**
  42  * i40e_program_fdir_filter - Program a Flow Director filter
  43  * @fdir_data: Packet data that will be filter parameters
  44  * @raw_packet: the pre-allocated packet buffer for FDir
  45  * @pf: The pf pointer
  46  * @add: True for add/update, False for remove
  47  **/
  48 int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
  49                              struct i40e_pf *pf, bool add)
  50 {
  51         struct i40e_filter_program_desc *fdir_desc;
  52         struct i40e_tx_buffer *tx_buf;
  53         struct i40e_tx_desc *tx_desc;
  54         struct i40e_ring *tx_ring;
  55         unsigned int fpt, dcc;
  56         struct i40e_vsi *vsi;
  57         struct device *dev;
  58         dma_addr_t dma;
  59         u32 td_cmd = 0;
  60         u16 i;
  61
  62         /* find existing FDIR VSI */
  63         vsi = NULL;
  64         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
  65                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
  66                         vsi = pf->vsi[i];
  67         if (!vsi)
  68                 return -ENOENT;
  69
  70         tx_ring = vsi->tx_rings[0];
  71         dev = tx_ring->dev;
  72
  73         dma = dma_map_single(dev, raw_packet,
  74                              I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
  75         if (dma_mapping_error(dev, dma))
  76                 goto dma_fail;
  77
  78         /* grab the next descriptor */
  79         i = tx_ring->next_to_use;
  80         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
  81
  82         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
  83
  84         fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
  85               I40E_TXD_FLTR_QW0_QINDEX_MASK;
  86
  87         fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
  88                I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
  89
  90         fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
  91                I40E_TXD_FLTR_QW0_PCTYPE_MASK;
  92
  93         /* Use LAN VSI Id if not programmed by user */
  94         if (fdir_data->dest_vsi == 0)
  95                 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
  96                        I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
  97         else
  98                 fpt |= ((u32)fdir_data->dest_vsi <<
  99                         I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
 100                        I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
 101
 102         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
 103
 104         dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
 105
 106         if (add)
 107                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
 108                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 109         else
 110                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
 111                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 112
 113         dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
 114                I40E_TXD_FLTR_QW1_DEST_MASK;
 115
 116         dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
 117                I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
 118
 119         if (fdir_data->cnt_index != 0) {
 120                 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
 121                 dcc |= ((u32)fdir_data->cnt_index <<
 122                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
 123                        I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
 124         }
 125
 126         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
 127         fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
 128
 129         /* Now program a dummy descriptor */
 130         i = tx_ring->next_to_use;
 131         tx_desc = I40E_TX_DESC(tx_ring, i);
 132         tx_buf = &tx_ring->tx_bi[i];
 133
 134         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
 135
 136         /* record length, and DMA address */
 137         dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
 138         dma_unmap_addr_set(tx_buf, dma, dma);
 139
 140         tx_desc->buffer_addr = cpu_to_le64(dma);
 141         td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
 142
 143         tx_desc->cmd_type_offset_bsz =
 144                 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
 145
 146         /* set the timestamp */
 147         tx_buf->time_stamp = jiffies;
 148
 149         /* Force memory writes to complete before letting h/w
 150          * know there are new descriptors to fetch.  (Only
 151          * applicable for weak-ordered memory model archs,
 152          * such as IA-64).
 153          */
 154         wmb();
 155
 156         /* Mark the data descriptor to be watched */
 157         tx_buf->next_to_watch = tx_desc;
 158
 159         writel(tx_ring->next_to_use, tx_ring->tail);
 160         return 0;
 161
 162 dma_fail:
 163         return -1;
 164 }
 165
 166 #define IP_HEADER_OFFSET 14
 167 #define I40E_UDPIP_DUMMY_PACKET_LEN 42
 168 /**
 169  * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
 170  * @vsi: pointer to the targeted VSI
 171  * @fd_data: the flow director data required for the FDir descriptor
 172  * @raw_packet: the pre-allocated packet buffer for FDir
 173  * @add: true adds a filter, false removes it
 174  *
 175  * Returns 0 if the filters were successfully added or removed
 176  **/
 177 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
 178                                    struct i40e_fdir_filter *fd_data,
 179                                    u8 *raw_packet, bool add)
 180 {
 181         struct i40e_pf *pf = vsi->back;
 182         struct udphdr *udp;
 183         struct iphdr *ip;
 184         bool err = false;
 185         int ret;
 186         int i;
 187         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 188                 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
 189                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 190
 191         memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
 192
 193         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 194         udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
 195               + sizeof(struct iphdr));
 196
 197         ip->daddr = fd_data->dst_ip[0];
 198         udp->dest = fd_data->dst_port;
 199         ip->saddr = fd_data->src_ip[0];
 200         udp->source = fd_data->src_port;
 201
 202         for (i = I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP;
 203              i <= I40E_FILTER_PCTYPE_NONF_IPV4_UDP; i++) {
 204                 fd_data->pctype = i;
 205                 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 206
 207                 if (ret) {
 208                         dev_info(&pf->pdev->dev,
 209                                  "Filter command send failed for PCTYPE %d (ret = %d)\n",
 210                                  fd_data->pctype, ret);
 211                         err = true;
 212                 } else {
 213                         dev_info(&pf->pdev->dev,
 214                                  "Filter OK for PCTYPE %d (ret = %d)\n",
 215                                  fd_data->pctype, ret);
 216                 }
 217         }
 218
 219         return err ? -EOPNOTSUPP : 0;
 220 }
 221
 222 #define I40E_TCPIP_DUMMY_PACKET_LEN 54
 223 /**
 224  * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
 225  * @vsi: pointer to the targeted VSI
 226  * @fd_data: the flow director data required for the FDir descriptor
 227  * @raw_packet: the pre-allocated packet buffer for FDir
 228  * @add: true adds a filter, false removes it
 229  *
 230  * Returns 0 if the filters were successfully added or removed
 231  **/
 232 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
 233                                    struct i40e_fdir_filter *fd_data,
 234                                    u8 *raw_packet, bool add)
 235 {
 236         struct i40e_pf *pf = vsi->back;
 237         struct tcphdr *tcp;
 238         struct iphdr *ip;
 239         bool err = false;
 240         int ret;
 241         /* Dummy packet */
 242         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 243                 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
 244                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
 245                 0x0, 0x72, 0, 0, 0, 0};
 246
 247         memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
 248
 249         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 250         tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
 251               + sizeof(struct iphdr));
 252
 253         ip->daddr = fd_data->dst_ip[0];
 254         tcp->dest = fd_data->dst_port;
 255         ip->saddr = fd_data->src_ip[0];
 256         tcp->source = fd_data->src_port;
 257
 258         if (add) {
 259                 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
 260                         dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
 261                         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 262                 }
 263         }
 264
 265         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN;
 266         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 267
 268         if (ret) {
 269                 dev_info(&pf->pdev->dev,
 270                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 271                          fd_data->pctype, ret);
 272                 err = true;
 273         } else {
 274                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 275                          fd_data->pctype, ret);
 276         }
 277
 278         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
 279
 280         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 281         if (ret) {
 282                 dev_info(&pf->pdev->dev,
 283                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 284                          fd_data->pctype, ret);
 285                 err = true;
 286         } else {
 287                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 288                           fd_data->pctype, ret);
 289         }
 290
 291         return err ? -EOPNOTSUPP : 0;
 292 }
 293
 294 /**
 295  * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
 296  * a specific flow spec
 297  * @vsi: pointer to the targeted VSI
 298  * @fd_data: the flow director data required for the FDir descriptor
 299  * @raw_packet: the pre-allocated packet buffer for FDir
 300  * @add: true adds a filter, false removes it
 301  *
 302  * Always returns -EOPNOTSUPP
 303  **/
 304 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
 305                                     struct i40e_fdir_filter *fd_data,
 306                                     u8 *raw_packet, bool add)
 307 {
 308         return -EOPNOTSUPP;
 309 }
 310
 311 #define I40E_IP_DUMMY_PACKET_LEN 34
 312 /**
 313  * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
 314  * a specific flow spec
 315  * @vsi: pointer to the targeted VSI
 316  * @fd_data: the flow director data required for the FDir descriptor
 317  * @raw_packet: the pre-allocated packet buffer for FDir
 318  * @add: true adds a filter, false removes it
 319  *
 320  * Returns 0 if the filters were successfully added or removed
 321  **/
 322 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
 323                                   struct i40e_fdir_filter *fd_data,
 324                                   u8 *raw_packet, bool add)
 325 {
 326         struct i40e_pf *pf = vsi->back;
 327         struct iphdr *ip;
 328         bool err = false;
 329         int ret;
 330         int i;
 331         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 332                 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
 333                 0, 0, 0, 0};
 334
 335         memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
 336         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 337
 338         ip->saddr = fd_data->src_ip[0];
 339         ip->daddr = fd_data->dst_ip[0];
 340         ip->protocol = 0;
 341
 342         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
 343              i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
 344                 fd_data->pctype = i;
 345                 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 346
 347                 if (ret) {
 348                         dev_info(&pf->pdev->dev,
 349                                  "Filter command send failed for PCTYPE %d (ret = %d)\n",
 350                                  fd_data->pctype, ret);
 351                         err = true;
 352                 } else {
 353                         dev_info(&pf->pdev->dev,
 354                                  "Filter OK for PCTYPE %d (ret = %d)\n",
 355                                  fd_data->pctype, ret);
 356                 }
 357         }
 358
 359         return err ? -EOPNOTSUPP : 0;
 360 }
 361
 362 /**
 363  * i40e_add_del_fdir - Build raw packets to add/del fdir filter
 364  * @vsi: pointer to the targeted VSI
 365  * @cmd: command to get or set RX flow classification rules
 366  * @add: true adds a filter, false removes it
 367  *
 368  **/
 369 int i40e_add_del_fdir(struct i40e_vsi *vsi,
 370                       struct i40e_fdir_filter *input, bool add)
 371 {
 372         struct i40e_pf *pf = vsi->back;
 373         u8 *raw_packet;
 374         int ret;
 375
 376         /* Populate the Flow Director that we have at the moment
 377          * and allocate the raw packet buffer for the calling functions
 378          */
 379         raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
 380         if (!raw_packet)
 381                 return -ENOMEM;
 382
 383         switch (input->flow_type & ~FLOW_EXT) {
 384         case TCP_V4_FLOW:
 385                 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
 386                                               add);
 387                 break;
 388         case UDP_V4_FLOW:
 389                 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
 390                                               add);
 391                 break;
 392         case SCTP_V4_FLOW:
 393                 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
 394                                                add);
 395                 break;
 396         case IPV4_FLOW:
 397                 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
 398                                              add);
 399                 break;
 400         case IP_USER_FLOW:
 401                 switch (input->ip4_proto) {
 402                 case IPPROTO_TCP:
 403                         ret = i40e_add_del_fdir_tcpv4(vsi, input,
 404                                                       raw_packet, add);
 405                         break;
 406                 case IPPROTO_UDP:
 407                         ret = i40e_add_del_fdir_udpv4(vsi, input,
 408                                                       raw_packet, add);
 409                         break;
 410                 case IPPROTO_SCTP:
 411                         ret = i40e_add_del_fdir_sctpv4(vsi, input,
 412                                                        raw_packet, add);
 413                         break;
 414                 default:
 415                         ret = i40e_add_del_fdir_ipv4(vsi, input,
 416                                                      raw_packet, add);
 417                         break;
 418                 }
 419                 break;
 420         default:
 421                 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
 422                          input->flow_type);
 423                 ret = -EINVAL;
 424         }
 425
 426         kfree(raw_packet);
 427         return ret;
 428 }
 429
 430 /**
 431  * i40e_fd_handle_status - check the Programming Status for FD
 432  * @rx_ring: the Rx ring for this descriptor
 433  * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
 434  * @prog_id: the id originally used for programming
 435  *
 436  * This is used to verify if the FD programming or invalidation
 437  * requested by SW to the HW is successful or not and take actions accordingly.
 438  **/
 439 static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
 440                                   union i40e_rx_desc *rx_desc, u8 prog_id)
 441 {
 442         struct i40e_pf *pf = rx_ring->vsi->back;
 443         struct pci_dev *pdev = pf->pdev;
 444         u32 fcnt_prog, fcnt_avail;
 445         u32 error;
 446         u64 qw;
 447
 448         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 449         error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
 450                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
 451
 452         if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
 453                 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
 454                          rx_desc->wb.qword0.hi_dword.fd_id);
 455
 456                 /* filter programming failed most likely due to table full */
 457                 fcnt_prog = i40e_get_current_fd_count(pf);
 458                 fcnt_avail = pf->hw.fdir_shared_filter_count +
 459                                                        pf->fdir_pf_filter_count;
 460
 461                 /* If ATR is running fcnt_prog can quickly change,
 462                  * if we are very close to full, it makes sense to disable
 463                  * FD ATR/SB and then re-enable it when there is room.
 464                  */
 465                 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
 466                         /* Turn off ATR first */
 467                         if (pf->flags | I40E_FLAG_FD_ATR_ENABLED) {
 468                                 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 469                                 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
 470                                 pf->auto_disable_flags |=
 471                                                        I40E_FLAG_FD_ATR_ENABLED;
 472                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 473                         } else if (pf->flags | I40E_FLAG_FD_SB_ENABLED) {
 474                                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
 475                                 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
 476                                 pf->auto_disable_flags |=
 477                                                         I40E_FLAG_FD_SB_ENABLED;
 478                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 479                         }
 480                 } else {
 481                         dev_info(&pdev->dev, "FD filter programming error\n");
 482                 }
 483         } else if (error ==
 484                           (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
 485                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
 486                         dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n",
 487                                  rx_desc->wb.qword0.hi_dword.fd_id);
 488         }
 489 }
 490
 491 /**
 492  * i40e_unmap_and_free_tx_resource - Release a Tx buffer
 493  * @ring:      the ring that owns the buffer
 494  * @tx_buffer: the buffer to free
 495  **/
 496 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 497                                             struct i40e_tx_buffer *tx_buffer)
 498 {
 499         if (tx_buffer->skb) {
 500                 dev_kfree_skb_any(tx_buffer->skb);
 501                 if (dma_unmap_len(tx_buffer, len))
 502                         dma_unmap_single(ring->dev,
 503                                          dma_unmap_addr(tx_buffer, dma),
 504                                          dma_unmap_len(tx_buffer, len),
 505                                          DMA_TO_DEVICE);
 506         } else if (dma_unmap_len(tx_buffer, len)) {
 507                 dma_unmap_page(ring->dev,
 508                                dma_unmap_addr(tx_buffer, dma),
 509                                dma_unmap_len(tx_buffer, len),
 510                                DMA_TO_DEVICE);
 511         }
 512         tx_buffer->next_to_watch = NULL;
 513         tx_buffer->skb = NULL;
 514         dma_unmap_len_set(tx_buffer, len, 0);
 515         /* tx_buffer must be completely set up in the transmit path */
 516 }
 517
 518 /**
 519  * i40e_clean_tx_ring - Free any empty Tx buffers
 520  * @tx_ring: ring to be cleaned
 521  **/
 522 void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
 523 {
 524         unsigned long bi_size;
 525         u16 i;
 526
 527         /* ring already cleared, nothing to do */
 528         if (!tx_ring->tx_bi)
 529                 return;
 530
 531         /* Free all the Tx ring sk_buffs */
 532         for (i = 0; i < tx_ring->count; i++)
 533                 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
 534
 535         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 536         memset(tx_ring->tx_bi, 0, bi_size);
 537
 538         /* Zero out the descriptor ring */
 539         memset(tx_ring->desc, 0, tx_ring->size);
 540
 541         tx_ring->next_to_use = 0;
 542         tx_ring->next_to_clean = 0;
 543
 544         if (!tx_ring->netdev)
 545                 return;
 546
 547         /* cleanup Tx queue statistics */
 548         netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
 549                                                   tx_ring->queue_index));
 550 }
 551
 552 /**
 553  * i40e_free_tx_resources - Free Tx resources per queue
 554  * @tx_ring: Tx descriptor ring for a specific queue
 555  *
 556  * Free all transmit software resources
 557  **/
 558 void i40e_free_tx_resources(struct i40e_ring *tx_ring)
 559 {
 560         i40e_clean_tx_ring(tx_ring);
 561         kfree(tx_ring->tx_bi);
 562         tx_ring->tx_bi = NULL;
 563
 564         if (tx_ring->desc) {
 565                 dma_free_coherent(tx_ring->dev, tx_ring->size,
 566                                   tx_ring->desc, tx_ring->dma);
 567                 tx_ring->desc = NULL;
 568         }
 569 }
 570
 571 /**
 572  * i40e_get_tx_pending - how many tx descriptors not processed
 573  * @tx_ring: the ring of descriptors
 574  *
 575  * Since there is no access to the ring head register
 576  * in XL710, we need to use our local copies
 577  **/
 578 static u32 i40e_get_tx_pending(struct i40e_ring *ring)
 579 {
 580         u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
 581                         ? ring->next_to_use
 582                         : ring->next_to_use + ring->count);
 583         return ntu - ring->next_to_clean;
 584 }
 585
 586 /**
 587  * i40e_check_tx_hang - Is there a hang in the Tx queue
 588  * @tx_ring: the ring of descriptors
 589  **/
 590 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
 591 {
 592         u32 tx_pending = i40e_get_tx_pending(tx_ring);
 593         bool ret = false;
 594
 595         clear_check_for_tx_hang(tx_ring);
 596
 597         /* Check for a hung queue, but be thorough. This verifies
 598          * that a transmit has been completed since the previous
 599          * check AND there is at least one packet pending. The
 600          * ARMED bit is set to indicate a potential hang. The
 601          * bit is cleared if a pause frame is received to remove
 602          * false hang detection due to PFC or 802.3x frames. By
 603          * requiring this to fail twice we avoid races with
 604          * PFC clearing the ARMED bit and conditions where we
 605          * run the check_tx_hang logic with a transmit completion
 606          * pending but without time to complete it yet.
 607          */
 608         if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
 609             tx_pending) {
 610                 /* make sure it is true for two checks in a row */
 611                 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
 612                                        &tx_ring->state);
 613         } else {
 614                 /* update completed stats and disarm the hang check */
 615                 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
 616                 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
 617         }
 618
 619         return ret;
 620 }
 621
 622 /**
 623  * i40e_get_head - Retrieve head from head writeback
 624  * @tx_ring:  tx ring to fetch head of
 625  *
 626  * Returns value of Tx ring head based on value stored
 627  * in head write-back location
 628  **/
 629 static inline u32 i40e_get_head(struct i40e_ring *tx_ring)
 630 {
 631         void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count;
 632
 633         return le32_to_cpu(*(volatile __le32 *)head);
 634 }
 635
 636 /**
 637  * i40e_clean_tx_irq - Reclaim resources after transmit completes
 638  * @tx_ring:  tx ring to clean
 639  * @budget:   how many cleans we're allowed
 640  *
 641  * Returns true if there's any budget left (e.g. the clean is finished)
 642  **/
 643 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
 644 {
 645         u16 i = tx_ring->next_to_clean;
 646         struct i40e_tx_buffer *tx_buf;
 647         struct i40e_tx_desc *tx_head;
 648         struct i40e_tx_desc *tx_desc;
 649         unsigned int total_packets = 0;
 650         unsigned int total_bytes = 0;
 651
 652         tx_buf = &tx_ring->tx_bi[i];
 653         tx_desc = I40E_TX_DESC(tx_ring, i);
 654         i -= tx_ring->count;
 655
 656         tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
 657
 658         do {
 659                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
 660
 661                 /* if next_to_watch is not set then there is no work pending */
 662                 if (!eop_desc)
 663                         break;
 664
 665                 /* prevent any other reads prior to eop_desc */
 666                 read_barrier_depends();
 667
 668                 /* we have caught up to head, no work left to do */
 669                 if (tx_head == tx_desc)
 670                         break;
 671
 672                 /* clear next_to_watch to prevent false hangs */
 673                 tx_buf->next_to_watch = NULL;
 674
 675                 /* update the statistics for this packet */
 676                 total_bytes += tx_buf->bytecount;
 677                 total_packets += tx_buf->gso_segs;
 678
 679                 /* free the skb */
 680                 dev_kfree_skb_any(tx_buf->skb);
 681
 682                 /* unmap skb header data */
 683                 dma_unmap_single(tx_ring->dev,
 684                                  dma_unmap_addr(tx_buf, dma),
 685                                  dma_unmap_len(tx_buf, len),
 686                                  DMA_TO_DEVICE);
 687
 688                 /* clear tx_buffer data */
 689                 tx_buf->skb = NULL;
 690                 dma_unmap_len_set(tx_buf, len, 0);
 691
 692                 /* unmap remaining buffers */
 693                 while (tx_desc != eop_desc) {
 694
 695                         tx_buf++;
 696                         tx_desc++;
 697                         i++;
 698                         if (unlikely(!i)) {
 699                                 i -= tx_ring->count;
 700                                 tx_buf = tx_ring->tx_bi;
 701                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
 702                         }
 703
 704                         /* unmap any remaining paged data */
 705                         if (dma_unmap_len(tx_buf, len)) {
 706                                 dma_unmap_page(tx_ring->dev,
 707                                                dma_unmap_addr(tx_buf, dma),
 708                                                dma_unmap_len(tx_buf, len),
 709                                                DMA_TO_DEVICE);
 710                                 dma_unmap_len_set(tx_buf, len, 0);
 711                         }
 712                 }
 713
 714                 /* move us one more past the eop_desc for start of next pkt */
 715                 tx_buf++;
 716                 tx_desc++;
 717                 i++;
 718                 if (unlikely(!i)) {
 719                         i -= tx_ring->count;
 720                         tx_buf = tx_ring->tx_bi;
 721                         tx_desc = I40E_TX_DESC(tx_ring, 0);
 722                 }
 723
 724                 /* update budget accounting */
 725                 budget--;
 726         } while (likely(budget));
 727
 728         i += tx_ring->count;
 729         tx_ring->next_to_clean = i;
 730         u64_stats_update_begin(&tx_ring->syncp);
 731         tx_ring->stats.bytes += total_bytes;
 732         tx_ring->stats.packets += total_packets;
 733         u64_stats_update_end(&tx_ring->syncp);
 734         tx_ring->q_vector->tx.total_bytes += total_bytes;
 735         tx_ring->q_vector->tx.total_packets += total_packets;
 736
 737         if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
 738                 /* schedule immediate reset if we believe we hung */
 739                 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
 740                          "  VSI                  <%d>\n"
 741                          "  Tx Queue             <%d>\n"
 742                          "  next_to_use          <%x>\n"
 743                          "  next_to_clean        <%x>\n",
 744                          tx_ring->vsi->seid,
 745                          tx_ring->queue_index,
 746                          tx_ring->next_to_use, i);
 747                 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
 748                          "  time_stamp           <%lx>\n"
 749                          "  jiffies              <%lx>\n",
 750                          tx_ring->tx_bi[i].time_stamp, jiffies);
 751
 752                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 753
 754                 dev_info(tx_ring->dev,
 755                          "tx hang detected on queue %d, resetting adapter\n",
 756                          tx_ring->queue_index);
 757
 758                 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
 759
 760                 /* the adapter is about to reset, no point in enabling stuff */
 761                 return true;
 762         }
 763
 764         netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
 765                                                       tx_ring->queue_index),
 766                                   total_packets, total_bytes);
 767
 768 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 769         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 770                      (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 771                 /* Make sure that anybody stopping the queue after this
 772                  * sees the new next_to_clean.
 773                  */
 774                 smp_mb();
 775                 if (__netif_subqueue_stopped(tx_ring->netdev,
 776                                              tx_ring->queue_index) &&
 777                    !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
 778                         netif_wake_subqueue(tx_ring->netdev,
 779                                             tx_ring->queue_index);
 780                         ++tx_ring->tx_stats.restart_queue;
 781                 }
 782         }
 783
 784         return budget > 0;
 785 }
 786
 787 /**
 788  * i40e_set_new_dynamic_itr - Find new ITR level
 789  * @rc: structure containing ring performance data
 790  *
 791  * Stores a new ITR value based on packets and byte counts during
 792  * the last interrupt.  The advantage of per interrupt computation
 793  * is faster updates and more accurate ITR for the current traffic
 794  * pattern.  Constants in this function were computed based on
 795  * theoretical maximum wire speed and thresholds were set based on
 796  * testing data as well as attempting to minimize response time
 797  * while increasing bulk throughput.
 798  **/
 799 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 800 {
 801         enum i40e_latency_range new_latency_range = rc->latency_range;
 802         u32 new_itr = rc->itr;
 803         int bytes_per_int;
 804
 805         if (rc->total_packets == 0 || !rc->itr)
 806                 return;
 807
 808         /* simple throttlerate management
 809          *   0-10MB/s   lowest (100000 ints/s)
 810          *  10-20MB/s   low    (20000 ints/s)
 811          *  20-1249MB/s bulk   (8000 ints/s)
 812          */
 813         bytes_per_int = rc->total_bytes / rc->itr;
 814         switch (rc->itr) {
 815         case I40E_LOWEST_LATENCY:
 816                 if (bytes_per_int > 10)
 817                         new_latency_range = I40E_LOW_LATENCY;
 818                 break;
 819         case I40E_LOW_LATENCY:
 820                 if (bytes_per_int > 20)
 821                         new_latency_range = I40E_BULK_LATENCY;
 822                 else if (bytes_per_int <= 10)
 823                         new_latency_range = I40E_LOWEST_LATENCY;
 824                 break;
 825         case I40E_BULK_LATENCY:
 826                 if (bytes_per_int <= 20)
 827                         rc->latency_range = I40E_LOW_LATENCY;
 828                 break;
 829         }
 830
 831         switch (new_latency_range) {
 832         case I40E_LOWEST_LATENCY:
 833                 new_itr = I40E_ITR_100K;
 834                 break;
 835         case I40E_LOW_LATENCY:
 836                 new_itr = I40E_ITR_20K;
 837                 break;
 838         case I40E_BULK_LATENCY:
 839                 new_itr = I40E_ITR_8K;
 840                 break;
 841         default:
 842                 break;
 843         }
 844
 845         if (new_itr != rc->itr) {
 846                 /* do an exponential smoothing */
 847                 new_itr = (10 * new_itr * rc->itr) /
 848                           ((9 * new_itr) + rc->itr);
 849                 rc->itr = new_itr & I40E_MAX_ITR;
 850         }
 851
 852         rc->total_bytes = 0;
 853         rc->total_packets = 0;
 854 }
 855
 856 /**
 857  * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
 858  * @q_vector: the vector to adjust
 859  **/
 860 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
 861 {
 862         u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
 863         struct i40e_hw *hw = &q_vector->vsi->back->hw;
 864         u32 reg_addr;
 865         u16 old_itr;
 866
 867         reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
 868         old_itr = q_vector->rx.itr;
 869         i40e_set_new_dynamic_itr(&q_vector->rx);
 870         if (old_itr != q_vector->rx.itr)
 871                 wr32(hw, reg_addr, q_vector->rx.itr);
 872
 873         reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
 874         old_itr = q_vector->tx.itr;
 875         i40e_set_new_dynamic_itr(&q_vector->tx);
 876         if (old_itr != q_vector->tx.itr)
 877                 wr32(hw, reg_addr, q_vector->tx.itr);
 878 }
 879
 880 /**
 881  * i40e_clean_programming_status - clean the programming status descriptor
 882  * @rx_ring: the rx ring that has this descriptor
 883  * @rx_desc: the rx descriptor written back by HW
 884  *
 885  * Flow director should handle FD_FILTER_STATUS to check its filter programming
 886  * status being successful or not and take actions accordingly. FCoE should
 887  * handle its context/filter programming/invalidation status and take actions.
 888  *
 889  **/
 890 static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
 891                                           union i40e_rx_desc *rx_desc)
 892 {
 893         u64 qw;
 894         u8 id;
 895
 896         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 897         id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
 898                   I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
 899
 900         if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
 901                 i40e_fd_handle_status(rx_ring, rx_desc, id);
 902 }
 903
 904 /**
 905  * i40e_setup_tx_descriptors - Allocate the Tx descriptors
 906  * @tx_ring: the tx ring to set up
 907  *
 908  * Return 0 on success, negative on error
 909  **/
 910 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
 911 {
 912         struct device *dev = tx_ring->dev;
 913         int bi_size;
 914
 915         if (!dev)
 916                 return -ENOMEM;
 917
 918         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 919         tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
 920         if (!tx_ring->tx_bi)
 921                 goto err;
 922
 923         /* round up to nearest 4K */
 924         tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
 925         /* add u32 for head writeback, align after this takes care of
 926          * guaranteeing this is at least one cache line in size
 927          */
 928         tx_ring->size += sizeof(u32);
 929         tx_ring->size = ALIGN(tx_ring->size, 4096);
 930         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 931                                            &tx_ring->dma, GFP_KERNEL);
 932         if (!tx_ring->desc) {
 933                 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
 934                          tx_ring->size);
 935                 goto err;
 936         }
 937
 938         tx_ring->next_to_use = 0;
 939         tx_ring->next_to_clean = 0;
 940         return 0;
 941
 942 err:
 943         kfree(tx_ring->tx_bi);
 944         tx_ring->tx_bi = NULL;
 945         return -ENOMEM;
 946 }
 947
 948 /**
 949  * i40e_clean_rx_ring - Free Rx buffers
 950  * @rx_ring: ring to be cleaned
 951  **/
 952 void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
 953 {
 954         struct device *dev = rx_ring->dev;
 955         struct i40e_rx_buffer *rx_bi;
 956         unsigned long bi_size;
 957         u16 i;
 958
 959         /* ring already cleared, nothing to do */
 960         if (!rx_ring->rx_bi)
 961                 return;
 962
 963         /* Free all the Rx ring sk_buffs */
 964         for (i = 0; i < rx_ring->count; i++) {
 965                 rx_bi = &rx_ring->rx_bi[i];
 966                 if (rx_bi->dma) {
 967                         dma_unmap_single(dev,
 968                                          rx_bi->dma,
 969                                          rx_ring->rx_buf_len,
 970                                          DMA_FROM_DEVICE);
 971                         rx_bi->dma = 0;
 972                 }
 973                 if (rx_bi->skb) {
 974                         dev_kfree_skb(rx_bi->skb);
 975                         rx_bi->skb = NULL;
 976                 }
 977                 if (rx_bi->page) {
 978                         if (rx_bi->page_dma) {
 979                                 dma_unmap_page(dev,
 980                                                rx_bi->page_dma,
 981                                                PAGE_SIZE / 2,
 982                                                DMA_FROM_DEVICE);
 983                                 rx_bi->page_dma = 0;
 984                         }
 985                         __free_page(rx_bi->page);
 986                         rx_bi->page = NULL;
 987                         rx_bi->page_offset = 0;
 988                 }
 989         }
 990
 991         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
 992         memset(rx_ring->rx_bi, 0, bi_size);
 993
 994         /* Zero out the descriptor ring */
 995         memset(rx_ring->desc, 0, rx_ring->size);
 996
 997         rx_ring->next_to_clean = 0;
 998         rx_ring->next_to_use = 0;
 999 }
1000
1001 /**
1002  * i40e_free_rx_resources - Free Rx resources
1003  * @rx_ring: ring to clean the resources from
1004  *
1005  * Free all receive software resources
1006  **/
1007 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1008 {
1009         i40e_clean_rx_ring(rx_ring);
1010         kfree(rx_ring->rx_bi);
1011         rx_ring->rx_bi = NULL;
1012
1013         if (rx_ring->desc) {
1014                 dma_free_coherent(rx_ring->dev, rx_ring->size,
1015                                   rx_ring->desc, rx_ring->dma);
1016                 rx_ring->desc = NULL;
1017         }
1018 }
1019
1020 /**
1021  * i40e_setup_rx_descriptors - Allocate Rx descriptors
1022  * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1023  *
1024  * Returns 0 on success, negative on failure
1025  **/
1026 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1027 {
1028         struct device *dev = rx_ring->dev;
1029         int bi_size;
1030
1031         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1032         rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1033         if (!rx_ring->rx_bi)
1034                 goto err;
1035
1036         /* Round up to nearest 4K */
1037         rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1038                 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1039                 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1040         rx_ring->size = ALIGN(rx_ring->size, 4096);
1041         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1042                                            &rx_ring->dma, GFP_KERNEL);
1043
1044         if (!rx_ring->desc) {
1045                 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1046                          rx_ring->size);
1047                 goto err;
1048         }
1049
1050         rx_ring->next_to_clean = 0;
1051         rx_ring->next_to_use = 0;
1052
1053         return 0;
1054 err:
1055         kfree(rx_ring->rx_bi);
1056         rx_ring->rx_bi = NULL;
1057         return -ENOMEM;
1058 }
1059
1060 /**
1061  * i40e_release_rx_desc - Store the new tail and head values
1062  * @rx_ring: ring to bump
1063  * @val: new head index
1064  **/
1065 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1066 {
1067         rx_ring->next_to_use = val;
1068         /* Force memory writes to complete before letting h/w
1069          * know there are new descriptors to fetch.  (Only
1070          * applicable for weak-ordered memory model archs,
1071          * such as IA-64).
1072          */
1073         wmb();
1074         writel(val, rx_ring->tail);
1075 }
1076
1077 /**
1078  * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1079  * @rx_ring: ring to place buffers on
1080  * @cleaned_count: number of buffers to replace
1081  **/
1082 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1083 {
1084         u16 i = rx_ring->next_to_use;
1085         union i40e_rx_desc *rx_desc;
1086         struct i40e_rx_buffer *bi;
1087         struct sk_buff *skb;
1088
1089         /* do nothing if no valid netdev defined */
1090         if (!rx_ring->netdev || !cleaned_count)
1091                 return;
1092
1093         while (cleaned_count--) {
1094                 rx_desc = I40E_RX_DESC(rx_ring, i);
1095                 bi = &rx_ring->rx_bi[i];
1096                 skb = bi->skb;
1097
1098                 if (!skb) {
1099                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1100                                                         rx_ring->rx_buf_len);
1101                         if (!skb) {
1102                                 rx_ring->rx_stats.alloc_buff_failed++;
1103                                 goto no_buffers;
1104                         }
1105                         /* initialize queue mapping */
1106                         skb_record_rx_queue(skb, rx_ring->queue_index);
1107                         bi->skb = skb;
1108                 }
1109
1110                 if (!bi->dma) {
1111                         bi->dma = dma_map_single(rx_ring->dev,
1112                                                  skb->data,
1113                                                  rx_ring->rx_buf_len,
1114                                                  DMA_FROM_DEVICE);
1115                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1116                                 rx_ring->rx_stats.alloc_buff_failed++;
1117                                 bi->dma = 0;
1118                                 goto no_buffers;
1119                         }
1120                 }
1121
1122                 if (ring_is_ps_enabled(rx_ring)) {
1123                         if (!bi->page) {
1124                                 bi->page = alloc_page(GFP_ATOMIC);
1125                                 if (!bi->page) {
1126                                         rx_ring->rx_stats.alloc_page_failed++;
1127                                         goto no_buffers;
1128                                 }
1129                         }
1130
1131                         if (!bi->page_dma) {
1132                                 /* use a half page if we're re-using */
1133                                 bi->page_offset ^= PAGE_SIZE / 2;
1134                                 bi->page_dma = dma_map_page(rx_ring->dev,
1135                                                             bi->page,
1136                                                             bi->page_offset,
1137                                                             PAGE_SIZE / 2,
1138                                                             DMA_FROM_DEVICE);
1139                                 if (dma_mapping_error(rx_ring->dev,
1140                                                       bi->page_dma)) {
1141                                         rx_ring->rx_stats.alloc_page_failed++;
1142                                         bi->page_dma = 0;
1143                                         goto no_buffers;
1144                                 }
1145                         }
1146
1147                         /* Refresh the desc even if buffer_addrs didn't change
1148                          * because each write-back erases this info.
1149                          */
1150                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1151                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1152                 } else {
1153                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1154                         rx_desc->read.hdr_addr = 0;
1155                 }
1156                 i++;
1157                 if (i == rx_ring->count)
1158                         i = 0;
1159         }
1160
1161 no_buffers:
1162         if (rx_ring->next_to_use != i)
1163                 i40e_release_rx_desc(rx_ring, i);
1164 }
1165
1166 /**
1167  * i40e_receive_skb - Send a completed packet up the stack
1168  * @rx_ring:  rx ring in play
1169  * @skb: packet to send up
1170  * @vlan_tag: vlan tag for packet
1171  **/
1172 static void i40e_receive_skb(struct i40e_ring *rx_ring,
1173                              struct sk_buff *skb, u16 vlan_tag)
1174 {
1175         struct i40e_q_vector *q_vector = rx_ring->q_vector;
1176         struct i40e_vsi *vsi = rx_ring->vsi;
1177         u64 flags = vsi->back->flags;
1178
1179         if (vlan_tag & VLAN_VID_MASK)
1180                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1181
1182         if (flags & I40E_FLAG_IN_NETPOLL)
1183                 netif_rx(skb);
1184         else
1185                 napi_gro_receive(&q_vector->napi, skb);
1186 }
1187
1188 /**
1189  * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1190  * @vsi: the VSI we care about
1191  * @skb: skb currently being received and modified
1192  * @rx_status: status value of last descriptor in packet
1193  * @rx_error: error value of last descriptor in packet
1194  * @rx_ptype: ptype value of last descriptor in packet
1195  **/
1196 static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1197                                     struct sk_buff *skb,
1198                                     u32 rx_status,
1199                                     u32 rx_error,
1200                                     u16 rx_ptype)
1201 {
1202         bool ipv4_tunnel, ipv6_tunnel;
1203         __wsum rx_udp_csum;
1204         __sum16 csum;
1205         struct iphdr *iph;
1206
1207         ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1208                       (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1209         ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1210                       (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1211
1212         skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
1213         skb->ip_summed = CHECKSUM_NONE;
1214
1215         /* Rx csum enabled and ip headers found? */
1216         if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
1217               rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1218                 return;
1219
1220         /* likely incorrect csum if alternate IP extension headers found */
1221         if (rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1222                 return;
1223
1224         /* IP or L4 or outmost IP checksum error */
1225         if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1226                         (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) |
1227                         (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) {
1228                 vsi->back->hw_csum_rx_error++;
1229                 return;
1230         }
1231
1232         if (ipv4_tunnel &&
1233             !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1234                 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1235                  * it in the driver, hardware does not do it for us.
1236                  * Since L3L4P bit was set we assume a valid IHL value (>=5)
1237                  * so the total length of IPv4 header is IHL*4 bytes
1238                  */
1239                 skb->transport_header = skb->mac_header +
1240                                         sizeof(struct ethhdr) +
1241                                         (ip_hdr(skb)->ihl * 4);
1242
1243                 /* Add 4 bytes for VLAN tagged packets */
1244                 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1245                                           skb->protocol == htons(ETH_P_8021AD))
1246                                           ? VLAN_HLEN : 0;
1247
1248                 rx_udp_csum = udp_csum(skb);
1249                 iph = ip_hdr(skb);
1250                 csum = csum_tcpudp_magic(
1251                                 iph->saddr, iph->daddr,
1252                                 (skb->len - skb_transport_offset(skb)),
1253                                 IPPROTO_UDP, rx_udp_csum);
1254
1255                 if (udp_hdr(skb)->check != csum) {
1256                         vsi->back->hw_csum_rx_error++;
1257                         return;
1258                 }
1259         }
1260
1261         skb->ip_summed = CHECKSUM_UNNECESSARY;
1262 }
1263
1264 /**
1265  * i40e_rx_hash - returns the hash value from the Rx descriptor
1266  * @ring: descriptor ring
1267  * @rx_desc: specific descriptor
1268  **/
1269 static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1270                                union i40e_rx_desc *rx_desc)
1271 {
1272         const __le64 rss_mask =
1273                 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1274                             I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1275
1276         if ((ring->netdev->features & NETIF_F_RXHASH) &&
1277             (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1278                 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1279         else
1280                 return 0;
1281 }
1282
1283 /**
1284  * i40e_ptype_to_hash - get a hash type
1285  * @ptype: the ptype value from the descriptor
1286  *
1287  * Returns a hash type to be used by skb_set_hash
1288  **/
1289 static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1290 {
1291         struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1292
1293         if (!decoded.known)
1294                 return PKT_HASH_TYPE_NONE;
1295
1296         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1297             decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1298                 return PKT_HASH_TYPE_L4;
1299         else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1300                  decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1301                 return PKT_HASH_TYPE_L3;
1302         else
1303                 return PKT_HASH_TYPE_L2;
1304 }
1305
1306 /**
1307  * i40e_clean_rx_irq - Reclaim resources after receive completes
1308  * @rx_ring:  rx ring to clean
1309  * @budget:   how many cleans we're allowed
1310  *
1311  * Returns true if there's any budget left (e.g. the clean is finished)
1312  **/
1313 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1314 {
1315         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1316         u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1317         u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1318         const int current_node = numa_node_id();
1319         struct i40e_vsi *vsi = rx_ring->vsi;
1320         u16 i = rx_ring->next_to_clean;
1321         union i40e_rx_desc *rx_desc;
1322         u32 rx_error, rx_status;
1323         u8 rx_ptype;
1324         u64 qword;
1325
1326         if (budget <= 0)
1327                 return 0;
1328
1329         rx_desc = I40E_RX_DESC(rx_ring, i);
1330         qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1331         rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1332                     I40E_RXD_QW1_STATUS_SHIFT;
1333
1334         while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1335                 union i40e_rx_desc *next_rxd;
1336                 struct i40e_rx_buffer *rx_bi;
1337                 struct sk_buff *skb;
1338                 u16 vlan_tag;
1339                 if (i40e_rx_is_programming_status(qword)) {
1340                         i40e_clean_programming_status(rx_ring, rx_desc);
1341                         I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1342                         goto next_desc;
1343                 }
1344                 rx_bi = &rx_ring->rx_bi[i];
1345                 skb = rx_bi->skb;
1346                 prefetch(skb->data);
1347
1348                 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1349                                 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1350                 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1351                                 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1352                 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1353                          I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1354
1355                 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1356                            I40E_RXD_QW1_ERROR_SHIFT;
1357                 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1358                 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1359
1360                 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1361                            I40E_RXD_QW1_PTYPE_SHIFT;
1362                 rx_bi->skb = NULL;
1363
1364                 /* This memory barrier is needed to keep us from reading
1365                  * any other fields out of the rx_desc until we know the
1366                  * STATUS_DD bit is set
1367                  */
1368                 rmb();
1369
1370                 /* Get the header and possibly the whole packet
1371                  * If this is an skb from previous receive dma will be 0
1372                  */
1373                 if (rx_bi->dma) {
1374                         u16 len;
1375
1376                         if (rx_hbo)
1377                                 len = I40E_RX_HDR_SIZE;
1378                         else if (rx_sph)
1379                                 len = rx_header_len;
1380                         else if (rx_packet_len)
1381                                 len = rx_packet_len;   /* 1buf/no split found */
1382                         else
1383                                 len = rx_header_len;   /* split always mode */
1384
1385                         skb_put(skb, len);
1386                         dma_unmap_single(rx_ring->dev,
1387                                          rx_bi->dma,
1388                                          rx_ring->rx_buf_len,
1389                                          DMA_FROM_DEVICE);
1390                         rx_bi->dma = 0;
1391                 }
1392
1393                 /* Get the rest of the data if this was a header split */
1394                 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1395
1396                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1397                                            rx_bi->page,
1398                                            rx_bi->page_offset,
1399                                            rx_packet_len);
1400
1401                         skb->len += rx_packet_len;
1402                         skb->data_len += rx_packet_len;
1403                         skb->truesize += rx_packet_len;
1404
1405                         if ((page_count(rx_bi->page) == 1) &&
1406                             (page_to_nid(rx_bi->page) == current_node))
1407                                 get_page(rx_bi->page);
1408                         else
1409                                 rx_bi->page = NULL;
1410
1411                         dma_unmap_page(rx_ring->dev,
1412                                        rx_bi->page_dma,
1413                                        PAGE_SIZE / 2,
1414                                        DMA_FROM_DEVICE);
1415                         rx_bi->page_dma = 0;
1416                 }
1417                 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1418
1419                 if (unlikely(
1420                     !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1421                         struct i40e_rx_buffer *next_buffer;
1422
1423                         next_buffer = &rx_ring->rx_bi[i];
1424
1425                         if (ring_is_ps_enabled(rx_ring)) {
1426                                 rx_bi->skb = next_buffer->skb;
1427                                 rx_bi->dma = next_buffer->dma;
1428                                 next_buffer->skb = skb;
1429                                 next_buffer->dma = 0;
1430                         }
1431                         rx_ring->rx_stats.non_eop_descs++;
1432                         goto next_desc;
1433                 }
1434
1435                 /* ERR_MASK will only have valid bits if EOP set */
1436                 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1437                         dev_kfree_skb_any(skb);
1438                         goto next_desc;
1439                 }
1440
1441                 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1442                              i40e_ptype_to_hash(rx_ptype));
1443                 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1444                         i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1445                                            I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1446                                            I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1447                         rx_ring->last_rx_timestamp = jiffies;
1448                 }
1449
1450                 /* probably a little skewed due to removing CRC */
1451                 total_rx_bytes += skb->len;
1452                 total_rx_packets++;
1453
1454                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1455
1456                 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1457
1458                 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1459                          ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1460                          : 0;
1461                 i40e_receive_skb(rx_ring, skb, vlan_tag);
1462
1463                 rx_ring->netdev->last_rx = jiffies;
1464                 budget--;
1465 next_desc:
1466                 rx_desc->wb.qword1.status_error_len = 0;
1467                 if (!budget)
1468                         break;
1469
1470                 cleaned_count++;
1471                 /* return some buffers to hardware, one at a time is too slow */
1472                 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1473                         i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1474                         cleaned_count = 0;
1475                 }
1476
1477                 /* use prefetched values */
1478                 rx_desc = next_rxd;
1479                 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1480                 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1481                             I40E_RXD_QW1_STATUS_SHIFT;
1482         }
1483
1484         rx_ring->next_to_clean = i;
1485         u64_stats_update_begin(&rx_ring->syncp);
1486         rx_ring->stats.packets += total_rx_packets;
1487         rx_ring->stats.bytes += total_rx_bytes;
1488         u64_stats_update_end(&rx_ring->syncp);
1489         rx_ring->q_vector->rx.total_packets += total_rx_packets;
1490         rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1491
1492         if (cleaned_count)
1493                 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1494
1495         return budget > 0;
1496 }
1497
1498 /**
1499  * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1500  * @napi: napi struct with our devices info in it
1501  * @budget: amount of work driver is allowed to do this pass, in packets
1502  *
1503  * This function will clean all queues associated with a q_vector.
1504  *
1505  * Returns the amount of work done
1506  **/
1507 int i40e_napi_poll(struct napi_struct *napi, int budget)
1508 {
1509         struct i40e_q_vector *q_vector =
1510                                container_of(napi, struct i40e_q_vector, napi);
1511         struct i40e_vsi *vsi = q_vector->vsi;
1512         struct i40e_ring *ring;
1513         bool clean_complete = true;
1514         int budget_per_ring;
1515
1516         if (test_bit(__I40E_DOWN, &vsi->state)) {
1517                 napi_complete(napi);
1518                 return 0;
1519         }
1520
1521         /* Since the actual Tx work is minimal, we can give the Tx a larger
1522          * budget and be more aggressive about cleaning up the Tx descriptors.
1523          */
1524         i40e_for_each_ring(ring, q_vector->tx)
1525                 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1526
1527         /* We attempt to distribute budget to each Rx queue fairly, but don't
1528          * allow the budget to go below 1 because that would exit polling early.
1529          */
1530         budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1531
1532         i40e_for_each_ring(ring, q_vector->rx)
1533                 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
1534
1535         /* If work not completed, return budget and polling will return */
1536         if (!clean_complete)
1537                 return budget;
1538
1539         /* Work is done so exit the polling mode and re-enable the interrupt */
1540         napi_complete(napi);
1541         if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1542             ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1543                 i40e_update_dynamic_itr(q_vector);
1544
1545         if (!test_bit(__I40E_DOWN, &vsi->state)) {
1546                 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1547                         i40e_irq_dynamic_enable(vsi,
1548                                         q_vector->v_idx + vsi->base_vector);
1549                 } else {
1550                         struct i40e_hw *hw = &vsi->back->hw;
1551                         /* We re-enable the queue 0 cause, but
1552                          * don't worry about dynamic_enable
1553                          * because we left it on for the other
1554                          * possible interrupts during napi
1555                          */
1556                         u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1557                         qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1558                         wr32(hw, I40E_QINT_RQCTL(0), qval);
1559
1560                         qval = rd32(hw, I40E_QINT_TQCTL(0));
1561                         qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1562                         wr32(hw, I40E_QINT_TQCTL(0), qval);
1563
1564                         i40e_irq_dynamic_enable_icr0(vsi->back);
1565                 }
1566         }
1567
1568         return 0;
1569 }
1570
1571 /**
1572  * i40e_atr - Add a Flow Director ATR filter
1573  * @tx_ring:  ring to add programming descriptor to
1574  * @skb:      send buffer
1575  * @flags:    send flags
1576  * @protocol: wire protocol
1577  **/
1578 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1579                      u32 flags, __be16 protocol)
1580 {
1581         struct i40e_filter_program_desc *fdir_desc;
1582         struct i40e_pf *pf = tx_ring->vsi->back;
1583         union {
1584                 unsigned char *network;
1585                 struct iphdr *ipv4;
1586                 struct ipv6hdr *ipv6;
1587         } hdr;
1588         struct tcphdr *th;
1589         unsigned int hlen;
1590         u32 flex_ptype, dtype_cmd;
1591         u16 i;
1592
1593         /* make sure ATR is enabled */
1594         if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
1595                 return;
1596
1597         /* if sampling is disabled do nothing */
1598         if (!tx_ring->atr_sample_rate)
1599                 return;
1600
1601         /* snag network header to get L4 type and address */
1602         hdr.network = skb_network_header(skb);
1603
1604         /* Currently only IPv4/IPv6 with TCP is supported */
1605         if (protocol == htons(ETH_P_IP)) {
1606                 if (hdr.ipv4->protocol != IPPROTO_TCP)
1607                         return;
1608
1609                 /* access ihl as a u8 to avoid unaligned access on ia64 */
1610                 hlen = (hdr.network[0] & 0x0F) << 2;
1611         } else if (protocol == htons(ETH_P_IPV6)) {
1612                 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1613                         return;
1614
1615                 hlen = sizeof(struct ipv6hdr);
1616         } else {
1617                 return;
1618         }
1619
1620         th = (struct tcphdr *)(hdr.network + hlen);
1621
1622         /* Due to lack of space, no more new filters can be programmed */
1623         if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1624                 return;
1625
1626         tx_ring->atr_count++;
1627
1628         /* sample on all syn/fin/rst packets or once every atr sample rate */
1629         if (!th->fin &&
1630             !th->syn &&
1631             !th->rst &&
1632             (tx_ring->atr_count < tx_ring->atr_sample_rate))
1633                 return;
1634
1635         tx_ring->atr_count = 0;
1636
1637         /* grab the next descriptor */
1638         i = tx_ring->next_to_use;
1639         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1640
1641         i++;
1642         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1643
1644         flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1645                       I40E_TXD_FLTR_QW0_QINDEX_MASK;
1646         flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1647                       (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1648                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1649                       (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1650                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1651
1652         flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1653
1654         dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1655
1656         dtype_cmd |= (th->fin || th->rst) ?
1657                      (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1658                       I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1659                      (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1660                       I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1661
1662         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1663                      I40E_TXD_FLTR_QW1_DEST_SHIFT;
1664
1665         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1666                      I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1667
1668         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1669         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1670 }
1671
1672 /**
1673  * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1674  * @skb:     send buffer
1675  * @tx_ring: ring to send buffer on
1676  * @flags:   the tx flags to be set
1677  *
1678  * Checks the skb and set up correspondingly several generic transmit flags
1679  * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1680  *
1681  * Returns error code indicate the frame should be dropped upon error and the
1682  * otherwise  returns 0 to indicate the flags has been set properly.
1683  **/
1684 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1685                                       struct i40e_ring *tx_ring,
1686                                       u32 *flags)
1687 {
1688         __be16 protocol = skb->protocol;
1689         u32  tx_flags = 0;
1690
1691         /* if we have a HW VLAN tag being added, default to the HW one */
1692         if (vlan_tx_tag_present(skb)) {
1693                 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1694                 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1695         /* else if it is a SW VLAN, check the next protocol and store the tag */
1696         } else if (protocol == htons(ETH_P_8021Q)) {
1697                 struct vlan_hdr *vhdr, _vhdr;
1698                 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1699                 if (!vhdr)
1700                         return -EINVAL;
1701
1702                 protocol = vhdr->h_vlan_encapsulated_proto;
1703                 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1704                 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1705         }
1706
1707         /* Insert 802.1p priority into VLAN header */
1708         if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1709             ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1710              (skb->priority != TC_PRIO_CONTROL))) {
1711                 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1712                 tx_flags |= (skb->priority & 0x7) <<
1713                                 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1714                 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1715                         struct vlan_ethhdr *vhdr;
1716                         int rc;
1717
1718                         rc = skb_cow_head(skb, 0);
1719                         if (rc < 0)
1720                                 return rc;
1721                         vhdr = (struct vlan_ethhdr *)skb->data;
1722                         vhdr->h_vlan_TCI = htons(tx_flags >>
1723                                                  I40E_TX_FLAGS_VLAN_SHIFT);
1724                 } else {
1725                         tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1726                 }
1727         }
1728         *flags = tx_flags;
1729         return 0;
1730 }
1731
1732 /**
1733  * i40e_tso - set up the tso context descriptor
1734  * @tx_ring:  ptr to the ring to send
1735  * @skb:      ptr to the skb we're sending
1736  * @tx_flags: the collected send information
1737  * @protocol: the send protocol
1738  * @hdr_len:  ptr to the size of the packet header
1739  * @cd_tunneling: ptr to context descriptor bits
1740  *
1741  * Returns 0 if no TSO can happen, 1 if tso is going, or error
1742  **/
1743 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1744                     u32 tx_flags, __be16 protocol, u8 *hdr_len,
1745                     u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1746 {
1747         u32 cd_cmd, cd_tso_len, cd_mss;
1748         struct ipv6hdr *ipv6h;
1749         struct tcphdr *tcph;
1750         struct iphdr *iph;
1751         u32 l4len;
1752         int err;
1753
1754         if (!skb_is_gso(skb))
1755                 return 0;
1756
1757         err = skb_cow_head(skb, 0);
1758         if (err < 0)
1759                 return err;
1760
1761         if (protocol == htons(ETH_P_IP)) {
1762                 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1763                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1764                 iph->tot_len = 0;
1765                 iph->check = 0;
1766                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1767                                                  0, IPPROTO_TCP, 0);
1768         } else if (skb_is_gso_v6(skb)) {
1769
1770                 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1771                                            : ipv6_hdr(skb);
1772                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1773                 ipv6h->payload_len = 0;
1774                 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1775                                                0, IPPROTO_TCP, 0);
1776         }
1777
1778         l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1779         *hdr_len = (skb->encapsulation
1780                     ? (skb_inner_transport_header(skb) - skb->data)
1781                     : skb_transport_offset(skb)) + l4len;
1782
1783         /* find the field values */
1784         cd_cmd = I40E_TX_CTX_DESC_TSO;
1785         cd_tso_len = skb->len - *hdr_len;
1786         cd_mss = skb_shinfo(skb)->gso_size;
1787         *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1788                                 ((u64)cd_tso_len <<
1789                                  I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1790                                 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1791         return 1;
1792 }
1793
1794 /**
1795  * i40e_tsyn - set up the tsyn context descriptor
1796  * @tx_ring:  ptr to the ring to send
1797  * @skb:      ptr to the skb we're sending
1798  * @tx_flags: the collected send information
1799  *
1800  * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1801  **/
1802 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1803                      u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1804 {
1805         struct i40e_pf *pf;
1806
1807         if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1808                 return 0;
1809
1810         /* Tx timestamps cannot be sampled when doing TSO */
1811         if (tx_flags & I40E_TX_FLAGS_TSO)
1812                 return 0;
1813
1814         /* only timestamp the outbound packet if the user has requested it and
1815          * we are not already transmitting a packet to be timestamped
1816          */
1817         pf = i40e_netdev_to_pf(tx_ring->netdev);
1818         if (pf->ptp_tx && !pf->ptp_tx_skb) {
1819                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1820                 pf->ptp_tx_skb = skb_get(skb);
1821         } else {
1822                 return 0;
1823         }
1824
1825         *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1826                                 I40E_TXD_CTX_QW1_CMD_SHIFT;
1827
1828         pf->ptp_tx_start = jiffies;
1829         schedule_work(&pf->ptp_tx_work);
1830
1831         return 1;
1832 }
1833
1834 /**
1835  * i40e_tx_enable_csum - Enable Tx checksum offloads
1836  * @skb: send buffer
1837  * @tx_flags: Tx flags currently set
1838  * @td_cmd: Tx descriptor command bits to set
1839  * @td_offset: Tx descriptor header offsets to set
1840  * @cd_tunneling: ptr to context desc bits
1841  **/
1842 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1843                                 u32 *td_cmd, u32 *td_offset,
1844                                 struct i40e_ring *tx_ring,
1845                                 u32 *cd_tunneling)
1846 {
1847         struct ipv6hdr *this_ipv6_hdr;
1848         unsigned int this_tcp_hdrlen;
1849         struct iphdr *this_ip_hdr;
1850         u32 network_hdr_len;
1851         u8 l4_hdr = 0;
1852
1853         if (skb->encapsulation) {
1854                 network_hdr_len = skb_inner_network_header_len(skb);
1855                 this_ip_hdr = inner_ip_hdr(skb);
1856                 this_ipv6_hdr = inner_ipv6_hdr(skb);
1857                 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1858
1859                 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1860
1861                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1862                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1863                                 ip_hdr(skb)->check = 0;
1864                         } else {
1865                                 *cd_tunneling |=
1866                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1867                         }
1868                 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1869                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1870                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1871                                 ip_hdr(skb)->check = 0;
1872                         } else {
1873                                 *cd_tunneling |=
1874                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1875                         }
1876                 }
1877
1878                 /* Now set the ctx descriptor fields */
1879                 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1880                                         I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1881                                    I40E_TXD_CTX_UDP_TUNNELING            |
1882                                    ((skb_inner_network_offset(skb) -
1883                                         skb_transport_offset(skb)) >> 1) <<
1884                                    I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1885
1886         } else {
1887                 network_hdr_len = skb_network_header_len(skb);
1888                 this_ip_hdr = ip_hdr(skb);
1889                 this_ipv6_hdr = ipv6_hdr(skb);
1890                 this_tcp_hdrlen = tcp_hdrlen(skb);
1891         }
1892
1893         /* Enable IP checksum offloads */
1894         if (tx_flags & I40E_TX_FLAGS_IPV4) {
1895                 l4_hdr = this_ip_hdr->protocol;
1896                 /* the stack computes the IP header already, the only time we
1897                  * need the hardware to recompute it is in the case of TSO.
1898                  */
1899                 if (tx_flags & I40E_TX_FLAGS_TSO) {
1900                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1901                         this_ip_hdr->check = 0;
1902                 } else {
1903                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1904                 }
1905                 /* Now set the td_offset for IP header length */
1906                 *td_offset = (network_hdr_len >> 2) <<
1907                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1908         } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1909                 l4_hdr = this_ipv6_hdr->nexthdr;
1910                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1911                 /* Now set the td_offset for IP header length */
1912                 *td_offset = (network_hdr_len >> 2) <<
1913                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1914         }
1915         /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1916         *td_offset |= (skb_network_offset(skb) >> 1) <<
1917                        I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1918
1919         /* Enable L4 checksum offloads */
1920         switch (l4_hdr) {
1921         case IPPROTO_TCP:
1922                 /* enable checksum offloads */
1923                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1924                 *td_offset |= (this_tcp_hdrlen >> 2) <<
1925                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1926                 break;
1927         case IPPROTO_SCTP:
1928                 /* enable SCTP checksum offload */
1929                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1930                 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1931                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1932                 break;
1933         case IPPROTO_UDP:
1934                 /* enable UDP checksum offload */
1935                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1936                 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1937                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1938                 break;
1939         default:
1940                 break;
1941         }
1942 }
1943
1944 /**
1945  * i40e_create_tx_ctx Build the Tx context descriptor
1946  * @tx_ring:  ring to create the descriptor on
1947  * @cd_type_cmd_tso_mss: Quad Word 1
1948  * @cd_tunneling: Quad Word 0 - bits 0-31
1949  * @cd_l2tag2: Quad Word 0 - bits 32-63
1950  **/
1951 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1952                                const u64 cd_type_cmd_tso_mss,
1953                                const u32 cd_tunneling, const u32 cd_l2tag2)
1954 {
1955         struct i40e_tx_context_desc *context_desc;
1956         int i = tx_ring->next_to_use;
1957
1958         if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1959             !cd_tunneling && !cd_l2tag2)
1960                 return;
1961
1962         /* grab the next descriptor */
1963         context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1964
1965         i++;
1966         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1967
1968         /* cpu_to_le32 and assign to struct fields */
1969         context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1970         context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1971         context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1972 }
1973
1974 /**
1975  * i40e_tx_map - Build the Tx descriptor
1976  * @tx_ring:  ring to send buffer on
1977  * @skb:      send buffer
1978  * @first:    first buffer info buffer to use
1979  * @tx_flags: collected send information
1980  * @hdr_len:  size of the packet header
1981  * @td_cmd:   the command field in the descriptor
1982  * @td_offset: offset for checksum or crc
1983  **/
1984 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1985                         struct i40e_tx_buffer *first, u32 tx_flags,
1986                         const u8 hdr_len, u32 td_cmd, u32 td_offset)
1987 {
1988         unsigned int data_len = skb->data_len;
1989         unsigned int size = skb_headlen(skb);
1990         struct skb_frag_struct *frag;
1991         struct i40e_tx_buffer *tx_bi;
1992         struct i40e_tx_desc *tx_desc;
1993         u16 i = tx_ring->next_to_use;
1994         u32 td_tag = 0;
1995         dma_addr_t dma;
1996         u16 gso_segs;
1997
1998         if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1999                 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2000                 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2001                          I40E_TX_FLAGS_VLAN_SHIFT;
2002         }
2003
2004         if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
2005                 gso_segs = skb_shinfo(skb)->gso_segs;
2006         else
2007                 gso_segs = 1;
2008
2009         /* multiply data chunks by size of headers */
2010         first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2011         first->gso_segs = gso_segs;
2012         first->skb = skb;
2013         first->tx_flags = tx_flags;
2014
2015         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2016
2017         tx_desc = I40E_TX_DESC(tx_ring, i);
2018         tx_bi = first;
2019
2020         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2021                 if (dma_mapping_error(tx_ring->dev, dma))
2022                         goto dma_error;
2023
2024                 /* record length, and DMA address */
2025                 dma_unmap_len_set(tx_bi, len, size);
2026                 dma_unmap_addr_set(tx_bi, dma, dma);
2027
2028                 tx_desc->buffer_addr = cpu_to_le64(dma);
2029
2030                 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2031                         tx_desc->cmd_type_offset_bsz =
2032                                 build_ctob(td_cmd, td_offset,
2033                                            I40E_MAX_DATA_PER_TXD, td_tag);
2034
2035                         tx_desc++;
2036                         i++;
2037                         if (i == tx_ring->count) {
2038                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
2039                                 i = 0;
2040                         }
2041
2042                         dma += I40E_MAX_DATA_PER_TXD;
2043                         size -= I40E_MAX_DATA_PER_TXD;
2044
2045                         tx_desc->buffer_addr = cpu_to_le64(dma);
2046                 }
2047
2048                 if (likely(!data_len))
2049                         break;
2050
2051                 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2052                                                           size, td_tag);
2053
2054                 tx_desc++;
2055                 i++;
2056                 if (i == tx_ring->count) {
2057                         tx_desc = I40E_TX_DESC(tx_ring, 0);
2058                         i = 0;
2059                 }
2060
2061                 size = skb_frag_size(frag);
2062                 data_len -= size;
2063
2064                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2065                                        DMA_TO_DEVICE);
2066
2067                 tx_bi = &tx_ring->tx_bi[i];
2068         }
2069
2070         /* Place RS bit on last descriptor of any packet that spans across the
2071          * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline.
2072          */
2073 #define WB_STRIDE 0x3
2074         if (((i & WB_STRIDE) != WB_STRIDE) &&
2075             (first <= &tx_ring->tx_bi[i]) &&
2076             (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) {
2077                 tx_desc->cmd_type_offset_bsz =
2078                         build_ctob(td_cmd, td_offset, size, td_tag) |
2079                         cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP <<
2080                                          I40E_TXD_QW1_CMD_SHIFT);
2081         } else {
2082                 tx_desc->cmd_type_offset_bsz =
2083                         build_ctob(td_cmd, td_offset, size, td_tag) |
2084                         cpu_to_le64((u64)I40E_TXD_CMD <<
2085                                          I40E_TXD_QW1_CMD_SHIFT);
2086         }
2087
2088         netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2089                                                  tx_ring->queue_index),
2090                              first->bytecount);
2091
2092         /* set the timestamp */
2093         first->time_stamp = jiffies;
2094
2095         /* Force memory writes to complete before letting h/w
2096          * know there are new descriptors to fetch.  (Only
2097          * applicable for weak-ordered memory model archs,
2098          * such as IA-64).
2099          */
2100         wmb();
2101
2102         /* set next_to_watch value indicating a packet is present */
2103         first->next_to_watch = tx_desc;
2104
2105         i++;
2106         if (i == tx_ring->count)
2107                 i = 0;
2108
2109         tx_ring->next_to_use = i;
2110
2111         /* notify HW of packet */
2112         writel(i, tx_ring->tail);
2113
2114         return;
2115
2116 dma_error:
2117         dev_info(tx_ring->dev, "TX DMA map failed\n");
2118
2119         /* clear dma mappings for failed tx_bi map */
2120         for (;;) {
2121                 tx_bi = &tx_ring->tx_bi[i];
2122                 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2123                 if (tx_bi == first)
2124                         break;
2125                 if (i == 0)
2126                         i = tx_ring->count;
2127                 i--;
2128         }
2129
2130         tx_ring->next_to_use = i;
2131 }
2132
2133 /**
2134  * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2135  * @tx_ring: the ring to be checked
2136  * @size:    the size buffer we want to assure is available
2137  *
2138  * Returns -EBUSY if a stop is needed, else 0
2139  **/
2140 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2141 {
2142         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2143         /* Memory barrier before checking head and tail */
2144         smp_mb();
2145
2146         /* Check again in a case another CPU has just made room available. */
2147         if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2148                 return -EBUSY;
2149
2150         /* A reprieve! - use start_queue because it doesn't call schedule */
2151         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2152         ++tx_ring->tx_stats.restart_queue;
2153         return 0;
2154 }
2155
2156 /**
2157  * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2158  * @tx_ring: the ring to be checked
2159  * @size:    the size buffer we want to assure is available
2160  *
2161  * Returns 0 if stop is not needed
2162  **/
2163 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2164 {
2165         if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2166                 return 0;
2167         return __i40e_maybe_stop_tx(tx_ring, size);
2168 }
2169
2170 /**
2171  * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2172  * @skb:     send buffer
2173  * @tx_ring: ring to send buffer on
2174  *
2175  * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2176  * there is not enough descriptors available in this ring since we need at least
2177  * one descriptor.
2178  **/
2179 static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2180                                       struct i40e_ring *tx_ring)
2181 {
2182 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2183         unsigned int f;
2184 #endif
2185         int count = 0;
2186
2187         /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2188          *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2189          *       + 4 desc gap to avoid the cache line where head is,
2190          *       + 1 desc for context descriptor,
2191          * otherwise try next time
2192          */
2193 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2194         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2195                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2196 #else
2197         count += skb_shinfo(skb)->nr_frags;
2198 #endif
2199         count += TXD_USE_COUNT(skb_headlen(skb));
2200         if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2201                 tx_ring->tx_stats.tx_busy++;
2202                 return 0;
2203         }
2204         return count;
2205 }
2206
2207 /**
2208  * i40e_xmit_frame_ring - Sends buffer on Tx ring
2209  * @skb:     send buffer
2210  * @tx_ring: ring to send buffer on
2211  *
2212  * Returns NETDEV_TX_OK if sent, else an error code
2213  **/
2214 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2215                                         struct i40e_ring *tx_ring)
2216 {
2217         u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2218         u32 cd_tunneling = 0, cd_l2tag2 = 0;
2219         struct i40e_tx_buffer *first;
2220         u32 td_offset = 0;
2221         u32 tx_flags = 0;
2222         __be16 protocol;
2223         u32 td_cmd = 0;
2224         u8 hdr_len = 0;
2225         int tsyn;
2226         int tso;
2227         if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2228                 return NETDEV_TX_BUSY;
2229
2230         /* prepare the xmit flags */
2231         if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2232                 goto out_drop;
2233
2234         /* obtain protocol of skb */
2235         protocol = skb->protocol;
2236
2237         /* record the location of the first descriptor for this packet */
2238         first = &tx_ring->tx_bi[tx_ring->next_to_use];
2239
2240         /* setup IPv4/IPv6 offloads */
2241         if (protocol == htons(ETH_P_IP))
2242                 tx_flags |= I40E_TX_FLAGS_IPV4;
2243         else if (protocol == htons(ETH_P_IPV6))
2244                 tx_flags |= I40E_TX_FLAGS_IPV6;
2245
2246         tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2247                        &cd_type_cmd_tso_mss, &cd_tunneling);
2248
2249         if (tso < 0)
2250                 goto out_drop;
2251         else if (tso)
2252                 tx_flags |= I40E_TX_FLAGS_TSO;
2253
2254         skb_tx_timestamp(skb);
2255
2256         tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2257
2258         if (tsyn)
2259                 tx_flags |= I40E_TX_FLAGS_TSYN;
2260
2261         /* always enable CRC insertion offload */
2262         td_cmd |= I40E_TX_DESC_CMD_ICRC;
2263
2264         /* Always offload the checksum, since it's in the data descriptor */
2265         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2266                 tx_flags |= I40E_TX_FLAGS_CSUM;
2267
2268                 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2269                                     tx_ring, &cd_tunneling);
2270         }
2271
2272         i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2273                            cd_tunneling, cd_l2tag2);
2274
2275         /* Add Flow Director ATR if it's enabled.
2276          *
2277          * NOTE: this must always be directly before the data descriptor.
2278          */
2279         i40e_atr(tx_ring, skb, tx_flags, protocol);
2280
2281         i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2282                     td_cmd, td_offset);
2283
2284         i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2285
2286         return NETDEV_TX_OK;
2287
2288 out_drop:
2289         dev_kfree_skb_any(skb);
2290         return NETDEV_TX_OK;
2291 }
2292
2293 /**
2294  * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2295  * @skb:    send buffer
2296  * @netdev: network interface device structure
2297  *
2298  * Returns NETDEV_TX_OK if sent, else an error code
2299  **/
2300 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2301 {
2302         struct i40e_netdev_priv *np = netdev_priv(netdev);
2303         struct i40e_vsi *vsi = np->vsi;
2304         struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
2305
2306         /* hardware can't handle really short frames, hardware padding works
2307          * beyond this point
2308          */
2309         if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2310                 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2311                         return NETDEV_TX_OK;
2312                 skb->len = I40E_MIN_TX_LEN;
2313                 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2314         }
2315
2316         return i40e_xmit_frame_ring(skb, tx_ring);
2317 }