drivers/net/ethernet/intel/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  29
  30 #include <linux/module.h>
  31 #include <linux/types.h>
  32 #include <linux/init.h>
  33 #include <linux/bitops.h>
  34 #include <linux/vmalloc.h>
  35 #include <linux/pagemap.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/ipv6.h>
  38 #include <linux/slab.h>
  39 #include <net/checksum.h>
  40 #include <net/ip6_checksum.h>
  41 #include <linux/net_tstamp.h>
  42 #include <linux/mii.h>
  43 #include <linux/ethtool.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/pci.h>
  47 #include <linux/pci-aspm.h>
  48 #include <linux/delay.h>
  49 #include <linux/interrupt.h>
  50 #include <linux/ip.h>
  51 #include <linux/tcp.h>
  52 #include <linux/sctp.h>
  53 #include <linux/if_ether.h>
  54 #include <linux/aer.h>
  55 #include <linux/prefetch.h>
  56 #include <linux/pm_runtime.h>
  57 #ifdef CONFIG_IGB_DCA
  58 #include <linux/dca.h>
  59 #endif
  60 #include "igb.h"
  61
  62 #define MAJ 4
  63 #define MIN 0
  64 #define BUILD 1
  65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  66 __stringify(BUILD) "-k"
  67 char igb_driver_name[] = "igb";
  68 char igb_driver_version[] = DRV_VERSION;
  69 static const char igb_driver_string[] =
  70                                 "Intel(R) Gigabit Ethernet Network Driver";
  71 static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
  72
  73 static const struct e1000_info *igb_info_tbl[] = {
  74         [board_82575] = &e1000_82575_info,
  75 };
  76
  77 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  99         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
 100         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
 101         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
 102         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
 103         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
 104         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
 105         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
 106         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
 107         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
 108         /* required last entry */
 109         {0, }
 110 };
 111
 112 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 113
 114 void igb_reset(struct igb_adapter *);
 115 static int igb_setup_all_tx_resources(struct igb_adapter *);
 116 static int igb_setup_all_rx_resources(struct igb_adapter *);
 117 static void igb_free_all_tx_resources(struct igb_adapter *);
 118 static void igb_free_all_rx_resources(struct igb_adapter *);
 119 static void igb_setup_mrqc(struct igb_adapter *);
 120 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 121 static void __devexit igb_remove(struct pci_dev *pdev);
 122 static int igb_sw_init(struct igb_adapter *);
 123 static int igb_open(struct net_device *);
 124 static int igb_close(struct net_device *);
 125 static void igb_configure_tx(struct igb_adapter *);
 126 static void igb_configure_rx(struct igb_adapter *);
 127 static void igb_clean_all_tx_rings(struct igb_adapter *);
 128 static void igb_clean_all_rx_rings(struct igb_adapter *);
 129 static void igb_clean_tx_ring(struct igb_ring *);
 130 static void igb_clean_rx_ring(struct igb_ring *);
 131 static void igb_set_rx_mode(struct net_device *);
 132 static void igb_update_phy_info(unsigned long);
 133 static void igb_watchdog(unsigned long);
 134 static void igb_watchdog_task(struct work_struct *);
 135 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 136 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
 137                                                  struct rtnl_link_stats64 *stats);
 138 static int igb_change_mtu(struct net_device *, int);
 139 static int igb_set_mac(struct net_device *, void *);
 140 static void igb_set_uta(struct igb_adapter *adapter);
 141 static irqreturn_t igb_intr(int irq, void *);
 142 static irqreturn_t igb_intr_msi(int irq, void *);
 143 static irqreturn_t igb_msix_other(int irq, void *);
 144 static irqreturn_t igb_msix_ring(int irq, void *);
 145 #ifdef CONFIG_IGB_DCA
 146 static void igb_update_dca(struct igb_q_vector *);
 147 static void igb_setup_dca(struct igb_adapter *);
 148 #endif /* CONFIG_IGB_DCA */
 149 static int igb_poll(struct napi_struct *, int);
 150 static bool igb_clean_tx_irq(struct igb_q_vector *);
 151 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
 152 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 153 static void igb_tx_timeout(struct net_device *);
 154 static void igb_reset_task(struct work_struct *);
 155 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
 156 static int igb_vlan_rx_add_vid(struct net_device *, u16);
 157 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
 158 static void igb_restore_vlan(struct igb_adapter *);
 159 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 160 static void igb_ping_all_vfs(struct igb_adapter *);
 161 static void igb_msg_task(struct igb_adapter *);
 162 static void igb_vmm_control(struct igb_adapter *);
 163 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 164 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 165 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 166 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 167                                int vf, u16 vlan, u8 qos);
 168 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 169 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 170                                  struct ifla_vf_info *ivi);
 171 static void igb_check_vf_rate_limit(struct igb_adapter *);
 172
 173 #ifdef CONFIG_PCI_IOV
 174 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 175 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
 176 static int igb_check_vf_assignment(struct igb_adapter *adapter);
 177 #endif
 178
 179 #ifdef CONFIG_PM
 180 #ifdef CONFIG_PM_SLEEP
 181 static int igb_suspend(struct device *);
 182 #endif
 183 static int igb_resume(struct device *);
 184 #ifdef CONFIG_PM_RUNTIME
 185 static int igb_runtime_suspend(struct device *dev);
 186 static int igb_runtime_resume(struct device *dev);
 187 static int igb_runtime_idle(struct device *dev);
 188 #endif
 189 static const struct dev_pm_ops igb_pm_ops = {
 190         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 191         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 192                         igb_runtime_idle)
 193 };
 194 #endif
 195 static void igb_shutdown(struct pci_dev *);
 196 #ifdef CONFIG_IGB_DCA
 197 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 198 static struct notifier_block dca_notifier = {
 199         .notifier_call  = igb_notify_dca,
 200         .next           = NULL,
 201         .priority       = 0
 202 };
 203 #endif
 204 #ifdef CONFIG_NET_POLL_CONTROLLER
 205 /* for netdump / net console */
 206 static void igb_netpoll(struct net_device *);
 207 #endif
 208 #ifdef CONFIG_PCI_IOV
 209 static unsigned int max_vfs = 0;
 210 module_param(max_vfs, uint, 0);
 211 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 212                  "per physical function");
 213 #endif /* CONFIG_PCI_IOV */
 214
 215 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 216                      pci_channel_state_t);
 217 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 218 static void igb_io_resume(struct pci_dev *);
 219
 220 static struct pci_error_handlers igb_err_handler = {
 221         .error_detected = igb_io_error_detected,
 222         .slot_reset = igb_io_slot_reset,
 223         .resume = igb_io_resume,
 224 };
 225
 226 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 227
 228 static struct pci_driver igb_driver = {
 229         .name     = igb_driver_name,
 230         .id_table = igb_pci_tbl,
 231         .probe    = igb_probe,
 232         .remove   = __devexit_p(igb_remove),
 233 #ifdef CONFIG_PM
 234         .driver.pm = &igb_pm_ops,
 235 #endif
 236         .shutdown = igb_shutdown,
 237         .err_handler = &igb_err_handler
 238 };
 239
 240 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 241 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 242 MODULE_LICENSE("GPL");
 243 MODULE_VERSION(DRV_VERSION);
 244
 245 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 246 static int debug = -1;
 247 module_param(debug, int, 0);
 248 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 249
 250 struct igb_reg_info {
 251         u32 ofs;
 252         char *name;
 253 };
 254
 255 static const struct igb_reg_info igb_reg_info_tbl[] = {
 256
 257         /* General Registers */
 258         {E1000_CTRL, "CTRL"},
 259         {E1000_STATUS, "STATUS"},
 260         {E1000_CTRL_EXT, "CTRL_EXT"},
 261
 262         /* Interrupt Registers */
 263         {E1000_ICR, "ICR"},
 264
 265         /* RX Registers */
 266         {E1000_RCTL, "RCTL"},
 267         {E1000_RDLEN(0), "RDLEN"},
 268         {E1000_RDH(0), "RDH"},
 269         {E1000_RDT(0), "RDT"},
 270         {E1000_RXDCTL(0), "RXDCTL"},
 271         {E1000_RDBAL(0), "RDBAL"},
 272         {E1000_RDBAH(0), "RDBAH"},
 273
 274         /* TX Registers */
 275         {E1000_TCTL, "TCTL"},
 276         {E1000_TDBAL(0), "TDBAL"},
 277         {E1000_TDBAH(0), "TDBAH"},
 278         {E1000_TDLEN(0), "TDLEN"},
 279         {E1000_TDH(0), "TDH"},
 280         {E1000_TDT(0), "TDT"},
 281         {E1000_TXDCTL(0), "TXDCTL"},
 282         {E1000_TDFH, "TDFH"},
 283         {E1000_TDFT, "TDFT"},
 284         {E1000_TDFHS, "TDFHS"},
 285         {E1000_TDFPC, "TDFPC"},
 286
 287         /* List Terminator */
 288         {}
 289 };
 290
 291 /*
 292  * igb_regdump - register printout routine
 293  */
 294 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
 295 {
 296         int n = 0;
 297         char rname[16];
 298         u32 regs[8];
 299
 300         switch (reginfo->ofs) {
 301         case E1000_RDLEN(0):
 302                 for (n = 0; n < 4; n++)
 303                         regs[n] = rd32(E1000_RDLEN(n));
 304                 break;
 305         case E1000_RDH(0):
 306                 for (n = 0; n < 4; n++)
 307                         regs[n] = rd32(E1000_RDH(n));
 308                 break;
 309         case E1000_RDT(0):
 310                 for (n = 0; n < 4; n++)
 311                         regs[n] = rd32(E1000_RDT(n));
 312                 break;
 313         case E1000_RXDCTL(0):
 314                 for (n = 0; n < 4; n++)
 315                         regs[n] = rd32(E1000_RXDCTL(n));
 316                 break;
 317         case E1000_RDBAL(0):
 318                 for (n = 0; n < 4; n++)
 319                         regs[n] = rd32(E1000_RDBAL(n));
 320                 break;
 321         case E1000_RDBAH(0):
 322                 for (n = 0; n < 4; n++)
 323                         regs[n] = rd32(E1000_RDBAH(n));
 324                 break;
 325         case E1000_TDBAL(0):
 326                 for (n = 0; n < 4; n++)
 327                         regs[n] = rd32(E1000_RDBAL(n));
 328                 break;
 329         case E1000_TDBAH(0):
 330                 for (n = 0; n < 4; n++)
 331                         regs[n] = rd32(E1000_TDBAH(n));
 332                 break;
 333         case E1000_TDLEN(0):
 334                 for (n = 0; n < 4; n++)
 335                         regs[n] = rd32(E1000_TDLEN(n));
 336                 break;
 337         case E1000_TDH(0):
 338                 for (n = 0; n < 4; n++)
 339                         regs[n] = rd32(E1000_TDH(n));
 340                 break;
 341         case E1000_TDT(0):
 342                 for (n = 0; n < 4; n++)
 343                         regs[n] = rd32(E1000_TDT(n));
 344                 break;
 345         case E1000_TXDCTL(0):
 346                 for (n = 0; n < 4; n++)
 347                         regs[n] = rd32(E1000_TXDCTL(n));
 348                 break;
 349         default:
 350                 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
 351                 return;
 352         }
 353
 354         snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
 355         pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
 356                 regs[2], regs[3]);
 357 }
 358
 359 /*
 360  * igb_dump - Print registers, tx-rings and rx-rings
 361  */
 362 static void igb_dump(struct igb_adapter *adapter)
 363 {
 364         struct net_device *netdev = adapter->netdev;
 365         struct e1000_hw *hw = &adapter->hw;
 366         struct igb_reg_info *reginfo;
 367         struct igb_ring *tx_ring;
 368         union e1000_adv_tx_desc *tx_desc;
 369         struct my_u0 { u64 a; u64 b; } *u0;
 370         struct igb_ring *rx_ring;
 371         union e1000_adv_rx_desc *rx_desc;
 372         u32 staterr;
 373         u16 i, n;
 374
 375         if (!netif_msg_hw(adapter))
 376                 return;
 377
 378         /* Print netdevice Info */
 379         if (netdev) {
 380                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 381                 pr_info("Device Name     state            trans_start      "
 382                         "last_rx\n");
 383                 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
 384                         netdev->state, netdev->trans_start, netdev->last_rx);
 385         }
 386
 387         /* Print Registers */
 388         dev_info(&adapter->pdev->dev, "Register Dump\n");
 389         pr_info(" Register Name   Value\n");
 390         for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
 391              reginfo->name; reginfo++) {
 392                 igb_regdump(hw, reginfo);
 393         }
 394
 395         /* Print TX Ring Summary */
 396         if (!netdev || !netif_running(netdev))
 397                 goto exit;
 398
 399         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 400         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 401         for (n = 0; n < adapter->num_tx_queues; n++) {
 402                 struct igb_tx_buffer *buffer_info;
 403                 tx_ring = adapter->tx_ring[n];
 404                 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 405                 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
 406                         n, tx_ring->next_to_use, tx_ring->next_to_clean,
 407                         (u64)buffer_info->dma,
 408                         buffer_info->length,
 409                         buffer_info->next_to_watch,
 410                         (u64)buffer_info->time_stamp);
 411         }
 412
 413         /* Print TX Rings */
 414         if (!netif_msg_tx_done(adapter))
 415                 goto rx_ring_summary;
 416
 417         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 418
 419         /* Transmit Descriptor Formats
 420          *
 421          * Advanced Transmit Descriptor
 422          *   +--------------------------------------------------------------+
 423          * 0 |         Buffer Address [63:0]                                |
 424          *   +--------------------------------------------------------------+
 425          * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
 426          *   +--------------------------------------------------------------+
 427          *   63      46 45    40 39 38 36 35 32 31   24             15       0
 428          */
 429
 430         for (n = 0; n < adapter->num_tx_queues; n++) {
 431                 tx_ring = adapter->tx_ring[n];
 432                 pr_info("------------------------------------\n");
 433                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 434                 pr_info("------------------------------------\n");
 435                 pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] "
 436                         "[bi->dma       ] leng  ntw timestamp        "
 437                         "bi->skb\n");
 438
 439                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 440                         const char *next_desc;
 441                         struct igb_tx_buffer *buffer_info;
 442                         tx_desc = IGB_TX_DESC(tx_ring, i);
 443                         buffer_info = &tx_ring->tx_buffer_info[i];
 444                         u0 = (struct my_u0 *)tx_desc;
 445                         if (i == tx_ring->next_to_use &&
 446                             i == tx_ring->next_to_clean)
 447                                 next_desc = " NTC/U";
 448                         else if (i == tx_ring->next_to_use)
 449                                 next_desc = " NTU";
 450                         else if (i == tx_ring->next_to_clean)
 451                                 next_desc = " NTC";
 452                         else
 453                                 next_desc = "";
 454
 455                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
 456                                 " %04X  %p %016llX %p%s\n", i,
 457                                 le64_to_cpu(u0->a),
 458                                 le64_to_cpu(u0->b),
 459                                 (u64)buffer_info->dma,
 460                                 buffer_info->length,
 461                                 buffer_info->next_to_watch,
 462                                 (u64)buffer_info->time_stamp,
 463                                 buffer_info->skb, next_desc);
 464
 465                         if (netif_msg_pktdata(adapter) && buffer_info->skb)
 466                                 print_hex_dump(KERN_INFO, "",
 467                                         DUMP_PREFIX_ADDRESS,
 468                                         16, 1, buffer_info->skb->data,
 469                                         buffer_info->length, true);
 470                 }
 471         }
 472
 473         /* Print RX Rings Summary */
 474 rx_ring_summary:
 475         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 476         pr_info("Queue [NTU] [NTC]\n");
 477         for (n = 0; n < adapter->num_rx_queues; n++) {
 478                 rx_ring = adapter->rx_ring[n];
 479                 pr_info(" %5d %5X %5X\n",
 480                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 481         }
 482
 483         /* Print RX Rings */
 484         if (!netif_msg_rx_status(adapter))
 485                 goto exit;
 486
 487         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 488
 489         /* Advanced Receive Descriptor (Read) Format
 490          *    63                                           1        0
 491          *    +-----------------------------------------------------+
 492          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 493          *    +----------------------------------------------+------+
 494          *  8 |       Header Buffer Address [63:1]           |  DD  |
 495          *    +-----------------------------------------------------+
 496          *
 497          *
 498          * Advanced Receive Descriptor (Write-Back) Format
 499          *
 500          *   63       48 47    32 31  30      21 20 17 16   4 3     0
 501          *   +------------------------------------------------------+
 502          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 503          *   | Checksum   Ident  |   |           |    | Type | Type |
 504          *   +------------------------------------------------------+
 505          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 506          *   +------------------------------------------------------+
 507          *   63       48 47    32 31            20 19               0
 508          */
 509
 510         for (n = 0; n < adapter->num_rx_queues; n++) {
 511                 rx_ring = adapter->rx_ring[n];
 512                 pr_info("------------------------------------\n");
 513                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 514                 pr_info("------------------------------------\n");
 515                 pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] "
 516                         "[bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
 517                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] -----"
 518                         "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
 519
 520                 for (i = 0; i < rx_ring->count; i++) {
 521                         const char *next_desc;
 522                         struct igb_rx_buffer *buffer_info;
 523                         buffer_info = &rx_ring->rx_buffer_info[i];
 524                         rx_desc = IGB_RX_DESC(rx_ring, i);
 525                         u0 = (struct my_u0 *)rx_desc;
 526                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 527
 528                         if (i == rx_ring->next_to_use)
 529                                 next_desc = " NTU";
 530                         else if (i == rx_ring->next_to_clean)
 531                                 next_desc = " NTC";
 532                         else
 533                                 next_desc = "";
 534
 535                         if (staterr & E1000_RXD_STAT_DD) {
 536                                 /* Descriptor Done */
 537                                 pr_info("%s[0x%03X]     %016llX %016llX -------"
 538                                         "--------- %p%s\n", "RWB", i,
 539                                         le64_to_cpu(u0->a),
 540                                         le64_to_cpu(u0->b),
 541                                         buffer_info->skb, next_desc);
 542                         } else {
 543                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX"
 544                                         " %p%s\n", "R  ", i,
 545                                         le64_to_cpu(u0->a),
 546                                         le64_to_cpu(u0->b),
 547                                         (u64)buffer_info->dma,
 548                                         buffer_info->skb, next_desc);
 549
 550                                 if (netif_msg_pktdata(adapter) &&
 551                                     buffer_info->dma && buffer_info->skb) {
 552                                         print_hex_dump(KERN_INFO, "",
 553                                                   DUMP_PREFIX_ADDRESS,
 554                                                   16, 1, buffer_info->skb->data,
 555                                                   IGB_RX_HDR_LEN, true);
 556                                         print_hex_dump(KERN_INFO, "",
 557                                           DUMP_PREFIX_ADDRESS,
 558                                           16, 1,
 559                                           page_address(buffer_info->page) +
 560                                                       buffer_info->page_offset,
 561                                           PAGE_SIZE/2, true);
 562                                 }
 563                         }
 564                 }
 565         }
 566
 567 exit:
 568         return;
 569 }
 570
 571 /**
 572  * igb_get_hw_dev - return device
 573  * used by hardware layer to print debugging information
 574  **/
 575 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 576 {
 577         struct igb_adapter *adapter = hw->back;
 578         return adapter->netdev;
 579 }
 580
 581 /**
 582  * igb_init_module - Driver Registration Routine
 583  *
 584  * igb_init_module is the first routine called when the driver is
 585  * loaded. All it does is register with the PCI subsystem.
 586  **/
 587 static int __init igb_init_module(void)
 588 {
 589         int ret;
 590         pr_info("%s - version %s\n",
 591                igb_driver_string, igb_driver_version);
 592
 593         pr_info("%s\n", igb_copyright);
 594
 595 #ifdef CONFIG_IGB_DCA
 596         dca_register_notify(&dca_notifier);
 597 #endif
 598         ret = pci_register_driver(&igb_driver);
 599         return ret;
 600 }
 601
 602 module_init(igb_init_module);
 603
 604 /**
 605  * igb_exit_module - Driver Exit Cleanup Routine
 606  *
 607  * igb_exit_module is called just before the driver is removed
 608  * from memory.
 609  **/
 610 static void __exit igb_exit_module(void)
 611 {
 612 #ifdef CONFIG_IGB_DCA
 613         dca_unregister_notify(&dca_notifier);
 614 #endif
 615         pci_unregister_driver(&igb_driver);
 616 }
 617
 618 module_exit(igb_exit_module);
 619
 620 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 621 /**
 622  * igb_cache_ring_register - Descriptor ring to register mapping
 623  * @adapter: board private structure to initialize
 624  *
 625  * Once we know the feature-set enabled for the device, we'll cache
 626  * the register offset the descriptor ring is assigned to.
 627  **/
 628 static void igb_cache_ring_register(struct igb_adapter *adapter)
 629 {
 630         int i = 0, j = 0;
 631         u32 rbase_offset = adapter->vfs_allocated_count;
 632
 633         switch (adapter->hw.mac.type) {
 634         case e1000_82576:
 635                 /* The queues are allocated for virtualization such that VF 0
 636                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 637                  * In order to avoid collision we start at the first free queue
 638                  * and continue consuming queues in the same sequence
 639                  */
 640                 if (adapter->vfs_allocated_count) {
 641                         for (; i < adapter->rss_queues; i++)
 642                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 643                                                                Q_IDX_82576(i);
 644                 }
 645         case e1000_82575:
 646         case e1000_82580:
 647         case e1000_i350:
 648         case e1000_i210:
 649         case e1000_i211:
 650         default:
 651                 for (; i < adapter->num_rx_queues; i++)
 652                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 653                 for (; j < adapter->num_tx_queues; j++)
 654                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 655                 break;
 656         }
 657 }
 658
 659 static void igb_free_queues(struct igb_adapter *adapter)
 660 {
 661         int i;
 662
 663         for (i = 0; i < adapter->num_tx_queues; i++) {
 664                 kfree(adapter->tx_ring[i]);
 665                 adapter->tx_ring[i] = NULL;
 666         }
 667         for (i = 0; i < adapter->num_rx_queues; i++) {
 668                 kfree(adapter->rx_ring[i]);
 669                 adapter->rx_ring[i] = NULL;
 670         }
 671         adapter->num_rx_queues = 0;
 672         adapter->num_tx_queues = 0;
 673 }
 674
 675 /**
 676  * igb_alloc_queues - Allocate memory for all rings
 677  * @adapter: board private structure to initialize
 678  *
 679  * We allocate one ring per queue at run-time since we don't know the
 680  * number of queues at compile-time.
 681  **/
 682 static int igb_alloc_queues(struct igb_adapter *adapter)
 683 {
 684         struct igb_ring *ring;
 685         int i;
 686         int orig_node = adapter->node;
 687
 688         for (i = 0; i < adapter->num_tx_queues; i++) {
 689                 if (orig_node == -1) {
 690                         int cur_node = next_online_node(adapter->node);
 691                         if (cur_node == MAX_NUMNODES)
 692                                 cur_node = first_online_node;
 693                         adapter->node = cur_node;
 694                 }
 695                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 696                                     adapter->node);
 697                 if (!ring)
 698                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 699                 if (!ring)
 700                         goto err;
 701                 ring->count = adapter->tx_ring_count;
 702                 ring->queue_index = i;
 703                 ring->dev = &adapter->pdev->dev;
 704                 ring->netdev = adapter->netdev;
 705                 ring->numa_node = adapter->node;
 706                 /* For 82575, context index must be unique per ring. */
 707                 if (adapter->hw.mac.type == e1000_82575)
 708                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
 709                 adapter->tx_ring[i] = ring;
 710         }
 711         /* Restore the adapter's original node */
 712         adapter->node = orig_node;
 713
 714         for (i = 0; i < adapter->num_rx_queues; i++) {
 715                 if (orig_node == -1) {
 716                         int cur_node = next_online_node(adapter->node);
 717                         if (cur_node == MAX_NUMNODES)
 718                                 cur_node = first_online_node;
 719                         adapter->node = cur_node;
 720                 }
 721                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 722                                     adapter->node);
 723                 if (!ring)
 724                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 725                 if (!ring)
 726                         goto err;
 727                 ring->count = adapter->rx_ring_count;
 728                 ring->queue_index = i;
 729                 ring->dev = &adapter->pdev->dev;
 730                 ring->netdev = adapter->netdev;
 731                 ring->numa_node = adapter->node;
 732                 /* set flag indicating ring supports SCTP checksum offload */
 733                 if (adapter->hw.mac.type >= e1000_82576)
 734                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
 735
 736                 /*
 737                  * On i350, i210, and i211, loopback VLAN packets
 738                  * have the tag byte-swapped.
 739                  * */
 740                 if (adapter->hw.mac.type >= e1000_i350)
 741                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
 742
 743                 adapter->rx_ring[i] = ring;
 744         }
 745         /* Restore the adapter's original node */
 746         adapter->node = orig_node;
 747
 748         igb_cache_ring_register(adapter);
 749
 750         return 0;
 751
 752 err:
 753         /* Restore the adapter's original node */
 754         adapter->node = orig_node;
 755         igb_free_queues(adapter);
 756
 757         return -ENOMEM;
 758 }
 759
 760 /**
 761  *  igb_write_ivar - configure ivar for given MSI-X vector
 762  *  @hw: pointer to the HW structure
 763  *  @msix_vector: vector number we are allocating to a given ring
 764  *  @index: row index of IVAR register to write within IVAR table
 765  *  @offset: column offset of in IVAR, should be multiple of 8
 766  *
 767  *  This function is intended to handle the writing of the IVAR register
 768  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 769  *  each containing an cause allocation for an Rx and Tx ring, and a
 770  *  variable number of rows depending on the number of queues supported.
 771  **/
 772 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 773                            int index, int offset)
 774 {
 775         u32 ivar = array_rd32(E1000_IVAR0, index);
 776
 777         /* clear any bits that are currently set */
 778         ivar &= ~((u32)0xFF << offset);
 779
 780         /* write vector and valid bit */
 781         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 782
 783         array_wr32(E1000_IVAR0, index, ivar);
 784 }
 785
 786 #define IGB_N0_QUEUE -1
 787 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 788 {
 789         struct igb_adapter *adapter = q_vector->adapter;
 790         struct e1000_hw *hw = &adapter->hw;
 791         int rx_queue = IGB_N0_QUEUE;
 792         int tx_queue = IGB_N0_QUEUE;
 793         u32 msixbm = 0;
 794
 795         if (q_vector->rx.ring)
 796                 rx_queue = q_vector->rx.ring->reg_idx;
 797         if (q_vector->tx.ring)
 798                 tx_queue = q_vector->tx.ring->reg_idx;
 799
 800         switch (hw->mac.type) {
 801         case e1000_82575:
 802                 /* The 82575 assigns vectors using a bitmask, which matches the
 803                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 804                    or more queues to a vector, we write the appropriate bits
 805                    into the MSIXBM register for that vector. */
 806                 if (rx_queue > IGB_N0_QUEUE)
 807                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 808                 if (tx_queue > IGB_N0_QUEUE)
 809                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 810                 if (!adapter->msix_entries && msix_vector == 0)
 811                         msixbm |= E1000_EIMS_OTHER;
 812                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 813                 q_vector->eims_value = msixbm;
 814                 break;
 815         case e1000_82576:
 816                 /*
 817                  * 82576 uses a table that essentially consists of 2 columns
 818                  * with 8 rows.  The ordering is column-major so we use the
 819                  * lower 3 bits as the row index, and the 4th bit as the
 820                  * column offset.
 821                  */
 822                 if (rx_queue > IGB_N0_QUEUE)
 823                         igb_write_ivar(hw, msix_vector,
 824                                        rx_queue & 0x7,
 825                                        (rx_queue & 0x8) << 1);
 826                 if (tx_queue > IGB_N0_QUEUE)
 827                         igb_write_ivar(hw, msix_vector,
 828                                        tx_queue & 0x7,
 829                                        ((tx_queue & 0x8) << 1) + 8);
 830                 q_vector->eims_value = 1 << msix_vector;
 831                 break;
 832         case e1000_82580:
 833         case e1000_i350:
 834         case e1000_i210:
 835         case e1000_i211:
 836                 /*
 837                  * On 82580 and newer adapters the scheme is similar to 82576
 838                  * however instead of ordering column-major we have things
 839                  * ordered row-major.  So we traverse the table by using
 840                  * bit 0 as the column offset, and the remaining bits as the
 841                  * row index.
 842                  */
 843                 if (rx_queue > IGB_N0_QUEUE)
 844                         igb_write_ivar(hw, msix_vector,
 845                                        rx_queue >> 1,
 846                                        (rx_queue & 0x1) << 4);
 847                 if (tx_queue > IGB_N0_QUEUE)
 848                         igb_write_ivar(hw, msix_vector,
 849                                        tx_queue >> 1,
 850                                        ((tx_queue & 0x1) << 4) + 8);
 851                 q_vector->eims_value = 1 << msix_vector;
 852                 break;
 853         default:
 854                 BUG();
 855                 break;
 856         }
 857
 858         /* add q_vector eims value to global eims_enable_mask */
 859         adapter->eims_enable_mask |= q_vector->eims_value;
 860
 861         /* configure q_vector to set itr on first interrupt */
 862         q_vector->set_itr = 1;
 863 }
 864
 865 /**
 866  * igb_configure_msix - Configure MSI-X hardware
 867  *
 868  * igb_configure_msix sets up the hardware to properly
 869  * generate MSI-X interrupts.
 870  **/
 871 static void igb_configure_msix(struct igb_adapter *adapter)
 872 {
 873         u32 tmp;
 874         int i, vector = 0;
 875         struct e1000_hw *hw = &adapter->hw;
 876
 877         adapter->eims_enable_mask = 0;
 878
 879         /* set vector for other causes, i.e. link changes */
 880         switch (hw->mac.type) {
 881         case e1000_82575:
 882                 tmp = rd32(E1000_CTRL_EXT);
 883                 /* enable MSI-X PBA support*/
 884                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 885
 886                 /* Auto-Mask interrupts upon ICR read. */
 887                 tmp |= E1000_CTRL_EXT_EIAME;
 888                 tmp |= E1000_CTRL_EXT_IRCA;
 889
 890                 wr32(E1000_CTRL_EXT, tmp);
 891
 892                 /* enable msix_other interrupt */
 893                 array_wr32(E1000_MSIXBM(0), vector++,
 894                                       E1000_EIMS_OTHER);
 895                 adapter->eims_other = E1000_EIMS_OTHER;
 896
 897                 break;
 898
 899         case e1000_82576:
 900         case e1000_82580:
 901         case e1000_i350:
 902         case e1000_i210:
 903         case e1000_i211:
 904                 /* Turn on MSI-X capability first, or our settings
 905                  * won't stick.  And it will take days to debug. */
 906                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 907                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 908                                 E1000_GPIE_NSICR);
 909
 910                 /* enable msix_other interrupt */
 911                 adapter->eims_other = 1 << vector;
 912                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 913
 914                 wr32(E1000_IVAR_MISC, tmp);
 915                 break;
 916         default:
 917                 /* do nothing, since nothing else supports MSI-X */
 918                 break;
 919         } /* switch (hw->mac.type) */
 920
 921         adapter->eims_enable_mask |= adapter->eims_other;
 922
 923         for (i = 0; i < adapter->num_q_vectors; i++)
 924                 igb_assign_vector(adapter->q_vector[i], vector++);
 925
 926         wrfl();
 927 }
 928
 929 /**
 930  * igb_request_msix - Initialize MSI-X interrupts
 931  *
 932  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 933  * kernel.
 934  **/
 935 static int igb_request_msix(struct igb_adapter *adapter)
 936 {
 937         struct net_device *netdev = adapter->netdev;
 938         struct e1000_hw *hw = &adapter->hw;
 939         int i, err = 0, vector = 0;
 940
 941         err = request_irq(adapter->msix_entries[vector].vector,
 942                           igb_msix_other, 0, netdev->name, adapter);
 943         if (err)
 944                 goto out;
 945         vector++;
 946
 947         for (i = 0; i < adapter->num_q_vectors; i++) {
 948                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 949
 950                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 951
 952                 if (q_vector->rx.ring && q_vector->tx.ring)
 953                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 954                                 q_vector->rx.ring->queue_index);
 955                 else if (q_vector->tx.ring)
 956                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 957                                 q_vector->tx.ring->queue_index);
 958                 else if (q_vector->rx.ring)
 959                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 960                                 q_vector->rx.ring->queue_index);
 961                 else
 962                         sprintf(q_vector->name, "%s-unused", netdev->name);
 963
 964                 err = request_irq(adapter->msix_entries[vector].vector,
 965                                   igb_msix_ring, 0, q_vector->name,
 966                                   q_vector);
 967                 if (err)
 968                         goto out;
 969                 vector++;
 970         }
 971
 972         igb_configure_msix(adapter);
 973         return 0;
 974 out:
 975         return err;
 976 }
 977
 978 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 979 {
 980         if (adapter->msix_entries) {
 981                 pci_disable_msix(adapter->pdev);
 982                 kfree(adapter->msix_entries);
 983                 adapter->msix_entries = NULL;
 984         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 985                 pci_disable_msi(adapter->pdev);
 986         }
 987 }
 988
 989 /**
 990  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 991  * @adapter: board private structure to initialize
 992  *
 993  * This function frees the memory allocated to the q_vectors.  In addition if
 994  * NAPI is enabled it will delete any references to the NAPI struct prior
 995  * to freeing the q_vector.
 996  **/
 997 static void igb_free_q_vectors(struct igb_adapter *adapter)
 998 {
 999         int v_idx;
1000
1001         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1002                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1003                 adapter->q_vector[v_idx] = NULL;
1004                 if (!q_vector)
1005                         continue;
1006                 netif_napi_del(&q_vector->napi);
1007                 kfree(q_vector);
1008         }
1009         adapter->num_q_vectors = 0;
1010 }
1011
1012 /**
1013  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1014  *
1015  * This function resets the device so that it has 0 rx queues, tx queues, and
1016  * MSI-X interrupts allocated.
1017  */
1018 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1019 {
1020         igb_free_queues(adapter);
1021         igb_free_q_vectors(adapter);
1022         igb_reset_interrupt_capability(adapter);
1023 }
1024
1025 /**
1026  * igb_set_interrupt_capability - set MSI or MSI-X if supported
1027  *
1028  * Attempt to configure interrupts using the best available
1029  * capabilities of the hardware and kernel.
1030  **/
1031 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1032 {
1033         int err;
1034         int numvecs, i;
1035
1036         /* Number of supported queues. */
1037         adapter->num_rx_queues = adapter->rss_queues;
1038         if (adapter->vfs_allocated_count)
1039                 adapter->num_tx_queues = 1;
1040         else
1041                 adapter->num_tx_queues = adapter->rss_queues;
1042
1043         /* start with one vector for every rx queue */
1044         numvecs = adapter->num_rx_queues;
1045
1046         /* if tx handler is separate add 1 for every tx queue */
1047         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1048                 numvecs += adapter->num_tx_queues;
1049
1050         /* store the number of vectors reserved for queues */
1051         adapter->num_q_vectors = numvecs;
1052
1053         /* add 1 vector for link status interrupts */
1054         numvecs++;
1055         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1056                                         GFP_KERNEL);
1057
1058         if (!adapter->msix_entries)
1059                 goto msi_only;
1060
1061         for (i = 0; i < numvecs; i++)
1062                 adapter->msix_entries[i].entry = i;
1063
1064         err = pci_enable_msix(adapter->pdev,
1065                               adapter->msix_entries,
1066                               numvecs);
1067         if (err == 0)
1068                 goto out;
1069
1070         igb_reset_interrupt_capability(adapter);
1071
1072         /* If we can't do MSI-X, try MSI */
1073 msi_only:
1074 #ifdef CONFIG_PCI_IOV
1075         /* disable SR-IOV for non MSI-X configurations */
1076         if (adapter->vf_data) {
1077                 struct e1000_hw *hw = &adapter->hw;
1078                 /* disable iov and allow time for transactions to clear */
1079                 pci_disable_sriov(adapter->pdev);
1080                 msleep(500);
1081
1082                 kfree(adapter->vf_data);
1083                 adapter->vf_data = NULL;
1084                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1085                 wrfl();
1086                 msleep(100);
1087                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1088         }
1089 #endif
1090         adapter->vfs_allocated_count = 0;
1091         adapter->rss_queues = 1;
1092         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1093         adapter->num_rx_queues = 1;
1094         adapter->num_tx_queues = 1;
1095         adapter->num_q_vectors = 1;
1096         if (!pci_enable_msi(adapter->pdev))
1097                 adapter->flags |= IGB_FLAG_HAS_MSI;
1098 out:
1099         /* Notify the stack of the (possibly) reduced queue counts. */
1100         rtnl_lock();
1101         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1102         err = netif_set_real_num_rx_queues(adapter->netdev,
1103                 adapter->num_rx_queues);
1104         rtnl_unlock();
1105         return err;
1106 }
1107
1108 /**
1109  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1110  * @adapter: board private structure to initialize
1111  *
1112  * We allocate one q_vector per queue interrupt.  If allocation fails we
1113  * return -ENOMEM.
1114  **/
1115 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1116 {
1117         struct igb_q_vector *q_vector;
1118         struct e1000_hw *hw = &adapter->hw;
1119         int v_idx;
1120         int orig_node = adapter->node;
1121
1122         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1123                 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1124                                                 adapter->num_tx_queues)) &&
1125                     (adapter->num_rx_queues == v_idx))
1126                         adapter->node = orig_node;
1127                 if (orig_node == -1) {
1128                         int cur_node = next_online_node(adapter->node);
1129                         if (cur_node == MAX_NUMNODES)
1130                                 cur_node = first_online_node;
1131                         adapter->node = cur_node;
1132                 }
1133                 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1134                                         adapter->node);
1135                 if (!q_vector)
1136                         q_vector = kzalloc(sizeof(struct igb_q_vector),
1137                                            GFP_KERNEL);
1138                 if (!q_vector)
1139                         goto err_out;
1140                 q_vector->adapter = adapter;
1141                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1142                 q_vector->itr_val = IGB_START_ITR;
1143                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1144                 adapter->q_vector[v_idx] = q_vector;
1145         }
1146         /* Restore the adapter's original node */
1147         adapter->node = orig_node;
1148
1149         return 0;
1150
1151 err_out:
1152         /* Restore the adapter's original node */
1153         adapter->node = orig_node;
1154         igb_free_q_vectors(adapter);
1155         return -ENOMEM;
1156 }
1157
1158 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1159                                       int ring_idx, int v_idx)
1160 {
1161         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1162
1163         q_vector->rx.ring = adapter->rx_ring[ring_idx];
1164         q_vector->rx.ring->q_vector = q_vector;
1165         q_vector->rx.count++;
1166         q_vector->itr_val = adapter->rx_itr_setting;
1167         if (q_vector->itr_val && q_vector->itr_val <= 3)
1168                 q_vector->itr_val = IGB_START_ITR;
1169 }
1170
1171 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1172                                       int ring_idx, int v_idx)
1173 {
1174         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1175
1176         q_vector->tx.ring = adapter->tx_ring[ring_idx];
1177         q_vector->tx.ring->q_vector = q_vector;
1178         q_vector->tx.count++;
1179         q_vector->itr_val = adapter->tx_itr_setting;
1180         q_vector->tx.work_limit = adapter->tx_work_limit;
1181         if (q_vector->itr_val && q_vector->itr_val <= 3)
1182                 q_vector->itr_val = IGB_START_ITR;
1183 }
1184
1185 /**
1186  * igb_map_ring_to_vector - maps allocated queues to vectors
1187  *
1188  * This function maps the recently allocated queues to vectors.
1189  **/
1190 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1191 {
1192         int i;
1193         int v_idx = 0;
1194
1195         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1196             (adapter->num_q_vectors < adapter->num_tx_queues))
1197                 return -ENOMEM;
1198
1199         if (adapter->num_q_vectors >=
1200             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1201                 for (i = 0; i < adapter->num_rx_queues; i++)
1202                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1203                 for (i = 0; i < adapter->num_tx_queues; i++)
1204                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1205         } else {
1206                 for (i = 0; i < adapter->num_rx_queues; i++) {
1207                         if (i < adapter->num_tx_queues)
1208                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1209                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1210                 }
1211                 for (; i < adapter->num_tx_queues; i++)
1212                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1213         }
1214         return 0;
1215 }
1216
1217 /**
1218  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1219  *
1220  * This function initializes the interrupts and allocates all of the queues.
1221  **/
1222 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1223 {
1224         struct pci_dev *pdev = adapter->pdev;
1225         int err;
1226
1227         err = igb_set_interrupt_capability(adapter);
1228         if (err)
1229                 return err;
1230
1231         err = igb_alloc_q_vectors(adapter);
1232         if (err) {
1233                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1234                 goto err_alloc_q_vectors;
1235         }
1236
1237         err = igb_alloc_queues(adapter);
1238         if (err) {
1239                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1240                 goto err_alloc_queues;
1241         }
1242
1243         err = igb_map_ring_to_vector(adapter);
1244         if (err) {
1245                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1246                 goto err_map_queues;
1247         }
1248
1249
1250         return 0;
1251 err_map_queues:
1252         igb_free_queues(adapter);
1253 err_alloc_queues:
1254         igb_free_q_vectors(adapter);
1255 err_alloc_q_vectors:
1256         igb_reset_interrupt_capability(adapter);
1257         return err;
1258 }
1259
1260 /**
1261  * igb_request_irq - initialize interrupts
1262  *
1263  * Attempts to configure interrupts using the best available
1264  * capabilities of the hardware and kernel.
1265  **/
1266 static int igb_request_irq(struct igb_adapter *adapter)
1267 {
1268         struct net_device *netdev = adapter->netdev;
1269         struct pci_dev *pdev = adapter->pdev;
1270         int err = 0;
1271
1272         if (adapter->msix_entries) {
1273                 err = igb_request_msix(adapter);
1274                 if (!err)
1275                         goto request_done;
1276                 /* fall back to MSI */
1277                 igb_clear_interrupt_scheme(adapter);
1278                 if (!pci_enable_msi(pdev))
1279                         adapter->flags |= IGB_FLAG_HAS_MSI;
1280                 igb_free_all_tx_resources(adapter);
1281                 igb_free_all_rx_resources(adapter);
1282                 adapter->num_tx_queues = 1;
1283                 adapter->num_rx_queues = 1;
1284                 adapter->num_q_vectors = 1;
1285                 err = igb_alloc_q_vectors(adapter);
1286                 if (err) {
1287                         dev_err(&pdev->dev,
1288                                 "Unable to allocate memory for vectors\n");
1289                         goto request_done;
1290                 }
1291                 err = igb_alloc_queues(adapter);
1292                 if (err) {
1293                         dev_err(&pdev->dev,
1294                                 "Unable to allocate memory for queues\n");
1295                         igb_free_q_vectors(adapter);
1296                         goto request_done;
1297                 }
1298                 igb_setup_all_tx_resources(adapter);
1299                 igb_setup_all_rx_resources(adapter);
1300         }
1301
1302         igb_assign_vector(adapter->q_vector[0], 0);
1303
1304         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1305                 err = request_irq(pdev->irq, igb_intr_msi, 0,
1306                                   netdev->name, adapter);
1307                 if (!err)
1308                         goto request_done;
1309
1310                 /* fall back to legacy interrupts */
1311                 igb_reset_interrupt_capability(adapter);
1312                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1313         }
1314
1315         err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1316                           netdev->name, adapter);
1317
1318         if (err)
1319                 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1320                         err);
1321
1322 request_done:
1323         return err;
1324 }
1325
1326 static void igb_free_irq(struct igb_adapter *adapter)
1327 {
1328         if (adapter->msix_entries) {
1329                 int vector = 0, i;
1330
1331                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1332
1333                 for (i = 0; i < adapter->num_q_vectors; i++)
1334                         free_irq(adapter->msix_entries[vector++].vector,
1335                                  adapter->q_vector[i]);
1336         } else {
1337                 free_irq(adapter->pdev->irq, adapter);
1338         }
1339 }
1340
1341 /**
1342  * igb_irq_disable - Mask off interrupt generation on the NIC
1343  * @adapter: board private structure
1344  **/
1345 static void igb_irq_disable(struct igb_adapter *adapter)
1346 {
1347         struct e1000_hw *hw = &adapter->hw;
1348
1349         /*
1350          * we need to be careful when disabling interrupts.  The VFs are also
1351          * mapped into these registers and so clearing the bits can cause
1352          * issues on the VF drivers so we only need to clear what we set
1353          */
1354         if (adapter->msix_entries) {
1355                 u32 regval = rd32(E1000_EIAM);
1356                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1357                 wr32(E1000_EIMC, adapter->eims_enable_mask);
1358                 regval = rd32(E1000_EIAC);
1359                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1360         }
1361
1362         wr32(E1000_IAM, 0);
1363         wr32(E1000_IMC, ~0);
1364         wrfl();
1365         if (adapter->msix_entries) {
1366                 int i;
1367                 for (i = 0; i < adapter->num_q_vectors; i++)
1368                         synchronize_irq(adapter->msix_entries[i].vector);
1369         } else {
1370                 synchronize_irq(adapter->pdev->irq);
1371         }
1372 }
1373
1374 /**
1375  * igb_irq_enable - Enable default interrupt generation settings
1376  * @adapter: board private structure
1377  **/
1378 static void igb_irq_enable(struct igb_adapter *adapter)
1379 {
1380         struct e1000_hw *hw = &adapter->hw;
1381
1382         if (adapter->msix_entries) {
1383                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1384                 u32 regval = rd32(E1000_EIAC);
1385                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1386                 regval = rd32(E1000_EIAM);
1387                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1388                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1389                 if (adapter->vfs_allocated_count) {
1390                         wr32(E1000_MBVFIMR, 0xFF);
1391                         ims |= E1000_IMS_VMMB;
1392                 }
1393                 wr32(E1000_IMS, ims);
1394         } else {
1395                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1396                                 E1000_IMS_DRSTA);
1397                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1398                                 E1000_IMS_DRSTA);
1399         }
1400 }
1401
1402 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1403 {
1404         struct e1000_hw *hw = &adapter->hw;
1405         u16 vid = adapter->hw.mng_cookie.vlan_id;
1406         u16 old_vid = adapter->mng_vlan_id;
1407
1408         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1409                 /* add VID to filter table */
1410                 igb_vfta_set(hw, vid, true);
1411                 adapter->mng_vlan_id = vid;
1412         } else {
1413                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1414         }
1415
1416         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1417             (vid != old_vid) &&
1418             !test_bit(old_vid, adapter->active_vlans)) {
1419                 /* remove VID from filter table */
1420                 igb_vfta_set(hw, old_vid, false);
1421         }
1422 }
1423
1424 /**
1425  * igb_release_hw_control - release control of the h/w to f/w
1426  * @adapter: address of board private structure
1427  *
1428  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1429  * For ASF and Pass Through versions of f/w this means that the
1430  * driver is no longer loaded.
1431  *
1432  **/
1433 static void igb_release_hw_control(struct igb_adapter *adapter)
1434 {
1435         struct e1000_hw *hw = &adapter->hw;
1436         u32 ctrl_ext;
1437
1438         /* Let firmware take over control of h/w */
1439         ctrl_ext = rd32(E1000_CTRL_EXT);
1440         wr32(E1000_CTRL_EXT,
1441                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1442 }
1443
1444 /**
1445  * igb_get_hw_control - get control of the h/w from f/w
1446  * @adapter: address of board private structure
1447  *
1448  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1449  * For ASF and Pass Through versions of f/w this means that
1450  * the driver is loaded.
1451  *
1452  **/
1453 static void igb_get_hw_control(struct igb_adapter *adapter)
1454 {
1455         struct e1000_hw *hw = &adapter->hw;
1456         u32 ctrl_ext;
1457
1458         /* Let firmware know the driver has taken over */
1459         ctrl_ext = rd32(E1000_CTRL_EXT);
1460         wr32(E1000_CTRL_EXT,
1461                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1462 }
1463
1464 /**
1465  * igb_configure - configure the hardware for RX and TX
1466  * @adapter: private board structure
1467  **/
1468 static void igb_configure(struct igb_adapter *adapter)
1469 {
1470         struct net_device *netdev = adapter->netdev;
1471         int i;
1472
1473         igb_get_hw_control(adapter);
1474         igb_set_rx_mode(netdev);
1475
1476         igb_restore_vlan(adapter);
1477
1478         igb_setup_tctl(adapter);
1479         igb_setup_mrqc(adapter);
1480         igb_setup_rctl(adapter);
1481
1482         igb_configure_tx(adapter);
1483         igb_configure_rx(adapter);
1484
1485         igb_rx_fifo_flush_82575(&adapter->hw);
1486
1487         /* call igb_desc_unused which always leaves
1488          * at least 1 descriptor unused to make sure
1489          * next_to_use != next_to_clean */
1490         for (i = 0; i < adapter->num_rx_queues; i++) {
1491                 struct igb_ring *ring = adapter->rx_ring[i];
1492                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1493         }
1494 }
1495
1496 /**
1497  * igb_power_up_link - Power up the phy/serdes link
1498  * @adapter: address of board private structure
1499  **/
1500 void igb_power_up_link(struct igb_adapter *adapter)
1501 {
1502         igb_reset_phy(&adapter->hw);
1503
1504         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1505                 igb_power_up_phy_copper(&adapter->hw);
1506         else
1507                 igb_power_up_serdes_link_82575(&adapter->hw);
1508 }
1509
1510 /**
1511  * igb_power_down_link - Power down the phy/serdes link
1512  * @adapter: address of board private structure
1513  */
1514 static void igb_power_down_link(struct igb_adapter *adapter)
1515 {
1516         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1517                 igb_power_down_phy_copper_82575(&adapter->hw);
1518         else
1519                 igb_shutdown_serdes_link_82575(&adapter->hw);
1520 }
1521
1522 /**
1523  * igb_up - Open the interface and prepare it to handle traffic
1524  * @adapter: board private structure
1525  **/
1526 int igb_up(struct igb_adapter *adapter)
1527 {
1528         struct e1000_hw *hw = &adapter->hw;
1529         int i;
1530
1531         /* hardware has been reset, we need to reload some things */
1532         igb_configure(adapter);
1533
1534         clear_bit(__IGB_DOWN, &adapter->state);
1535
1536         for (i = 0; i < adapter->num_q_vectors; i++)
1537                 napi_enable(&(adapter->q_vector[i]->napi));
1538
1539         if (adapter->msix_entries)
1540                 igb_configure_msix(adapter);
1541         else
1542                 igb_assign_vector(adapter->q_vector[0], 0);
1543
1544         /* Clear any pending interrupts. */
1545         rd32(E1000_ICR);
1546         igb_irq_enable(adapter);
1547
1548         /* notify VFs that reset has been completed */
1549         if (adapter->vfs_allocated_count) {
1550                 u32 reg_data = rd32(E1000_CTRL_EXT);
1551                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1552                 wr32(E1000_CTRL_EXT, reg_data);
1553         }
1554
1555         netif_tx_start_all_queues(adapter->netdev);
1556
1557         /* start the watchdog. */
1558         hw->mac.get_link_status = 1;
1559         schedule_work(&adapter->watchdog_task);
1560
1561         return 0;
1562 }
1563
1564 void igb_down(struct igb_adapter *adapter)
1565 {
1566         struct net_device *netdev = adapter->netdev;
1567         struct e1000_hw *hw = &adapter->hw;
1568         u32 tctl, rctl;
1569         int i;
1570
1571         /* signal that we're down so the interrupt handler does not
1572          * reschedule our watchdog timer */
1573         set_bit(__IGB_DOWN, &adapter->state);
1574
1575         /* disable receives in the hardware */
1576         rctl = rd32(E1000_RCTL);
1577         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1578         /* flush and sleep below */
1579
1580         netif_tx_stop_all_queues(netdev);
1581
1582         /* disable transmits in the hardware */
1583         tctl = rd32(E1000_TCTL);
1584         tctl &= ~E1000_TCTL_EN;
1585         wr32(E1000_TCTL, tctl);
1586         /* flush both disables and wait for them to finish */
1587         wrfl();
1588         msleep(10);
1589
1590         for (i = 0; i < adapter->num_q_vectors; i++)
1591                 napi_disable(&(adapter->q_vector[i]->napi));
1592
1593         igb_irq_disable(adapter);
1594
1595         del_timer_sync(&adapter->watchdog_timer);
1596         del_timer_sync(&adapter->phy_info_timer);
1597
1598         netif_carrier_off(netdev);
1599
1600         /* record the stats before reset*/
1601         spin_lock(&adapter->stats64_lock);
1602         igb_update_stats(adapter, &adapter->stats64);
1603         spin_unlock(&adapter->stats64_lock);
1604
1605         adapter->link_speed = 0;
1606         adapter->link_duplex = 0;
1607
1608         if (!pci_channel_offline(adapter->pdev))
1609                 igb_reset(adapter);
1610         igb_clean_all_tx_rings(adapter);
1611         igb_clean_all_rx_rings(adapter);
1612 #ifdef CONFIG_IGB_DCA
1613
1614         /* since we reset the hardware DCA settings were cleared */
1615         igb_setup_dca(adapter);
1616 #endif
1617 }
1618
1619 void igb_reinit_locked(struct igb_adapter *adapter)
1620 {
1621         WARN_ON(in_interrupt());
1622         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1623                 msleep(1);
1624         igb_down(adapter);
1625         igb_up(adapter);
1626         clear_bit(__IGB_RESETTING, &adapter->state);
1627 }
1628
1629 void igb_reset(struct igb_adapter *adapter)
1630 {
1631         struct pci_dev *pdev = adapter->pdev;
1632         struct e1000_hw *hw = &adapter->hw;
1633         struct e1000_mac_info *mac = &hw->mac;
1634         struct e1000_fc_info *fc = &hw->fc;
1635         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1636         u16 hwm;
1637
1638         /* Repartition Pba for greater than 9k mtu
1639          * To take effect CTRL.RST is required.
1640          */
1641         switch (mac->type) {
1642         case e1000_i350:
1643         case e1000_82580:
1644                 pba = rd32(E1000_RXPBS);
1645                 pba = igb_rxpbs_adjust_82580(pba);
1646                 break;
1647         case e1000_82576:
1648                 pba = rd32(E1000_RXPBS);
1649                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1650                 break;
1651         case e1000_82575:
1652         case e1000_i210:
1653         case e1000_i211:
1654         default:
1655                 pba = E1000_PBA_34K;
1656                 break;
1657         }
1658
1659         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1660             (mac->type < e1000_82576)) {
1661                 /* adjust PBA for jumbo frames */
1662                 wr32(E1000_PBA, pba);
1663
1664                 /* To maintain wire speed transmits, the Tx FIFO should be
1665                  * large enough to accommodate two full transmit packets,
1666                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1667                  * the Rx FIFO should be large enough to accommodate at least
1668                  * one full receive packet and is similarly rounded up and
1669                  * expressed in KB. */
1670                 pba = rd32(E1000_PBA);
1671                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1672                 tx_space = pba >> 16;
1673                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1674                 pba &= 0xffff;
1675                 /* the tx fifo also stores 16 bytes of information about the tx
1676                  * but don't include ethernet FCS because hardware appends it */
1677                 min_tx_space = (adapter->max_frame_size +
1678                                 sizeof(union e1000_adv_tx_desc) -
1679                                 ETH_FCS_LEN) * 2;
1680                 min_tx_space = ALIGN(min_tx_space, 1024);
1681                 min_tx_space >>= 10;
1682                 /* software strips receive CRC, so leave room for it */
1683                 min_rx_space = adapter->max_frame_size;
1684                 min_rx_space = ALIGN(min_rx_space, 1024);
1685                 min_rx_space >>= 10;
1686
1687                 /* If current Tx allocation is less than the min Tx FIFO size,
1688                  * and the min Tx FIFO size is less than the current Rx FIFO
1689                  * allocation, take space away from current Rx allocation */
1690                 if (tx_space < min_tx_space &&
1691                     ((min_tx_space - tx_space) < pba)) {
1692                         pba = pba - (min_tx_space - tx_space);
1693
1694                         /* if short on rx space, rx wins and must trump tx
1695                          * adjustment */
1696                         if (pba < min_rx_space)
1697                                 pba = min_rx_space;
1698                 }
1699                 wr32(E1000_PBA, pba);
1700         }
1701
1702         /* flow control settings */
1703         /* The high water mark must be low enough to fit one full frame
1704          * (or the size used for early receive) above it in the Rx FIFO.
1705          * Set it to the lower of:
1706          * - 90% of the Rx FIFO size, or
1707          * - the full Rx FIFO size minus one full frame */
1708         hwm = min(((pba << 10) * 9 / 10),
1709                         ((pba << 10) - 2 * adapter->max_frame_size));
1710
1711         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1712         fc->low_water = fc->high_water - 16;
1713         fc->pause_time = 0xFFFF;
1714         fc->send_xon = 1;
1715         fc->current_mode = fc->requested_mode;
1716
1717         /* disable receive for all VFs and wait one second */
1718         if (adapter->vfs_allocated_count) {
1719                 int i;
1720                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1721                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1722
1723                 /* ping all the active vfs to let them know we are going down */
1724                 igb_ping_all_vfs(adapter);
1725
1726                 /* disable transmits and receives */
1727                 wr32(E1000_VFRE, 0);
1728                 wr32(E1000_VFTE, 0);
1729         }
1730
1731         /* Allow time for pending master requests to run */
1732         hw->mac.ops.reset_hw(hw);
1733         wr32(E1000_WUC, 0);
1734
1735         if (hw->mac.ops.init_hw(hw))
1736                 dev_err(&pdev->dev, "Hardware Error\n");
1737
1738         /*
1739          * Flow control settings reset on hardware reset, so guarantee flow
1740          * control is off when forcing speed.
1741          */
1742         if (!hw->mac.autoneg)
1743                 igb_force_mac_fc(hw);
1744
1745         igb_init_dmac(adapter, pba);
1746         if (!netif_running(adapter->netdev))
1747                 igb_power_down_link(adapter);
1748
1749         igb_update_mng_vlan(adapter);
1750
1751         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1752         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1753
1754         igb_get_phy_info(hw);
1755 }
1756
1757 static netdev_features_t igb_fix_features(struct net_device *netdev,
1758         netdev_features_t features)
1759 {
1760         /*
1761          * Since there is no support for separate rx/tx vlan accel
1762          * enable/disable make sure tx flag is always in same state as rx.
1763          */
1764         if (features & NETIF_F_HW_VLAN_RX)
1765                 features |= NETIF_F_HW_VLAN_TX;
1766         else
1767                 features &= ~NETIF_F_HW_VLAN_TX;
1768
1769         return features;
1770 }
1771
1772 static int igb_set_features(struct net_device *netdev,
1773         netdev_features_t features)
1774 {
1775         netdev_features_t changed = netdev->features ^ features;
1776         struct igb_adapter *adapter = netdev_priv(netdev);
1777
1778         if (changed & NETIF_F_HW_VLAN_RX)
1779                 igb_vlan_mode(netdev, features);
1780
1781         if (!(changed & NETIF_F_RXALL))
1782                 return 0;
1783
1784         netdev->features = features;
1785
1786         if (netif_running(netdev))
1787                 igb_reinit_locked(adapter);
1788         else
1789                 igb_reset(adapter);
1790
1791         return 0;
1792 }
1793
1794 static const struct net_device_ops igb_netdev_ops = {
1795         .ndo_open               = igb_open,
1796         .ndo_stop               = igb_close,
1797         .ndo_start_xmit         = igb_xmit_frame,
1798         .ndo_get_stats64        = igb_get_stats64,
1799         .ndo_set_rx_mode        = igb_set_rx_mode,
1800         .ndo_set_mac_address    = igb_set_mac,
1801         .ndo_change_mtu         = igb_change_mtu,
1802         .ndo_do_ioctl           = igb_ioctl,
1803         .ndo_tx_timeout         = igb_tx_timeout,
1804         .ndo_validate_addr      = eth_validate_addr,
1805         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1806         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1807         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1808         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1809         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1810         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1811 #ifdef CONFIG_NET_POLL_CONTROLLER
1812         .ndo_poll_controller    = igb_netpoll,
1813 #endif
1814         .ndo_fix_features       = igb_fix_features,
1815         .ndo_set_features       = igb_set_features,
1816 };
1817
1818 /**
1819  * igb_set_fw_version - Configure version string for ethtool
1820  * @adapter: adapter struct
1821  *
1822  **/
1823 void igb_set_fw_version(struct igb_adapter *adapter)
1824 {
1825         struct e1000_hw *hw = &adapter->hw;
1826         u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1827         u16 major, build, patch, fw_version;
1828         u32 etrack_id;
1829
1830         hw->nvm.ops.read(hw, 5, 1, &fw_version);
1831         if (adapter->hw.mac.type != e1000_i211) {
1832                 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1833                 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1834                 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1835
1836                 /* combo image version needs to be found */
1837                 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1838                 if ((comb_offset != 0x0) &&
1839                     (comb_offset != IGB_NVM_VER_INVALID)) {
1840                         hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1841                                          + 1), 1, &comb_verh);
1842                         hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1843                                          1, &comb_verl);
1844
1845                         /* Only display Option Rom if it exists and is valid */
1846                         if ((comb_verh && comb_verl) &&
1847                             ((comb_verh != IGB_NVM_VER_INVALID) &&
1848                              (comb_verl != IGB_NVM_VER_INVALID))) {
1849                                 major = comb_verl >> IGB_COMB_VER_SHFT;
1850                                 build = (comb_verl << IGB_COMB_VER_SHFT) |
1851                                         (comb_verh >> IGB_COMB_VER_SHFT);
1852                                 patch = comb_verh & IGB_COMB_VER_MASK;
1853                                 snprintf(adapter->fw_version,
1854                                          sizeof(adapter->fw_version),
1855                                          "%d.%d%d, 0x%08x, %d.%d.%d",
1856                                          (fw_version & IGB_MAJOR_MASK) >>
1857                                          IGB_MAJOR_SHIFT,
1858                                          (fw_version & IGB_MINOR_MASK) >>
1859                                          IGB_MINOR_SHIFT,
1860                                          (fw_version & IGB_BUILD_MASK),
1861                                          etrack_id, major, build, patch);
1862                                 goto out;
1863                         }
1864                 }
1865                 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1866                          "%d.%d%d, 0x%08x",
1867                          (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1868                          (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1869                          (fw_version & IGB_BUILD_MASK), etrack_id);
1870         } else {
1871                 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1872                          "%d.%d%d",
1873                          (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1874                          (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1875                          (fw_version & IGB_BUILD_MASK));
1876         }
1877 out:
1878         return;
1879 }
1880
1881 /**
1882  * igb_probe - Device Initialization Routine
1883  * @pdev: PCI device information struct
1884  * @ent: entry in igb_pci_tbl
1885  *
1886  * Returns 0 on success, negative on failure
1887  *
1888  * igb_probe initializes an adapter identified by a pci_dev structure.
1889  * The OS initialization, configuring of the adapter private structure,
1890  * and a hardware reset occur.
1891  **/
1892 static int __devinit igb_probe(struct pci_dev *pdev,
1893                                const struct pci_device_id *ent)
1894 {
1895         struct net_device *netdev;
1896         struct igb_adapter *adapter;
1897         struct e1000_hw *hw;
1898         u16 eeprom_data = 0;
1899         s32 ret_val;
1900         static int global_quad_port_a; /* global quad port a indication */
1901         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1902         unsigned long mmio_start, mmio_len;
1903         int err, pci_using_dac;
1904         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1905         u8 part_str[E1000_PBANUM_LENGTH];
1906
1907         /* Catch broken hardware that put the wrong VF device ID in
1908          * the PCIe SR-IOV capability.
1909          */
1910         if (pdev->is_virtfn) {
1911                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1912                         pci_name(pdev), pdev->vendor, pdev->device);
1913                 return -EINVAL;
1914         }
1915
1916         err = pci_enable_device_mem(pdev);
1917         if (err)
1918                 return err;
1919
1920         pci_using_dac = 0;
1921         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1922         if (!err) {
1923                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1924                 if (!err)
1925                         pci_using_dac = 1;
1926         } else {
1927                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1928                 if (err) {
1929                         err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1930                         if (err) {
1931                                 dev_err(&pdev->dev, "No usable DMA "
1932                                         "configuration, aborting\n");
1933                                 goto err_dma;
1934                         }
1935                 }
1936         }
1937
1938         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1939                                            IORESOURCE_MEM),
1940                                            igb_driver_name);
1941         if (err)
1942                 goto err_pci_reg;
1943
1944         pci_enable_pcie_error_reporting(pdev);
1945
1946         pci_set_master(pdev);
1947         pci_save_state(pdev);
1948
1949         err = -ENOMEM;
1950         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1951                                    IGB_MAX_TX_QUEUES);
1952         if (!netdev)
1953                 goto err_alloc_etherdev;
1954
1955         SET_NETDEV_DEV(netdev, &pdev->dev);
1956
1957         pci_set_drvdata(pdev, netdev);
1958         adapter = netdev_priv(netdev);
1959         adapter->netdev = netdev;
1960         adapter->pdev = pdev;
1961         hw = &adapter->hw;
1962         hw->back = adapter;
1963         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
1964
1965         mmio_start = pci_resource_start(pdev, 0);
1966         mmio_len = pci_resource_len(pdev, 0);
1967
1968         err = -EIO;
1969         hw->hw_addr = ioremap(mmio_start, mmio_len);
1970         if (!hw->hw_addr)
1971                 goto err_ioremap;
1972
1973         netdev->netdev_ops = &igb_netdev_ops;
1974         igb_set_ethtool_ops(netdev);
1975         netdev->watchdog_timeo = 5 * HZ;
1976
1977         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1978
1979         netdev->mem_start = mmio_start;
1980         netdev->mem_end = mmio_start + mmio_len;
1981
1982         /* PCI config space info */
1983         hw->vendor_id = pdev->vendor;
1984         hw->device_id = pdev->device;
1985         hw->revision_id = pdev->revision;
1986         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1987         hw->subsystem_device_id = pdev->subsystem_device;
1988
1989         /* Copy the default MAC, PHY and NVM function pointers */
1990         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1991         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1992         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1993         /* Initialize skew-specific constants */
1994         err = ei->get_invariants(hw);
1995         if (err)
1996                 goto err_sw_init;
1997
1998         /* setup the private structure */
1999         err = igb_sw_init(adapter);
2000         if (err)
2001                 goto err_sw_init;
2002
2003         igb_get_bus_info_pcie(hw);
2004
2005         hw->phy.autoneg_wait_to_complete = false;
2006
2007         /* Copper options */
2008         if (hw->phy.media_type == e1000_media_type_copper) {
2009                 hw->phy.mdix = AUTO_ALL_MODES;
2010                 hw->phy.disable_polarity_correction = false;
2011                 hw->phy.ms_type = e1000_ms_hw_default;
2012         }
2013
2014         if (igb_check_reset_block(hw))
2015                 dev_info(&pdev->dev,
2016                         "PHY reset is blocked due to SOL/IDER session.\n");
2017
2018         /*
2019          * features is initialized to 0 in allocation, it might have bits
2020          * set by igb_sw_init so we should use an or instead of an
2021          * assignment.
2022          */
2023         netdev->features |= NETIF_F_SG |
2024                             NETIF_F_IP_CSUM |
2025                             NETIF_F_IPV6_CSUM |
2026                             NETIF_F_TSO |
2027                             NETIF_F_TSO6 |
2028                             NETIF_F_RXHASH |
2029                             NETIF_F_RXCSUM |
2030                             NETIF_F_HW_VLAN_RX |
2031                             NETIF_F_HW_VLAN_TX;
2032
2033         /* copy netdev features into list of user selectable features */
2034         netdev->hw_features |= netdev->features;
2035         netdev->hw_features |= NETIF_F_RXALL;
2036
2037         /* set this bit last since it cannot be part of hw_features */
2038         netdev->features |= NETIF_F_HW_VLAN_FILTER;
2039
2040         netdev->vlan_features |= NETIF_F_TSO |
2041                                  NETIF_F_TSO6 |
2042                                  NETIF_F_IP_CSUM |
2043                                  NETIF_F_IPV6_CSUM |
2044                                  NETIF_F_SG;
2045
2046         netdev->priv_flags |= IFF_SUPP_NOFCS;
2047
2048         if (pci_using_dac) {
2049                 netdev->features |= NETIF_F_HIGHDMA;
2050                 netdev->vlan_features |= NETIF_F_HIGHDMA;
2051         }
2052
2053         if (hw->mac.type >= e1000_82576) {
2054                 netdev->hw_features |= NETIF_F_SCTP_CSUM;
2055                 netdev->features |= NETIF_F_SCTP_CSUM;
2056         }
2057
2058         netdev->priv_flags |= IFF_UNICAST_FLT;
2059
2060         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2061
2062         /* before reading the NVM, reset the controller to put the device in a
2063          * known good starting state */
2064         hw->mac.ops.reset_hw(hw);
2065
2066         /*
2067          * make sure the NVM is good , i211 parts have special NVM that
2068          * doesn't contain a checksum
2069          */
2070         if (hw->mac.type != e1000_i211) {
2071                 if (hw->nvm.ops.validate(hw) < 0) {
2072                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2073                         err = -EIO;
2074                         goto err_eeprom;
2075                 }
2076         }
2077
2078         /* copy the MAC address out of the NVM */
2079         if (hw->mac.ops.read_mac_addr(hw))
2080                 dev_err(&pdev->dev, "NVM Read Error\n");
2081
2082         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2083         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2084
2085         if (!is_valid_ether_addr(netdev->perm_addr)) {
2086                 dev_err(&pdev->dev, "Invalid MAC Address\n");
2087                 err = -EIO;
2088                 goto err_eeprom;
2089         }
2090
2091         /* get firmware version for ethtool -i */
2092         igb_set_fw_version(adapter);
2093
2094         setup_timer(&adapter->watchdog_timer, igb_watchdog,
2095                     (unsigned long) adapter);
2096         setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2097                     (unsigned long) adapter);
2098
2099         INIT_WORK(&adapter->reset_task, igb_reset_task);
2100         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2101
2102         /* Initialize link properties that are user-changeable */
2103         adapter->fc_autoneg = true;
2104         hw->mac.autoneg = true;
2105         hw->phy.autoneg_advertised = 0x2f;
2106
2107         hw->fc.requested_mode = e1000_fc_default;
2108         hw->fc.current_mode = e1000_fc_default;
2109
2110         igb_validate_mdi_setting(hw);
2111
2112         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2113          * enable the ACPI Magic Packet filter
2114          */
2115
2116         if (hw->bus.func == 0)
2117                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2118         else if (hw->mac.type >= e1000_82580)
2119                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2120                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2121                                  &eeprom_data);
2122         else if (hw->bus.func == 1)
2123                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2124
2125         if (eeprom_data & eeprom_apme_mask)
2126                 adapter->eeprom_wol |= E1000_WUFC_MAG;
2127
2128         /* now that we have the eeprom settings, apply the special cases where
2129          * the eeprom may be wrong or the board simply won't support wake on
2130          * lan on a particular port */
2131         switch (pdev->device) {
2132         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2133                 adapter->eeprom_wol = 0;
2134                 break;
2135         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2136         case E1000_DEV_ID_82576_FIBER:
2137         case E1000_DEV_ID_82576_SERDES:
2138                 /* Wake events only supported on port A for dual fiber
2139                  * regardless of eeprom setting */
2140                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2141                         adapter->eeprom_wol = 0;
2142                 break;
2143         case E1000_DEV_ID_82576_QUAD_COPPER:
2144         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2145                 /* if quad port adapter, disable WoL on all but port A */
2146                 if (global_quad_port_a != 0)
2147                         adapter->eeprom_wol = 0;
2148                 else
2149                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2150                 /* Reset for multiple quad port adapters */
2151                 if (++global_quad_port_a == 4)
2152                         global_quad_port_a = 0;
2153                 break;
2154         }
2155
2156         /* initialize the wol settings based on the eeprom settings */
2157         adapter->wol = adapter->eeprom_wol;
2158         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2159
2160         /* reset the hardware with the new settings */
2161         igb_reset(adapter);
2162
2163         /* let the f/w know that the h/w is now under the control of the
2164          * driver. */
2165         igb_get_hw_control(adapter);
2166
2167         strcpy(netdev->name, "eth%d");
2168         err = register_netdev(netdev);
2169         if (err)
2170                 goto err_register;
2171
2172         /* carrier off reporting is important to ethtool even BEFORE open */
2173         netif_carrier_off(netdev);
2174
2175 #ifdef CONFIG_IGB_DCA
2176         if (dca_add_requester(&pdev->dev) == 0) {
2177                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2178                 dev_info(&pdev->dev, "DCA enabled\n");
2179                 igb_setup_dca(adapter);
2180         }
2181
2182 #endif
2183 #ifdef CONFIG_IGB_PTP
2184         /* do hw tstamp init after resetting */
2185         igb_ptp_init(adapter);
2186
2187 #endif
2188         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2189         /* print bus type/speed/width info */
2190         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2191                  netdev->name,
2192                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2193                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2194                                                             "unknown"),
2195                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2196                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2197                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2198                    "unknown"),
2199                  netdev->dev_addr);
2200
2201         ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2202         if (ret_val)
2203                 strcpy(part_str, "Unknown");
2204         dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2205         dev_info(&pdev->dev,
2206                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2207                 adapter->msix_entries ? "MSI-X" :
2208                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2209                 adapter->num_rx_queues, adapter->num_tx_queues);
2210         switch (hw->mac.type) {
2211         case e1000_i350:
2212         case e1000_i210:
2213         case e1000_i211:
2214                 igb_set_eee_i350(hw);
2215                 break;
2216         default:
2217                 break;
2218         }
2219
2220         pm_runtime_put_noidle(&pdev->dev);
2221         return 0;
2222
2223 err_register:
2224         igb_release_hw_control(adapter);
2225 err_eeprom:
2226         if (!igb_check_reset_block(hw))
2227                 igb_reset_phy(hw);
2228
2229         if (hw->flash_address)
2230                 iounmap(hw->flash_address);
2231 err_sw_init:
2232         igb_clear_interrupt_scheme(adapter);
2233         iounmap(hw->hw_addr);
2234 err_ioremap:
2235         free_netdev(netdev);
2236 err_alloc_etherdev:
2237         pci_release_selected_regions(pdev,
2238                                      pci_select_bars(pdev, IORESOURCE_MEM));
2239 err_pci_reg:
2240 err_dma:
2241         pci_disable_device(pdev);
2242         return err;
2243 }
2244
2245 /**
2246  * igb_remove - Device Removal Routine
2247  * @pdev: PCI device information struct
2248  *
2249  * igb_remove is called by the PCI subsystem to alert the driver
2250  * that it should release a PCI device.  The could be caused by a
2251  * Hot-Plug event, or because the driver is going to be removed from
2252  * memory.
2253  **/
2254 static void __devexit igb_remove(struct pci_dev *pdev)
2255 {
2256         struct net_device *netdev = pci_get_drvdata(pdev);
2257         struct igb_adapter *adapter = netdev_priv(netdev);
2258         struct e1000_hw *hw = &adapter->hw;
2259
2260         pm_runtime_get_noresume(&pdev->dev);
2261 #ifdef CONFIG_IGB_PTP
2262         igb_ptp_remove(adapter);
2263
2264 #endif
2265         /*
2266          * The watchdog timer may be rescheduled, so explicitly
2267          * disable watchdog from being rescheduled.
2268          */
2269         set_bit(__IGB_DOWN, &adapter->state);
2270         del_timer_sync(&adapter->watchdog_timer);
2271         del_timer_sync(&adapter->phy_info_timer);
2272
2273         cancel_work_sync(&adapter->reset_task);
2274         cancel_work_sync(&adapter->watchdog_task);
2275
2276 #ifdef CONFIG_IGB_DCA
2277         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2278                 dev_info(&pdev->dev, "DCA disabled\n");
2279                 dca_remove_requester(&pdev->dev);
2280                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2281                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2282         }
2283 #endif
2284
2285         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2286          * would have already happened in close and is redundant. */
2287         igb_release_hw_control(adapter);
2288
2289         unregister_netdev(netdev);
2290
2291         igb_clear_interrupt_scheme(adapter);
2292
2293 #ifdef CONFIG_PCI_IOV
2294         /* reclaim resources allocated to VFs */
2295         if (adapter->vf_data) {
2296                 /* disable iov and allow time for transactions to clear */
2297                 if (!igb_check_vf_assignment(adapter)) {
2298                         pci_disable_sriov(pdev);
2299                         msleep(500);
2300                 } else {
2301                         dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2302                 }
2303
2304                 kfree(adapter->vf_data);
2305                 adapter->vf_data = NULL;
2306                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2307                 wrfl();
2308                 msleep(100);
2309                 dev_info(&pdev->dev, "IOV Disabled\n");
2310         }
2311 #endif
2312
2313         iounmap(hw->hw_addr);
2314         if (hw->flash_address)
2315                 iounmap(hw->flash_address);
2316         pci_release_selected_regions(pdev,
2317                                      pci_select_bars(pdev, IORESOURCE_MEM));
2318
2319         kfree(adapter->shadow_vfta);
2320         free_netdev(netdev);
2321
2322         pci_disable_pcie_error_reporting(pdev);
2323
2324         pci_disable_device(pdev);
2325 }
2326
2327 /**
2328  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2329  * @adapter: board private structure to initialize
2330  *
2331  * This function initializes the vf specific data storage and then attempts to
2332  * allocate the VFs.  The reason for ordering it this way is because it is much
2333  * mor expensive time wise to disable SR-IOV than it is to allocate and free
2334  * the memory for the VFs.
2335  **/
2336 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2337 {
2338 #ifdef CONFIG_PCI_IOV
2339         struct pci_dev *pdev = adapter->pdev;
2340         struct e1000_hw *hw = &adapter->hw;
2341         int old_vfs = igb_find_enabled_vfs(adapter);
2342         int i;
2343
2344         /* Virtualization features not supported on i210 family. */
2345         if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2346                 return;
2347
2348         if (old_vfs) {
2349                 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2350                          "max_vfs setting of %d\n", old_vfs, max_vfs);
2351                 adapter->vfs_allocated_count = old_vfs;
2352         }
2353
2354         if (!adapter->vfs_allocated_count)
2355                 return;
2356
2357         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2358                                 sizeof(struct vf_data_storage), GFP_KERNEL);
2359
2360         /* if allocation failed then we do not support SR-IOV */
2361         if (!adapter->vf_data) {
2362                 adapter->vfs_allocated_count = 0;
2363                 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2364                         "Data Storage\n");
2365                 goto out;
2366         }
2367
2368         if (!old_vfs) {
2369                 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2370                         goto err_out;
2371         }
2372         dev_info(&pdev->dev, "%d VFs allocated\n",
2373                  adapter->vfs_allocated_count);
2374         for (i = 0; i < adapter->vfs_allocated_count; i++)
2375                 igb_vf_configure(adapter, i);
2376
2377         /* DMA Coalescing is not supported in IOV mode. */
2378         adapter->flags &= ~IGB_FLAG_DMAC;
2379         goto out;
2380 err_out:
2381         kfree(adapter->vf_data);
2382         adapter->vf_data = NULL;
2383         adapter->vfs_allocated_count = 0;
2384 out:
2385         return;
2386 #endif /* CONFIG_PCI_IOV */
2387 }
2388
2389 /**
2390  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2391  * @adapter: board private structure to initialize
2392  *
2393  * igb_sw_init initializes the Adapter private data structure.
2394  * Fields are initialized based on PCI device information and
2395  * OS network device settings (MTU size).
2396  **/
2397 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2398 {
2399         struct e1000_hw *hw = &adapter->hw;
2400         struct net_device *netdev = adapter->netdev;
2401         struct pci_dev *pdev = adapter->pdev;
2402         u32 max_rss_queues;
2403
2404         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2405
2406         /* set default ring sizes */
2407         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2408         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2409
2410         /* set default ITR values */
2411         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2412         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2413
2414         /* set default work limits */
2415         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2416
2417         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2418                                   VLAN_HLEN;
2419         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2420
2421         adapter->node = -1;
2422
2423         spin_lock_init(&adapter->stats64_lock);
2424 #ifdef CONFIG_PCI_IOV
2425         switch (hw->mac.type) {
2426         case e1000_82576:
2427         case e1000_i350:
2428                 if (max_vfs > 7) {
2429                         dev_warn(&pdev->dev,
2430                                  "Maximum of 7 VFs per PF, using max\n");
2431                         adapter->vfs_allocated_count = 7;
2432                 } else
2433                         adapter->vfs_allocated_count = max_vfs;
2434                 break;
2435         default:
2436                 break;
2437         }
2438 #endif /* CONFIG_PCI_IOV */
2439
2440         /* Determine the maximum number of RSS queues supported. */
2441         switch (hw->mac.type) {
2442         case e1000_i211:
2443                 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2444                 break;
2445         case e1000_82575:
2446         case e1000_i210:
2447                 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2448                 break;
2449         case e1000_i350:
2450                 /* I350 cannot do RSS and SR-IOV at the same time */
2451                 if (!!adapter->vfs_allocated_count) {
2452                         max_rss_queues = 1;
2453                         break;
2454                 }
2455                 /* fall through */
2456         case e1000_82576:
2457                 if (!!adapter->vfs_allocated_count) {
2458                         max_rss_queues = 2;
2459                         break;
2460                 }
2461                 /* fall through */
2462         case e1000_82580:
2463         default:
2464                 max_rss_queues = IGB_MAX_RX_QUEUES;
2465                 break;
2466         }
2467
2468         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2469
2470         /* Determine if we need to pair queues. */
2471         switch (hw->mac.type) {
2472         case e1000_82575:
2473         case e1000_i211:
2474                 /* Device supports enough interrupts without queue pairing. */
2475                 break;
2476         case e1000_82576:
2477                 /*
2478                  * If VFs are going to be allocated with RSS queues then we
2479                  * should pair the queues in order to conserve interrupts due
2480                  * to limited supply.
2481                  */
2482                 if ((adapter->rss_queues > 1) &&
2483                     (adapter->vfs_allocated_count > 6))
2484                         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2485                 /* fall through */
2486         case e1000_82580:
2487         case e1000_i350:
2488         case e1000_i210:
2489         default:
2490                 /*
2491                  * If rss_queues > half of max_rss_queues, pair the queues in
2492                  * order to conserve interrupts due to limited supply.
2493                  */
2494                 if (adapter->rss_queues > (max_rss_queues / 2))
2495                         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2496                 break;
2497         }
2498
2499         /* Setup and initialize a copy of the hw vlan table array */
2500         adapter->shadow_vfta = kzalloc(sizeof(u32) *
2501                                 E1000_VLAN_FILTER_TBL_SIZE,
2502                                 GFP_ATOMIC);
2503
2504         /* This call may decrease the number of queues */
2505         if (igb_init_interrupt_scheme(adapter)) {
2506                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2507                 return -ENOMEM;
2508         }
2509
2510         igb_probe_vfs(adapter);
2511
2512         /* Explicitly disable IRQ since the NIC can be in any state. */
2513         igb_irq_disable(adapter);
2514
2515         if (hw->mac.type >= e1000_i350)
2516                 adapter->flags &= ~IGB_FLAG_DMAC;
2517
2518         set_bit(__IGB_DOWN, &adapter->state);
2519         return 0;
2520 }
2521
2522 /**
2523  * igb_open - Called when a network interface is made active
2524  * @netdev: network interface device structure
2525  *
2526  * Returns 0 on success, negative value on failure
2527  *
2528  * The open entry point is called when a network interface is made
2529  * active by the system (IFF_UP).  At this point all resources needed
2530  * for transmit and receive operations are allocated, the interrupt
2531  * handler is registered with the OS, the watchdog timer is started,
2532  * and the stack is notified that the interface is ready.
2533  **/
2534 static int __igb_open(struct net_device *netdev, bool resuming)
2535 {
2536         struct igb_adapter *adapter = netdev_priv(netdev);
2537         struct e1000_hw *hw = &adapter->hw;
2538         struct pci_dev *pdev = adapter->pdev;
2539         int err;
2540         int i;
2541
2542         /* disallow open during test */
2543         if (test_bit(__IGB_TESTING, &adapter->state)) {
2544                 WARN_ON(resuming);
2545                 return -EBUSY;
2546         }
2547
2548         if (!resuming)
2549                 pm_runtime_get_sync(&pdev->dev);
2550
2551         netif_carrier_off(netdev);
2552
2553         /* allocate transmit descriptors */
2554         err = igb_setup_all_tx_resources(adapter);
2555         if (err)
2556                 goto err_setup_tx;
2557
2558         /* allocate receive descriptors */
2559         err = igb_setup_all_rx_resources(adapter);
2560         if (err)
2561                 goto err_setup_rx;
2562
2563         igb_power_up_link(adapter);
2564
2565         /* before we allocate an interrupt, we must be ready to handle it.
2566          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2567          * as soon as we call pci_request_irq, so we have to setup our
2568          * clean_rx handler before we do so.  */
2569         igb_configure(adapter);
2570
2571         err = igb_request_irq(adapter);
2572         if (err)
2573                 goto err_req_irq;
2574
2575         /* From here on the code is the same as igb_up() */
2576         clear_bit(__IGB_DOWN, &adapter->state);
2577
2578         for (i = 0; i < adapter->num_q_vectors; i++)
2579                 napi_enable(&(adapter->q_vector[i]->napi));
2580
2581         /* Clear any pending interrupts. */
2582         rd32(E1000_ICR);
2583
2584         igb_irq_enable(adapter);
2585
2586         /* notify VFs that reset has been completed */
2587         if (adapter->vfs_allocated_count) {
2588                 u32 reg_data = rd32(E1000_CTRL_EXT);
2589                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2590                 wr32(E1000_CTRL_EXT, reg_data);
2591         }
2592
2593         netif_tx_start_all_queues(netdev);
2594
2595         if (!resuming)
2596                 pm_runtime_put(&pdev->dev);
2597
2598         /* start the watchdog. */
2599         hw->mac.get_link_status = 1;
2600         schedule_work(&adapter->watchdog_task);
2601
2602         return 0;
2603
2604 err_req_irq:
2605         igb_release_hw_control(adapter);
2606         igb_power_down_link(adapter);
2607         igb_free_all_rx_resources(adapter);
2608 err_setup_rx:
2609         igb_free_all_tx_resources(adapter);
2610 err_setup_tx:
2611         igb_reset(adapter);
2612         if (!resuming)
2613                 pm_runtime_put(&pdev->dev);
2614
2615         return err;
2616 }
2617
2618 static int igb_open(struct net_device *netdev)
2619 {
2620         return __igb_open(netdev, false);
2621 }
2622
2623 /**
2624  * igb_close - Disables a network interface
2625  * @netdev: network interface device structure
2626  *
2627  * Returns 0, this is not allowed to fail
2628  *
2629  * The close entry point is called when an interface is de-activated
2630  * by the OS.  The hardware is still under the driver's control, but
2631  * needs to be disabled.  A global MAC reset is issued to stop the
2632  * hardware, and all transmit and receive resources are freed.
2633  **/
2634 static int __igb_close(struct net_device *netdev, bool suspending)
2635 {
2636         struct igb_adapter *adapter = netdev_priv(netdev);
2637         struct pci_dev *pdev = adapter->pdev;
2638
2639         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2640
2641         if (!suspending)
2642                 pm_runtime_get_sync(&pdev->dev);
2643
2644         igb_down(adapter);
2645         igb_free_irq(adapter);
2646
2647         igb_free_all_tx_resources(adapter);
2648         igb_free_all_rx_resources(adapter);
2649
2650         if (!suspending)
2651                 pm_runtime_put_sync(&pdev->dev);
2652         return 0;
2653 }
2654
2655 static int igb_close(struct net_device *netdev)
2656 {
2657         return __igb_close(netdev, false);
2658 }
2659
2660 /**
2661  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2662  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2663  *
2664  * Return 0 on success, negative on failure
2665  **/
2666 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2667 {
2668         struct device *dev = tx_ring->dev;
2669         int orig_node = dev_to_node(dev);
2670         int size;
2671
2672         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2673         tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2674         if (!tx_ring->tx_buffer_info)
2675                 tx_ring->tx_buffer_info = vzalloc(size);
2676         if (!tx_ring->tx_buffer_info)
2677                 goto err;
2678
2679         /* round up to nearest 4K */
2680         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2681         tx_ring->size = ALIGN(tx_ring->size, 4096);
2682
2683         set_dev_node(dev, tx_ring->numa_node);
2684         tx_ring->desc = dma_alloc_coherent(dev,
2685                                            tx_ring->size,
2686                                            &tx_ring->dma,
2687                                            GFP_KERNEL);
2688         set_dev_node(dev, orig_node);
2689         if (!tx_ring->desc)
2690                 tx_ring->desc = dma_alloc_coherent(dev,
2691                                                    tx_ring->size,
2692                                                    &tx_ring->dma,
2693                                                    GFP_KERNEL);
2694
2695         if (!tx_ring->desc)
2696                 goto err;
2697
2698         tx_ring->next_to_use = 0;
2699         tx_ring->next_to_clean = 0;
2700
2701         return 0;
2702
2703 err:
2704         vfree(tx_ring->tx_buffer_info);
2705         dev_err(dev,
2706                 "Unable to allocate memory for the transmit descriptor ring\n");
2707         return -ENOMEM;
2708 }
2709
2710 /**
2711  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2712  *                                (Descriptors) for all queues
2713  * @adapter: board private structure
2714  *
2715  * Return 0 on success, negative on failure
2716  **/
2717 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2718 {
2719         struct pci_dev *pdev = adapter->pdev;
2720         int i, err = 0;
2721
2722         for (i = 0; i < adapter->num_tx_queues; i++) {
2723                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2724                 if (err) {
2725                         dev_err(&pdev->dev,
2726                                 "Allocation for Tx Queue %u failed\n", i);
2727                         for (i--; i >= 0; i--)
2728                                 igb_free_tx_resources(adapter->tx_ring[i]);
2729                         break;
2730                 }
2731         }
2732
2733         return err;
2734 }
2735
2736 /**
2737  * igb_setup_tctl - configure the transmit control registers
2738  * @adapter: Board private structure
2739  **/
2740 void igb_setup_tctl(struct igb_adapter *adapter)
2741 {
2742         struct e1000_hw *hw = &adapter->hw;
2743         u32 tctl;
2744
2745         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2746         wr32(E1000_TXDCTL(0), 0);
2747
2748         /* Program the Transmit Control Register */
2749         tctl = rd32(E1000_TCTL);
2750         tctl &= ~E1000_TCTL_CT;
2751         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2752                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2753
2754         igb_config_collision_dist(hw);
2755
2756         /* Enable transmits */
2757         tctl |= E1000_TCTL_EN;
2758
2759         wr32(E1000_TCTL, tctl);
2760 }
2761
2762 /**
2763  * igb_configure_tx_ring - Configure transmit ring after Reset
2764  * @adapter: board private structure
2765  * @ring: tx ring to configure
2766  *
2767  * Configure a transmit ring after a reset.
2768  **/
2769 void igb_configure_tx_ring(struct igb_adapter *adapter,
2770                            struct igb_ring *ring)
2771 {
2772         struct e1000_hw *hw = &adapter->hw;
2773         u32 txdctl = 0;
2774         u64 tdba = ring->dma;
2775         int reg_idx = ring->reg_idx;
2776
2777         /* disable the queue */
2778         wr32(E1000_TXDCTL(reg_idx), 0);
2779         wrfl();
2780         mdelay(10);
2781
2782         wr32(E1000_TDLEN(reg_idx),
2783                         ring->count * sizeof(union e1000_adv_tx_desc));
2784         wr32(E1000_TDBAL(reg_idx),
2785                         tdba & 0x00000000ffffffffULL);
2786         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2787
2788         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2789         wr32(E1000_TDH(reg_idx), 0);
2790         writel(0, ring->tail);
2791
2792         txdctl |= IGB_TX_PTHRESH;
2793         txdctl |= IGB_TX_HTHRESH << 8;
2794         txdctl |= IGB_TX_WTHRESH << 16;
2795
2796         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2797         wr32(E1000_TXDCTL(reg_idx), txdctl);
2798 }
2799
2800 /**
2801  * igb_configure_tx - Configure transmit Unit after Reset
2802  * @adapter: board private structure
2803  *
2804  * Configure the Tx unit of the MAC after a reset.
2805  **/
2806 static void igb_configure_tx(struct igb_adapter *adapter)
2807 {
2808         int i;
2809
2810         for (i = 0; i < adapter->num_tx_queues; i++)
2811                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2812 }
2813
2814 /**
2815  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2816  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2817  *
2818  * Returns 0 on success, negative on failure
2819  **/
2820 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2821 {
2822         struct device *dev = rx_ring->dev;
2823         int orig_node = dev_to_node(dev);
2824         int size, desc_len;
2825
2826         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2827         rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2828         if (!rx_ring->rx_buffer_info)
2829                 rx_ring->rx_buffer_info = vzalloc(size);
2830         if (!rx_ring->rx_buffer_info)
2831                 goto err;
2832
2833         desc_len = sizeof(union e1000_adv_rx_desc);
2834
2835         /* Round up to nearest 4K */
2836         rx_ring->size = rx_ring->count * desc_len;
2837         rx_ring->size = ALIGN(rx_ring->size, 4096);
2838
2839         set_dev_node(dev, rx_ring->numa_node);
2840         rx_ring->desc = dma_alloc_coherent(dev,
2841                                            rx_ring->size,
2842                                            &rx_ring->dma,
2843                                            GFP_KERNEL);
2844         set_dev_node(dev, orig_node);
2845         if (!rx_ring->desc)
2846                 rx_ring->desc = dma_alloc_coherent(dev,
2847                                                    rx_ring->size,
2848                                                    &rx_ring->dma,
2849                                                    GFP_KERNEL);
2850
2851         if (!rx_ring->desc)
2852                 goto err;
2853
2854         rx_ring->next_to_clean = 0;
2855         rx_ring->next_to_use = 0;
2856
2857         return 0;
2858
2859 err:
2860         vfree(rx_ring->rx_buffer_info);
2861         rx_ring->rx_buffer_info = NULL;
2862         dev_err(dev, "Unable to allocate memory for the receive descriptor"
2863                 " ring\n");
2864         return -ENOMEM;
2865 }
2866
2867 /**
2868  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2869  *                                (Descriptors) for all queues
2870  * @adapter: board private structure
2871  *
2872  * Return 0 on success, negative on failure
2873  **/
2874 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2875 {
2876         struct pci_dev *pdev = adapter->pdev;
2877         int i, err = 0;
2878
2879         for (i = 0; i < adapter->num_rx_queues; i++) {
2880                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2881                 if (err) {
2882                         dev_err(&pdev->dev,
2883                                 "Allocation for Rx Queue %u failed\n", i);
2884                         for (i--; i >= 0; i--)
2885                                 igb_free_rx_resources(adapter->rx_ring[i]);
2886                         break;
2887                 }
2888         }
2889
2890         return err;
2891 }
2892
2893 /**
2894  * igb_setup_mrqc - configure the multiple receive queue control registers
2895  * @adapter: Board private structure
2896  **/
2897 static void igb_setup_mrqc(struct igb_adapter *adapter)
2898 {
2899         struct e1000_hw *hw = &adapter->hw;
2900         u32 mrqc, rxcsum;
2901         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2902         union e1000_reta {
2903                 u32 dword;
2904                 u8  bytes[4];
2905         } reta;
2906         static const u8 rsshash[40] = {
2907                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2908                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2909                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2910                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2911
2912         /* Fill out hash function seeds */
2913         for (j = 0; j < 10; j++) {
2914                 u32 rsskey = rsshash[(j * 4)];
2915                 rsskey |= rsshash[(j * 4) + 1] << 8;
2916                 rsskey |= rsshash[(j * 4) + 2] << 16;
2917                 rsskey |= rsshash[(j * 4) + 3] << 24;
2918                 array_wr32(E1000_RSSRK(0), j, rsskey);
2919         }
2920
2921         num_rx_queues = adapter->rss_queues;
2922
2923         if (adapter->vfs_allocated_count) {
2924                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2925                 switch (hw->mac.type) {
2926                 case e1000_i350:
2927                 case e1000_82580:
2928                         num_rx_queues = 1;
2929                         shift = 0;
2930                         break;
2931                 case e1000_82576:
2932                         shift = 3;
2933                         num_rx_queues = 2;
2934                         break;
2935                 case e1000_82575:
2936                         shift = 2;
2937                         shift2 = 6;
2938                 default:
2939                         break;
2940                 }
2941         } else {
2942                 if (hw->mac.type == e1000_82575)
2943                         shift = 6;
2944         }
2945
2946         for (j = 0; j < (32 * 4); j++) {
2947                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2948                 if (shift2)
2949                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2950                 if ((j & 3) == 3)
2951                         wr32(E1000_RETA(j >> 2), reta.dword);
2952         }
2953
2954         /*
2955          * Disable raw packet checksumming so that RSS hash is placed in
2956          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2957          * offloads as they are enabled by default
2958          */
2959         rxcsum = rd32(E1000_RXCSUM);
2960         rxcsum |= E1000_RXCSUM_PCSD;
2961
2962         if (adapter->hw.mac.type >= e1000_82576)
2963                 /* Enable Receive Checksum Offload for SCTP */
2964                 rxcsum |= E1000_RXCSUM_CRCOFL;
2965
2966         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2967         wr32(E1000_RXCSUM, rxcsum);
2968         /*
2969          * Generate RSS hash based on TCP port numbers and/or
2970          * IPv4/v6 src and dst addresses since UDP cannot be
2971          * hashed reliably due to IP fragmentation
2972          */
2973
2974         mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2975                E1000_MRQC_RSS_FIELD_IPV4_TCP |
2976                E1000_MRQC_RSS_FIELD_IPV6 |
2977                E1000_MRQC_RSS_FIELD_IPV6_TCP |
2978                E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2979
2980         /* If VMDq is enabled then we set the appropriate mode for that, else
2981          * we default to RSS so that an RSS hash is calculated per packet even
2982          * if we are only using one queue */
2983         if (adapter->vfs_allocated_count) {
2984                 if (hw->mac.type > e1000_82575) {
2985                         /* Set the default pool for the PF's first queue */
2986                         u32 vtctl = rd32(E1000_VT_CTL);
2987                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2988                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2989                         vtctl |= adapter->vfs_allocated_count <<
2990                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2991                         wr32(E1000_VT_CTL, vtctl);
2992                 }
2993                 if (adapter->rss_queues > 1)
2994                         mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2995                 else
2996                         mrqc |= E1000_MRQC_ENABLE_VMDQ;
2997         } else {
2998                 if (hw->mac.type != e1000_i211)
2999                         mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
3000         }
3001         igb_vmm_control(adapter);
3002
3003         wr32(E1000_MRQC, mrqc);
3004 }
3005
3006 /**
3007  * igb_setup_rctl - configure the receive control registers
3008  * @adapter: Board private structure
3009  **/
3010 void igb_setup_rctl(struct igb_adapter *adapter)
3011 {
3012         struct e1000_hw *hw = &adapter->hw;
3013         u32 rctl;
3014
3015         rctl = rd32(E1000_RCTL);
3016
3017         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3018         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3019
3020         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3021                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3022
3023         /*
3024          * enable stripping of CRC. It's unlikely this will break BMC
3025          * redirection as it did with e1000. Newer features require
3026          * that the HW strips the CRC.
3027          */
3028         rctl |= E1000_RCTL_SECRC;
3029
3030         /* disable store bad packets and clear size bits. */
3031         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3032
3033         /* enable LPE to prevent packets larger than max_frame_size */
3034         rctl |= E1000_RCTL_LPE;
3035
3036         /* disable queue 0 to prevent tail write w/o re-config */
3037         wr32(E1000_RXDCTL(0), 0);
3038
3039         /* Attention!!!  For SR-IOV PF driver operations you must enable
3040          * queue drop for all VF and PF queues to prevent head of line blocking
3041          * if an un-trusted VF does not provide descriptors to hardware.
3042          */
3043         if (adapter->vfs_allocated_count) {
3044                 /* set all queue drop enable bits */
3045                 wr32(E1000_QDE, ALL_QUEUES);
3046         }
3047
3048         /* This is useful for sniffing bad packets. */
3049         if (adapter->netdev->features & NETIF_F_RXALL) {
3050                 /* UPE and MPE will be handled by normal PROMISC logic
3051                  * in e1000e_set_rx_mode */
3052                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3053                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
3054                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3055
3056                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3057                           E1000_RCTL_DPF | /* Allow filtered pause */
3058                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3059                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3060                  * and that breaks VLANs.
3061                  */
3062         }
3063
3064         wr32(E1000_RCTL, rctl);
3065 }
3066
3067 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3068                                    int vfn)
3069 {
3070         struct e1000_hw *hw = &adapter->hw;
3071         u32 vmolr;
3072
3073         /* if it isn't the PF check to see if VFs are enabled and
3074          * increase the size to support vlan tags */
3075         if (vfn < adapter->vfs_allocated_count &&
3076             adapter->vf_data[vfn].vlans_enabled)
3077                 size += VLAN_TAG_SIZE;
3078
3079         vmolr = rd32(E1000_VMOLR(vfn));
3080         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3081         vmolr |= size | E1000_VMOLR_LPE;
3082         wr32(E1000_VMOLR(vfn), vmolr);
3083
3084         return 0;
3085 }
3086
3087 /**
3088  * igb_rlpml_set - set maximum receive packet size
3089  * @adapter: board private structure
3090  *
3091  * Configure maximum receivable packet size.
3092  **/
3093 static void igb_rlpml_set(struct igb_adapter *adapter)
3094 {
3095         u32 max_frame_size = adapter->max_frame_size;
3096         struct e1000_hw *hw = &adapter->hw;
3097         u16 pf_id = adapter->vfs_allocated_count;
3098
3099         if (pf_id) {
3100                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3101                 /*
3102                  * If we're in VMDQ or SR-IOV mode, then set global RLPML
3103                  * to our max jumbo frame size, in case we need to enable
3104                  * jumbo frames on one of the rings later.
3105                  * This will not pass over-length frames into the default
3106                  * queue because it's gated by the VMOLR.RLPML.
3107                  */
3108                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3109         }
3110
3111         wr32(E1000_RLPML, max_frame_size);
3112 }
3113
3114 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3115                                  int vfn, bool aupe)
3116 {
3117         struct e1000_hw *hw = &adapter->hw;
3118         u32 vmolr;
3119
3120         /*
3121          * This register exists only on 82576 and newer so if we are older then
3122          * we should exit and do nothing
3123          */
3124         if (hw->mac.type < e1000_82576)
3125                 return;
3126
3127         vmolr = rd32(E1000_VMOLR(vfn));
3128         vmolr |= E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
3129         if (aupe)
3130                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
3131         else
3132                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3133
3134         /* clear all bits that might not be set */
3135         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3136
3137         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3138                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3139         /*
3140          * for VMDq only allow the VFs and pool 0 to accept broadcast and
3141          * multicast packets
3142          */
3143         if (vfn <= adapter->vfs_allocated_count)
3144                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
3145
3146         wr32(E1000_VMOLR(vfn), vmolr);
3147 }
3148
3149 /**
3150  * igb_configure_rx_ring - Configure a receive ring after Reset
3151  * @adapter: board private structure
3152  * @ring: receive ring to be configured
3153  *
3154  * Configure the Rx unit of the MAC after a reset.
3155  **/
3156 void igb_configure_rx_ring(struct igb_adapter *adapter,
3157                            struct igb_ring *ring)
3158 {
3159         struct e1000_hw *hw = &adapter->hw;
3160         u64 rdba = ring->dma;
3161         int reg_idx = ring->reg_idx;
3162         u32 srrctl = 0, rxdctl = 0;
3163
3164         /* disable the queue */
3165         wr32(E1000_RXDCTL(reg_idx), 0);
3166
3167         /* Set DMA base address registers */
3168         wr32(E1000_RDBAL(reg_idx),
3169              rdba & 0x00000000ffffffffULL);
3170         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3171         wr32(E1000_RDLEN(reg_idx),
3172                        ring->count * sizeof(union e1000_adv_rx_desc));
3173
3174         /* initialize head and tail */
3175         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3176         wr32(E1000_RDH(reg_idx), 0);
3177         writel(0, ring->tail);
3178
3179         /* set descriptor configuration */
3180         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3181 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3182         srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3183 #else
3184         srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3185 #endif
3186         srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3187         if (hw->mac.type >= e1000_82580)
3188                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3189         /* Only set Drop Enable if we are supporting multiple queues */
3190         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3191                 srrctl |= E1000_SRRCTL_DROP_EN;
3192
3193         wr32(E1000_SRRCTL(reg_idx), srrctl);
3194
3195         /* set filtering for VMDQ pools */
3196         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3197
3198         rxdctl |= IGB_RX_PTHRESH;
3199         rxdctl |= IGB_RX_HTHRESH << 8;
3200         rxdctl |= IGB_RX_WTHRESH << 16;
3201
3202         /* enable receive descriptor fetching */
3203         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3204         wr32(E1000_RXDCTL(reg_idx), rxdctl);
3205 }
3206
3207 /**
3208  * igb_configure_rx - Configure receive Unit after Reset
3209  * @adapter: board private structure
3210  *
3211  * Configure the Rx unit of the MAC after a reset.
3212  **/
3213 static void igb_configure_rx(struct igb_adapter *adapter)
3214 {
3215         int i;
3216
3217         /* set UTA to appropriate mode */
3218         igb_set_uta(adapter);
3219
3220         /* set the correct pool for the PF default MAC address in entry 0 */
3221         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3222                          adapter->vfs_allocated_count);
3223
3224         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3225          * the Base and Length of the Rx Descriptor Ring */
3226         for (i = 0; i < adapter->num_rx_queues; i++)
3227                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3228 }
3229
3230 /**
3231  * igb_free_tx_resources - Free Tx Resources per Queue
3232  * @tx_ring: Tx descriptor ring for a specific queue
3233  *
3234  * Free all transmit software resources
3235  **/
3236 void igb_free_tx_resources(struct igb_ring *tx_ring)
3237 {
3238         igb_clean_tx_ring(tx_ring);
3239
3240         vfree(tx_ring->tx_buffer_info);
3241         tx_ring->tx_buffer_info = NULL;
3242
3243         /* if not set, then don't free */
3244         if (!tx_ring->desc)
3245                 return;
3246
3247         dma_free_coherent(tx_ring->dev, tx_ring->size,
3248                           tx_ring->desc, tx_ring->dma);
3249
3250         tx_ring->desc = NULL;
3251 }
3252
3253 /**
3254  * igb_free_all_tx_resources - Free Tx Resources for All Queues
3255  * @adapter: board private structure
3256  *
3257  * Free all transmit software resources
3258  **/
3259 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3260 {
3261         int i;
3262
3263         for (i = 0; i < adapter->num_tx_queues; i++)
3264                 igb_free_tx_resources(adapter->tx_ring[i]);
3265 }
3266
3267 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3268                                     struct igb_tx_buffer *tx_buffer)
3269 {
3270         if (tx_buffer->skb) {
3271                 dev_kfree_skb_any(tx_buffer->skb);
3272                 if (tx_buffer->dma)
3273                         dma_unmap_single(ring->dev,
3274                                          tx_buffer->dma,
3275                                          tx_buffer->length,
3276                                          DMA_TO_DEVICE);
3277         } else if (tx_buffer->dma) {
3278                 dma_unmap_page(ring->dev,
3279                                tx_buffer->dma,
3280                                tx_buffer->length,
3281                                DMA_TO_DEVICE);
3282         }
3283         tx_buffer->next_to_watch = NULL;
3284         tx_buffer->skb = NULL;
3285         tx_buffer->dma = 0;
3286         /* buffer_info must be completely set up in the transmit path */
3287 }
3288
3289 /**
3290  * igb_clean_tx_ring - Free Tx Buffers
3291  * @tx_ring: ring to be cleaned
3292  **/
3293 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3294 {
3295         struct igb_tx_buffer *buffer_info;
3296         unsigned long size;
3297         u16 i;
3298
3299         if (!tx_ring->tx_buffer_info)
3300                 return;
3301         /* Free all the Tx ring sk_buffs */
3302
3303         for (i = 0; i < tx_ring->count; i++) {
3304                 buffer_info = &tx_ring->tx_buffer_info[i];
3305                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3306         }
3307
3308         netdev_tx_reset_queue(txring_txq(tx_ring));
3309
3310         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3311         memset(tx_ring->tx_buffer_info, 0, size);
3312
3313         /* Zero out the descriptor ring */
3314         memset(tx_ring->desc, 0, tx_ring->size);
3315
3316         tx_ring->next_to_use = 0;
3317         tx_ring->next_to_clean = 0;
3318 }
3319
3320 /**
3321  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3322  * @adapter: board private structure
3323  **/
3324 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3325 {
3326         int i;
3327
3328         for (i = 0; i < adapter->num_tx_queues; i++)
3329                 igb_clean_tx_ring(adapter->tx_ring[i]);
3330 }
3331
3332 /**
3333  * igb_free_rx_resources - Free Rx Resources
3334  * @rx_ring: ring to clean the resources from
3335  *
3336  * Free all receive software resources
3337  **/
3338 void igb_free_rx_resources(struct igb_ring *rx_ring)
3339 {
3340         igb_clean_rx_ring(rx_ring);
3341
3342         vfree(rx_ring->rx_buffer_info);
3343         rx_ring->rx_buffer_info = NULL;
3344
3345         /* if not set, then don't free */
3346         if (!rx_ring->desc)
3347                 return;
3348
3349         dma_free_coherent(rx_ring->dev, rx_ring->size,
3350                           rx_ring->desc, rx_ring->dma);
3351
3352         rx_ring->desc = NULL;
3353 }
3354
3355 /**
3356  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3357  * @adapter: board private structure
3358  *
3359  * Free all receive software resources
3360  **/
3361 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3362 {
3363         int i;
3364
3365         for (i = 0; i < adapter->num_rx_queues; i++)
3366                 igb_free_rx_resources(adapter->rx_ring[i]);
3367 }
3368
3369 /**
3370  * igb_clean_rx_ring - Free Rx Buffers per Queue
3371  * @rx_ring: ring to free buffers from
3372  **/
3373 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3374 {
3375         unsigned long size;
3376         u16 i;
3377
3378         if (!rx_ring->rx_buffer_info)
3379                 return;
3380
3381         /* Free all the Rx ring sk_buffs */
3382         for (i = 0; i < rx_ring->count; i++) {
3383                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3384                 if (buffer_info->dma) {
3385                         dma_unmap_single(rx_ring->dev,
3386                                          buffer_info->dma,
3387                                          IGB_RX_HDR_LEN,
3388                                          DMA_FROM_DEVICE);
3389                         buffer_info->dma = 0;
3390                 }
3391
3392                 if (buffer_info->skb) {
3393                         dev_kfree_skb(buffer_info->skb);
3394                         buffer_info->skb = NULL;
3395                 }
3396                 if (buffer_info->page_dma) {
3397                         dma_unmap_page(rx_ring->dev,
3398                                        buffer_info->page_dma,
3399                                        PAGE_SIZE / 2,
3400                                        DMA_FROM_DEVICE);
3401                         buffer_info->page_dma = 0;
3402                 }
3403                 if (buffer_info->page) {
3404                         put_page(buffer_info->page);
3405                         buffer_info->page = NULL;
3406                         buffer_info->page_offset = 0;
3407                 }
3408         }
3409
3410         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3411         memset(rx_ring->rx_buffer_info, 0, size);
3412
3413         /* Zero out the descriptor ring */
3414         memset(rx_ring->desc, 0, rx_ring->size);
3415
3416         rx_ring->next_to_clean = 0;
3417         rx_ring->next_to_use = 0;
3418 }
3419
3420 /**
3421  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3422  * @adapter: board private structure
3423  **/
3424 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3425 {
3426         int i;
3427
3428         for (i = 0; i < adapter->num_rx_queues; i++)
3429                 igb_clean_rx_ring(adapter->rx_ring[i]);
3430 }
3431
3432 /**
3433  * igb_set_mac - Change the Ethernet Address of the NIC
3434  * @netdev: network interface device structure
3435  * @p: pointer to an address structure
3436  *
3437  * Returns 0 on success, negative on failure
3438  **/
3439 static int igb_set_mac(struct net_device *netdev, void *p)
3440 {
3441         struct igb_adapter *adapter = netdev_priv(netdev);
3442         struct e1000_hw *hw = &adapter->hw;
3443         struct sockaddr *addr = p;
3444
3445         if (!is_valid_ether_addr(addr->sa_data))
3446                 return -EADDRNOTAVAIL;
3447
3448         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3449         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3450
3451         /* set the correct pool for the new PF MAC address in entry 0 */
3452         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3453                          adapter->vfs_allocated_count);
3454
3455         return 0;
3456 }
3457
3458 /**
3459  * igb_write_mc_addr_list - write multicast addresses to MTA
3460  * @netdev: network interface device structure
3461  *
3462  * Writes multicast address list to the MTA hash table.
3463  * Returns: -ENOMEM on failure
3464  *                0 on no addresses written
3465  *                X on writing X addresses to MTA
3466  **/
3467 static int igb_write_mc_addr_list(struct net_device *netdev)
3468 {
3469         struct igb_adapter *adapter = netdev_priv(netdev);
3470         struct e1000_hw *hw = &adapter->hw;
3471         struct netdev_hw_addr *ha;
3472         u8  *mta_list;
3473         int i;
3474
3475         if (netdev_mc_empty(netdev)) {
3476                 /* nothing to program, so clear mc list */
3477                 igb_update_mc_addr_list(hw, NULL, 0);
3478                 igb_restore_vf_multicasts(adapter);
3479                 return 0;
3480         }
3481
3482         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3483         if (!mta_list)
3484                 return -ENOMEM;
3485
3486         /* The shared function expects a packed array of only addresses. */
3487         i = 0;
3488         netdev_for_each_mc_addr(ha, netdev)
3489                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3490
3491         igb_update_mc_addr_list(hw, mta_list, i);
3492         kfree(mta_list);
3493
3494         return netdev_mc_count(netdev);
3495 }
3496
3497 /**
3498  * igb_write_uc_addr_list - write unicast addresses to RAR table
3499  * @netdev: network interface device structure
3500  *
3501  * Writes unicast address list to the RAR table.
3502  * Returns: -ENOMEM on failure/insufficient address space
3503  *                0 on no addresses written
3504  *                X on writing X addresses to the RAR table
3505  **/
3506 static int igb_write_uc_addr_list(struct net_device *netdev)
3507 {
3508         struct igb_adapter *adapter = netdev_priv(netdev);
3509         struct e1000_hw *hw = &adapter->hw;
3510         unsigned int vfn = adapter->vfs_allocated_count;
3511         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3512         int count = 0;
3513
3514         /* return ENOMEM indicating insufficient memory for addresses */
3515         if (netdev_uc_count(netdev) > rar_entries)
3516                 return -ENOMEM;
3517
3518         if (!netdev_uc_empty(netdev) && rar_entries) {
3519                 struct netdev_hw_addr *ha;
3520
3521                 netdev_for_each_uc_addr(ha, netdev) {
3522                         if (!rar_entries)
3523                                 break;
3524                         igb_rar_set_qsel(adapter, ha->addr,
3525                                          rar_entries--,
3526                                          vfn);
3527                         count++;
3528                 }
3529         }
3530         /* write the addresses in reverse order to avoid write combining */
3531         for (; rar_entries > 0 ; rar_entries--) {
3532                 wr32(E1000_RAH(rar_entries), 0);
3533                 wr32(E1000_RAL(rar_entries), 0);
3534         }
3535         wrfl();
3536
3537         return count;
3538 }
3539
3540 /**
3541  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3542  * @netdev: network interface device structure
3543  *
3544  * The set_rx_mode entry point is called whenever the unicast or multicast
3545  * address lists or the network interface flags are updated.  This routine is
3546  * responsible for configuring the hardware for proper unicast, multicast,
3547  * promiscuous mode, and all-multi behavior.
3548  **/
3549 static void igb_set_rx_mode(struct net_device *netdev)
3550 {
3551         struct igb_adapter *adapter = netdev_priv(netdev);
3552         struct e1000_hw *hw = &adapter->hw;
3553         unsigned int vfn = adapter->vfs_allocated_count;
3554         u32 rctl, vmolr = 0;
3555         int count;
3556
3557         /* Check for Promiscuous and All Multicast modes */
3558         rctl = rd32(E1000_RCTL);
3559
3560         /* clear the effected bits */
3561         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3562
3563         if (netdev->flags & IFF_PROMISC) {
3564                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3565                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3566         } else {
3567                 if (netdev->flags & IFF_ALLMULTI) {
3568                         rctl |= E1000_RCTL_MPE;
3569                         vmolr |= E1000_VMOLR_MPME;
3570                 } else {
3571                         /*
3572                          * Write addresses to the MTA, if the attempt fails
3573                          * then we should just turn on promiscuous mode so
3574                          * that we can at least receive multicast traffic
3575                          */
3576                         count = igb_write_mc_addr_list(netdev);
3577                         if (count < 0) {
3578                                 rctl |= E1000_RCTL_MPE;
3579                                 vmolr |= E1000_VMOLR_MPME;
3580                         } else if (count) {
3581                                 vmolr |= E1000_VMOLR_ROMPE;
3582                         }
3583                 }
3584                 /*
3585                  * Write addresses to available RAR registers, if there is not
3586                  * sufficient space to store all the addresses then enable
3587                  * unicast promiscuous mode
3588                  */
3589                 count = igb_write_uc_addr_list(netdev);
3590                 if (count < 0) {
3591                         rctl |= E1000_RCTL_UPE;
3592                         vmolr |= E1000_VMOLR_ROPE;
3593                 }
3594                 rctl |= E1000_RCTL_VFE;
3595         }
3596         wr32(E1000_RCTL, rctl);
3597
3598         /*
3599          * In order to support SR-IOV and eventually VMDq it is necessary to set
3600          * the VMOLR to enable the appropriate modes.  Without this workaround
3601          * we will have issues with VLAN tag stripping not being done for frames
3602          * that are only arriving because we are the default pool
3603          */
3604         if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3605                 return;
3606
3607         vmolr |= rd32(E1000_VMOLR(vfn)) &
3608                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3609         wr32(E1000_VMOLR(vfn), vmolr);
3610         igb_restore_vf_multicasts(adapter);
3611 }
3612
3613 static void igb_check_wvbr(struct igb_adapter *adapter)
3614 {
3615         struct e1000_hw *hw = &adapter->hw;
3616         u32 wvbr = 0;
3617
3618         switch (hw->mac.type) {
3619         case e1000_82576:
3620         case e1000_i350:
3621                 if (!(wvbr = rd32(E1000_WVBR)))
3622                         return;
3623                 break;
3624         default:
3625                 break;
3626         }
3627
3628         adapter->wvbr |= wvbr;
3629 }
3630
3631 #define IGB_STAGGERED_QUEUE_OFFSET 8
3632
3633 static void igb_spoof_check(struct igb_adapter *adapter)
3634 {
3635         int j;
3636
3637         if (!adapter->wvbr)
3638                 return;
3639
3640         for(j = 0; j < adapter->vfs_allocated_count; j++) {
3641                 if (adapter->wvbr & (1 << j) ||
3642                     adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3643                         dev_warn(&adapter->pdev->dev,
3644                                 "Spoof event(s) detected on VF %d\n", j);
3645                         adapter->wvbr &=
3646                                 ~((1 << j) |
3647                                   (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3648                 }
3649         }
3650 }
3651
3652 /* Need to wait a few seconds after link up to get diagnostic information from
3653  * the phy */
3654 static void igb_update_phy_info(unsigned long data)
3655 {
3656         struct igb_adapter *adapter = (struct igb_adapter *) data;
3657         igb_get_phy_info(&adapter->hw);
3658 }
3659
3660 /**
3661  * igb_has_link - check shared code for link and determine up/down
3662  * @adapter: pointer to driver private info
3663  **/
3664 bool igb_has_link(struct igb_adapter *adapter)
3665 {
3666         struct e1000_hw *hw = &adapter->hw;
3667         bool link_active = false;
3668         s32 ret_val = 0;
3669
3670         /* get_link_status is set on LSC (link status) interrupt or
3671          * rx sequence error interrupt.  get_link_status will stay
3672          * false until the e1000_check_for_link establishes link
3673          * for copper adapters ONLY
3674          */
3675         switch (hw->phy.media_type) {
3676         case e1000_media_type_copper:
3677                 if (hw->mac.get_link_status) {
3678                         ret_val = hw->mac.ops.check_for_link(hw);
3679                         link_active = !hw->mac.get_link_status;
3680                 } else {
3681                         link_active = true;
3682                 }
3683                 break;
3684         case e1000_media_type_internal_serdes:
3685                 ret_val = hw->mac.ops.check_for_link(hw);
3686                 link_active = hw->mac.serdes_has_link;
3687                 break;
3688         default:
3689         case e1000_media_type_unknown:
3690                 break;
3691         }
3692
3693         return link_active;
3694 }
3695
3696 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3697 {
3698         bool ret = false;
3699         u32 ctrl_ext, thstat;
3700
3701         /* check for thermal sensor event on i350 copper only */
3702         if (hw->mac.type == e1000_i350) {
3703                 thstat = rd32(E1000_THSTAT);
3704                 ctrl_ext = rd32(E1000_CTRL_EXT);
3705
3706                 if ((hw->phy.media_type == e1000_media_type_copper) &&
3707                     !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3708                         ret = !!(thstat & event);
3709                 }
3710         }
3711
3712         return ret;
3713 }
3714
3715 /**
3716  * igb_watchdog - Timer Call-back
3717  * @data: pointer to adapter cast into an unsigned long
3718  **/
3719 static void igb_watchdog(unsigned long data)
3720 {
3721         struct igb_adapter *adapter = (struct igb_adapter *)data;
3722         /* Do the rest outside of interrupt context */
3723         schedule_work(&adapter->watchdog_task);
3724 }
3725
3726 static void igb_watchdog_task(struct work_struct *work)
3727 {
3728         struct igb_adapter *adapter = container_of(work,
3729                                                    struct igb_adapter,
3730                                                    watchdog_task);
3731         struct e1000_hw *hw = &adapter->hw;
3732         struct net_device *netdev = adapter->netdev;
3733         u32 link;
3734         int i;
3735
3736         link = igb_has_link(adapter);
3737         if (link) {
3738                 /* Cancel scheduled suspend requests. */
3739                 pm_runtime_resume(netdev->dev.parent);
3740
3741                 if (!netif_carrier_ok(netdev)) {
3742                         u32 ctrl;
3743                         hw->mac.ops.get_speed_and_duplex(hw,
3744                                                          &adapter->link_speed,
3745                                                          &adapter->link_duplex);
3746
3747                         ctrl = rd32(E1000_CTRL);
3748                         /* Links status message must follow this format */
3749                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3750                                "Duplex, Flow Control: %s\n",
3751                                netdev->name,
3752                                adapter->link_speed,
3753                                adapter->link_duplex == FULL_DUPLEX ?
3754                                "Full" : "Half",
3755                                (ctrl & E1000_CTRL_TFCE) &&
3756                                (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3757                                (ctrl & E1000_CTRL_RFCE) ?  "RX" :
3758                                (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
3759
3760                         /* check for thermal sensor event */
3761                         if (igb_thermal_sensor_event(hw,
3762                             E1000_THSTAT_LINK_THROTTLE)) {
3763                                 netdev_info(netdev, "The network adapter link "
3764                                             "speed was downshifted because it "
3765                                             "overheated\n");
3766                         }
3767
3768                         /* adjust timeout factor according to speed/duplex */
3769                         adapter->tx_timeout_factor = 1;
3770                         switch (adapter->link_speed) {
3771                         case SPEED_10:
3772                                 adapter->tx_timeout_factor = 14;
3773                                 break;
3774                         case SPEED_100:
3775                                 /* maybe add some timeout factor ? */
3776                                 break;
3777                         }
3778
3779                         netif_carrier_on(netdev);
3780
3781                         igb_ping_all_vfs(adapter);
3782                         igb_check_vf_rate_limit(adapter);
3783
3784                         /* link state has changed, schedule phy info update */
3785                         if (!test_bit(__IGB_DOWN, &adapter->state))
3786                                 mod_timer(&adapter->phy_info_timer,
3787                                           round_jiffies(jiffies + 2 * HZ));
3788                 }
3789         } else {
3790                 if (netif_carrier_ok(netdev)) {
3791                         adapter->link_speed = 0;
3792                         adapter->link_duplex = 0;
3793
3794                         /* check for thermal sensor event */
3795                         if (igb_thermal_sensor_event(hw,
3796                             E1000_THSTAT_PWR_DOWN)) {
3797                                 netdev_err(netdev, "The network adapter was "
3798                                            "stopped because it overheated\n");
3799                         }
3800
3801                         /* Links status message must follow this format */
3802                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3803                                netdev->name);
3804                         netif_carrier_off(netdev);
3805
3806                         igb_ping_all_vfs(adapter);
3807
3808                         /* link state has changed, schedule phy info update */
3809                         if (!test_bit(__IGB_DOWN, &adapter->state))
3810                                 mod_timer(&adapter->phy_info_timer,
3811                                           round_jiffies(jiffies + 2 * HZ));
3812
3813                         pm_schedule_suspend(netdev->dev.parent,
3814                                             MSEC_PER_SEC * 5);
3815                 }
3816         }
3817
3818         spin_lock(&adapter->stats64_lock);
3819         igb_update_stats(adapter, &adapter->stats64);
3820         spin_unlock(&adapter->stats64_lock);
3821
3822         for (i = 0; i < adapter->num_tx_queues; i++) {
3823                 struct igb_ring *tx_ring = adapter->tx_ring[i];
3824                 if (!netif_carrier_ok(netdev)) {
3825                         /* We've lost link, so the controller stops DMA,
3826                          * but we've got queued Tx work that's never going
3827                          * to get done, so reset controller to flush Tx.
3828                          * (Do the reset outside of interrupt context). */
3829                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3830                                 adapter->tx_timeout_count++;
3831                                 schedule_work(&adapter->reset_task);
3832                                 /* return immediately since reset is imminent */
3833                                 return;
3834                         }
3835                 }
3836
3837                 /* Force detection of hung controller every watchdog period */
3838                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3839         }
3840
3841         /* Cause software interrupt to ensure rx ring is cleaned */
3842         if (adapter->msix_entries) {
3843                 u32 eics = 0;
3844                 for (i = 0; i < adapter->num_q_vectors; i++)
3845                         eics |= adapter->q_vector[i]->eims_value;
3846                 wr32(E1000_EICS, eics);
3847         } else {
3848                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3849         }
3850
3851         igb_spoof_check(adapter);
3852
3853         /* Reset the timer */
3854         if (!test_bit(__IGB_DOWN, &adapter->state))
3855                 mod_timer(&adapter->watchdog_timer,
3856                           round_jiffies(jiffies + 2 * HZ));
3857 }
3858
3859 enum latency_range {
3860         lowest_latency = 0,
3861         low_latency = 1,
3862         bulk_latency = 2,
3863         latency_invalid = 255
3864 };
3865
3866 /**
3867  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3868  *
3869  *      Stores a new ITR value based on strictly on packet size.  This
3870  *      algorithm is less sophisticated than that used in igb_update_itr,
3871  *      due to the difficulty of synchronizing statistics across multiple
3872  *      receive rings.  The divisors and thresholds used by this function
3873  *      were determined based on theoretical maximum wire speed and testing
3874  *      data, in order to minimize response time while increasing bulk
3875  *      throughput.
3876  *      This functionality is controlled by the InterruptThrottleRate module
3877  *      parameter (see igb_param.c)
3878  *      NOTE:  This function is called only when operating in a multiqueue
3879  *             receive environment.
3880  * @q_vector: pointer to q_vector
3881  **/
3882 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3883 {
3884         int new_val = q_vector->itr_val;
3885         int avg_wire_size = 0;
3886         struct igb_adapter *adapter = q_vector->adapter;
3887         unsigned int packets;
3888
3889         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3890          * ints/sec - ITR timer value of 120 ticks.
3891          */
3892         if (adapter->link_speed != SPEED_1000) {
3893                 new_val = IGB_4K_ITR;
3894                 goto set_itr_val;
3895         }
3896
3897         packets = q_vector->rx.total_packets;
3898         if (packets)
3899                 avg_wire_size = q_vector->rx.total_bytes / packets;
3900
3901         packets = q_vector->tx.total_packets;
3902         if (packets)
3903                 avg_wire_size = max_t(u32, avg_wire_size,
3904                                       q_vector->tx.total_bytes / packets);
3905
3906         /* if avg_wire_size isn't set no work was done */
3907         if (!avg_wire_size)
3908                 goto clear_counts;
3909
3910         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3911         avg_wire_size += 24;
3912
3913         /* Don't starve jumbo frames */
3914         avg_wire_size = min(avg_wire_size, 3000);
3915
3916         /* Give a little boost to mid-size frames */
3917         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3918                 new_val = avg_wire_size / 3;
3919         else
3920                 new_val = avg_wire_size / 2;
3921
3922         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3923         if (new_val < IGB_20K_ITR &&
3924             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3925              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3926                 new_val = IGB_20K_ITR;
3927
3928 set_itr_val:
3929         if (new_val != q_vector->itr_val) {
3930                 q_vector->itr_val = new_val;
3931                 q_vector->set_itr = 1;
3932         }
3933 clear_counts:
3934         q_vector->rx.total_bytes = 0;
3935         q_vector->rx.total_packets = 0;
3936         q_vector->tx.total_bytes = 0;
3937         q_vector->tx.total_packets = 0;
3938 }
3939
3940 /**
3941  * igb_update_itr - update the dynamic ITR value based on statistics
3942  *      Stores a new ITR value based on packets and byte
3943  *      counts during the last interrupt.  The advantage of per interrupt
3944  *      computation is faster updates and more accurate ITR for the current
3945  *      traffic pattern.  Constants in this function were computed
3946  *      based on theoretical maximum wire speed and thresholds were set based
3947  *      on testing data as well as attempting to minimize response time
3948  *      while increasing bulk throughput.
3949  *      this functionality is controlled by the InterruptThrottleRate module
3950  *      parameter (see igb_param.c)
3951  *      NOTE:  These calculations are only valid when operating in a single-
3952  *             queue environment.
3953  * @q_vector: pointer to q_vector
3954  * @ring_container: ring info to update the itr for
3955  **/
3956 static void igb_update_itr(struct igb_q_vector *q_vector,
3957                            struct igb_ring_container *ring_container)
3958 {
3959         unsigned int packets = ring_container->total_packets;
3960         unsigned int bytes = ring_container->total_bytes;
3961         u8 itrval = ring_container->itr;
3962
3963         /* no packets, exit with status unchanged */
3964         if (packets == 0)
3965                 return;
3966
3967         switch (itrval) {
3968         case lowest_latency:
3969                 /* handle TSO and jumbo frames */
3970                 if (bytes/packets > 8000)
3971                         itrval = bulk_latency;
3972                 else if ((packets < 5) && (bytes > 512))
3973                         itrval = low_latency;
3974                 break;
3975         case low_latency:  /* 50 usec aka 20000 ints/s */
3976                 if (bytes > 10000) {
3977                         /* this if handles the TSO accounting */
3978                         if (bytes/packets > 8000) {
3979                                 itrval = bulk_latency;
3980                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3981                                 itrval = bulk_latency;
3982                         } else if ((packets > 35)) {
3983                                 itrval = lowest_latency;
3984                         }
3985                 } else if (bytes/packets > 2000) {
3986                         itrval = bulk_latency;
3987                 } else if (packets <= 2 && bytes < 512) {
3988                         itrval = lowest_latency;
3989                 }
3990                 break;
3991         case bulk_latency: /* 250 usec aka 4000 ints/s */
3992                 if (bytes > 25000) {
3993                         if (packets > 35)
3994                                 itrval = low_latency;
3995                 } else if (bytes < 1500) {
3996                         itrval = low_latency;
3997                 }
3998                 break;
3999         }
4000
4001         /* clear work counters since we have the values we need */
4002         ring_container->total_bytes = 0;
4003         ring_container->total_packets = 0;
4004
4005         /* write updated itr to ring container */
4006         ring_container->itr = itrval;
4007 }
4008
4009 static void igb_set_itr(struct igb_q_vector *q_vector)
4010 {
4011         struct igb_adapter *adapter = q_vector->adapter;
4012         u32 new_itr = q_vector->itr_val;
4013         u8 current_itr = 0;
4014
4015         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4016         if (adapter->link_speed != SPEED_1000) {
4017                 current_itr = 0;
4018                 new_itr = IGB_4K_ITR;
4019                 goto set_itr_now;
4020         }
4021
4022         igb_update_itr(q_vector, &q_vector->tx);
4023         igb_update_itr(q_vector, &q_vector->rx);
4024
4025         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4026
4027         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4028         if (current_itr == lowest_latency &&
4029             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4030              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4031                 current_itr = low_latency;
4032
4033         switch (current_itr) {
4034         /* counts and packets in update_itr are dependent on these numbers */
4035         case lowest_latency:
4036                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4037                 break;
4038         case low_latency:
4039                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4040                 break;
4041         case bulk_latency:
4042                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4043                 break;
4044         default:
4045                 break;
4046         }
4047
4048 set_itr_now:
4049         if (new_itr != q_vector->itr_val) {
4050                 /* this attempts to bias the interrupt rate towards Bulk
4051                  * by adding intermediate steps when interrupt rate is
4052                  * increasing */
4053                 new_itr = new_itr > q_vector->itr_val ?
4054                              max((new_itr * q_vector->itr_val) /
4055                                  (new_itr + (q_vector->itr_val >> 2)),
4056                                  new_itr) :
4057                              new_itr;
4058                 /* Don't write the value here; it resets the adapter's
4059                  * internal timer, and causes us to delay far longer than
4060                  * we should between interrupts.  Instead, we write the ITR
4061                  * value at the beginning of the next interrupt so the timing
4062                  * ends up being correct.
4063                  */
4064                 q_vector->itr_val = new_itr;
4065                 q_vector->set_itr = 1;
4066         }
4067 }
4068
4069 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4070                             u32 type_tucmd, u32 mss_l4len_idx)
4071 {
4072         struct e1000_adv_tx_context_desc *context_desc;
4073         u16 i = tx_ring->next_to_use;
4074
4075         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4076
4077         i++;
4078         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4079
4080         /* set bits to identify this as an advanced context descriptor */
4081         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4082
4083         /* For 82575, context index must be unique per ring. */
4084         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4085                 mss_l4len_idx |= tx_ring->reg_idx << 4;
4086
4087         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
4088         context_desc->seqnum_seed       = 0;
4089         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
4090         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
4091 }
4092
4093 static int igb_tso(struct igb_ring *tx_ring,
4094                    struct igb_tx_buffer *first,
4095                    u8 *hdr_len)
4096 {
4097         struct sk_buff *skb = first->skb;
4098         u32 vlan_macip_lens, type_tucmd;
4099         u32 mss_l4len_idx, l4len;
4100
4101         if (!skb_is_gso(skb))
4102                 return 0;
4103
4104         if (skb_header_cloned(skb)) {
4105                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4106                 if (err)
4107                         return err;
4108         }
4109
4110         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4111         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4112
4113         if (first->protocol == __constant_htons(ETH_P_IP)) {
4114                 struct iphdr *iph = ip_hdr(skb);
4115                 iph->tot_len = 0;
4116                 iph->check = 0;
4117                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4118                                                          iph->daddr, 0,
4119                                                          IPPROTO_TCP,
4120                                                          0);
4121                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4122                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4123                                    IGB_TX_FLAGS_CSUM |
4124                                    IGB_TX_FLAGS_IPV4;
4125         } else if (skb_is_gso_v6(skb)) {
4126                 ipv6_hdr(skb)->payload_len = 0;
4127                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4128                                                        &ipv6_hdr(skb)->daddr,
4129                                                        0, IPPROTO_TCP, 0);
4130                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4131                                    IGB_TX_FLAGS_CSUM;
4132         }
4133
4134         /* compute header lengths */
4135         l4len = tcp_hdrlen(skb);
4136         *hdr_len = skb_transport_offset(skb) + l4len;
4137
4138         /* update gso size and bytecount with header size */
4139         first->gso_segs = skb_shinfo(skb)->gso_segs;
4140         first->bytecount += (first->gso_segs - 1) * *hdr_len;
4141
4142         /* MSS L4LEN IDX */
4143         mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4144         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4145
4146         /* VLAN MACLEN IPLEN */
4147         vlan_macip_lens = skb_network_header_len(skb);
4148         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4149         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4150
4151         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4152
4153         return 1;
4154 }
4155
4156 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4157 {
4158         struct sk_buff *skb = first->skb;
4159         u32 vlan_macip_lens = 0;
4160         u32 mss_l4len_idx = 0;
4161         u32 type_tucmd = 0;
4162
4163         if (skb->ip_summed != CHECKSUM_PARTIAL) {
4164                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4165                         return;
4166         } else {
4167                 u8 l4_hdr = 0;
4168                 switch (first->protocol) {
4169                 case __constant_htons(ETH_P_IP):
4170                         vlan_macip_lens |= skb_network_header_len(skb);
4171                         type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4172                         l4_hdr = ip_hdr(skb)->protocol;
4173                         break;
4174                 case __constant_htons(ETH_P_IPV6):
4175                         vlan_macip_lens |= skb_network_header_len(skb);
4176                         l4_hdr = ipv6_hdr(skb)->nexthdr;
4177                         break;
4178                 default:
4179                         if (unlikely(net_ratelimit())) {
4180                                 dev_warn(tx_ring->dev,
4181                                  "partial checksum but proto=%x!\n",
4182                                  first->protocol);
4183                         }
4184                         break;
4185                 }
4186
4187                 switch (l4_hdr) {
4188                 case IPPROTO_TCP:
4189                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4190                         mss_l4len_idx = tcp_hdrlen(skb) <<
4191                                         E1000_ADVTXD_L4LEN_SHIFT;
4192                         break;
4193                 case IPPROTO_SCTP:
4194                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4195                         mss_l4len_idx = sizeof(struct sctphdr) <<
4196                                         E1000_ADVTXD_L4LEN_SHIFT;
4197                         break;
4198                 case IPPROTO_UDP:
4199                         mss_l4len_idx = sizeof(struct udphdr) <<
4200                                         E1000_ADVTXD_L4LEN_SHIFT;
4201                         break;
4202                 default:
4203                         if (unlikely(net_ratelimit())) {
4204                                 dev_warn(tx_ring->dev,
4205                                  "partial checksum but l4 proto=%x!\n",
4206                                  l4_hdr);
4207                         }
4208                         break;
4209                 }
4210
4211                 /* update TX checksum flag */
4212                 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4213         }
4214
4215         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4216         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4217
4218         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4219 }
4220
4221 static __le32 igb_tx_cmd_type(u32 tx_flags)
4222 {
4223         /* set type for advanced descriptor with frame checksum insertion */
4224         __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4225                                       E1000_ADVTXD_DCMD_IFCS |
4226                                       E1000_ADVTXD_DCMD_DEXT);
4227
4228         /* set HW vlan bit if vlan is present */
4229         if (tx_flags & IGB_TX_FLAGS_VLAN)
4230                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4231
4232         /* set timestamp bit if present */
4233         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4234                 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4235
4236         /* set segmentation bits for TSO */
4237         if (tx_flags & IGB_TX_FLAGS_TSO)
4238                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4239
4240         return cmd_type;
4241 }
4242
4243 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4244                                  union e1000_adv_tx_desc *tx_desc,
4245                                  u32 tx_flags, unsigned int paylen)
4246 {
4247         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4248
4249         /* 82575 requires a unique index per ring if any offload is enabled */
4250         if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4251             test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4252                 olinfo_status |= tx_ring->reg_idx << 4;
4253
4254         /* insert L4 checksum */
4255         if (tx_flags & IGB_TX_FLAGS_CSUM) {
4256                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4257
4258                 /* insert IPv4 checksum */
4259                 if (tx_flags & IGB_TX_FLAGS_IPV4)
4260                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4261         }
4262
4263         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4264 }
4265
4266 /*
4267  * The largest size we can write to the descriptor is 65535.  In order to
4268  * maintain a power of two alignment we have to limit ourselves to 32K.
4269  */
4270 #define IGB_MAX_TXD_PWR 15
4271 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
4272
4273 static void igb_tx_map(struct igb_ring *tx_ring,
4274                        struct igb_tx_buffer *first,
4275                        const u8 hdr_len)
4276 {
4277         struct sk_buff *skb = first->skb;
4278         struct igb_tx_buffer *tx_buffer_info;
4279         union e1000_adv_tx_desc *tx_desc;
4280         dma_addr_t dma;
4281         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4282         unsigned int data_len = skb->data_len;
4283         unsigned int size = skb_headlen(skb);
4284         unsigned int paylen = skb->len - hdr_len;
4285         __le32 cmd_type;
4286         u32 tx_flags = first->tx_flags;
4287         u16 i = tx_ring->next_to_use;
4288
4289         tx_desc = IGB_TX_DESC(tx_ring, i);
4290
4291         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4292         cmd_type = igb_tx_cmd_type(tx_flags);
4293
4294         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4295         if (dma_mapping_error(tx_ring->dev, dma))
4296                 goto dma_error;
4297
4298         /* record length, and DMA address */
4299         first->length = size;
4300         first->dma = dma;
4301         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4302
4303         for (;;) {
4304                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4305                         tx_desc->read.cmd_type_len =
4306                                 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4307
4308                         i++;
4309                         tx_desc++;
4310                         if (i == tx_ring->count) {
4311                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
4312                                 i = 0;
4313                         }
4314
4315                         dma += IGB_MAX_DATA_PER_TXD;
4316                         size -= IGB_MAX_DATA_PER_TXD;
4317
4318                         tx_desc->read.olinfo_status = 0;
4319                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4320                 }
4321
4322                 if (likely(!data_len))
4323                         break;
4324
4325                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4326
4327                 i++;
4328                 tx_desc++;
4329                 if (i == tx_ring->count) {
4330                         tx_desc = IGB_TX_DESC(tx_ring, 0);
4331                         i = 0;
4332                 }
4333
4334                 size = skb_frag_size(frag);
4335                 data_len -= size;
4336
4337                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4338                                    size, DMA_TO_DEVICE);
4339                 if (dma_mapping_error(tx_ring->dev, dma))
4340                         goto dma_error;
4341
4342                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4343                 tx_buffer_info->length = size;
4344                 tx_buffer_info->dma = dma;
4345
4346                 tx_desc->read.olinfo_status = 0;
4347                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4348
4349                 frag++;
4350         }
4351
4352         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4353
4354         /* write last descriptor with RS and EOP bits */
4355         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4356         if (unlikely(skb->no_fcs))
4357                 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
4358         tx_desc->read.cmd_type_len = cmd_type;
4359
4360         /* set the timestamp */
4361         first->time_stamp = jiffies;
4362
4363         /*
4364          * Force memory writes to complete before letting h/w know there
4365          * are new descriptors to fetch.  (Only applicable for weak-ordered
4366          * memory model archs, such as IA-64).
4367          *
4368          * We also need this memory barrier to make certain all of the
4369          * status bits have been updated before next_to_watch is written.
4370          */
4371         wmb();
4372
4373         /* set next_to_watch value indicating a packet is present */
4374         first->next_to_watch = tx_desc;
4375
4376         i++;
4377         if (i == tx_ring->count)
4378                 i = 0;
4379
4380         tx_ring->next_to_use = i;
4381
4382         writel(i, tx_ring->tail);
4383
4384         /* we need this if more than one processor can write to our tail
4385          * at a time, it syncronizes IO on IA64/Altix systems */
4386         mmiowb();
4387
4388         return;
4389
4390 dma_error:
4391         dev_err(tx_ring->dev, "TX DMA map failed\n");
4392
4393         /* clear dma mappings for failed tx_buffer_info map */
4394         for (;;) {
4395                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4396                 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4397                 if (tx_buffer_info == first)
4398                         break;
4399                 if (i == 0)
4400                         i = tx_ring->count;
4401                 i--;
4402         }
4403
4404         tx_ring->next_to_use = i;
4405 }
4406
4407 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4408 {
4409         struct net_device *netdev = tx_ring->netdev;
4410
4411         netif_stop_subqueue(netdev, tx_ring->queue_index);
4412
4413         /* Herbert's original patch had:
4414          *  smp_mb__after_netif_stop_queue();
4415          * but since that doesn't exist yet, just open code it. */
4416         smp_mb();
4417
4418         /* We need to check again in a case another CPU has just
4419          * made room available. */
4420         if (igb_desc_unused(tx_ring) < size)
4421                 return -EBUSY;
4422
4423         /* A reprieve! */
4424         netif_wake_subqueue(netdev, tx_ring->queue_index);
4425
4426         u64_stats_update_begin(&tx_ring->tx_syncp2);
4427         tx_ring->tx_stats.restart_queue2++;
4428         u64_stats_update_end(&tx_ring->tx_syncp2);
4429
4430         return 0;
4431 }
4432
4433 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4434 {
4435         if (igb_desc_unused(tx_ring) >= size)
4436                 return 0;
4437         return __igb_maybe_stop_tx(tx_ring, size);
4438 }
4439
4440 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4441                                 struct igb_ring *tx_ring)
4442 {
4443         struct igb_tx_buffer *first;
4444         int tso;
4445         u32 tx_flags = 0;
4446         __be16 protocol = vlan_get_protocol(skb);
4447         u8 hdr_len = 0;
4448
4449         /* need: 1 descriptor per page,
4450          *       + 2 desc gap to keep tail from touching head,
4451          *       + 1 desc for skb->data,
4452          *       + 1 desc for context descriptor,
4453          * otherwise try next time */
4454         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4455                 /* this is a hard error */
4456                 return NETDEV_TX_BUSY;
4457         }
4458
4459         /* record the location of the first descriptor for this packet */
4460         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4461         first->skb = skb;
4462         first->bytecount = skb->len;
4463         first->gso_segs = 1;
4464
4465         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4466                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4467                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4468         }
4469
4470         if (vlan_tx_tag_present(skb)) {
4471                 tx_flags |= IGB_TX_FLAGS_VLAN;
4472                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4473         }
4474
4475         /* record initial flags and protocol */
4476         first->tx_flags = tx_flags;
4477         first->protocol = protocol;
4478
4479         tso = igb_tso(tx_ring, first, &hdr_len);
4480         if (tso < 0)
4481                 goto out_drop;
4482         else if (!tso)
4483                 igb_tx_csum(tx_ring, first);
4484
4485         igb_tx_map(tx_ring, first, hdr_len);
4486
4487         /* Make sure there is space in the ring for the next send. */
4488         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4489
4490         return NETDEV_TX_OK;
4491
4492 out_drop:
4493         igb_unmap_and_free_tx_resource(tx_ring, first);
4494
4495         return NETDEV_TX_OK;
4496 }
4497
4498 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4499                                                     struct sk_buff *skb)
4500 {
4501         unsigned int r_idx = skb->queue_mapping;
4502
4503         if (r_idx >= adapter->num_tx_queues)
4504                 r_idx = r_idx % adapter->num_tx_queues;
4505
4506         return adapter->tx_ring[r_idx];
4507 }
4508
4509 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4510                                   struct net_device *netdev)
4511 {
4512         struct igb_adapter *adapter = netdev_priv(netdev);
4513
4514         if (test_bit(__IGB_DOWN, &adapter->state)) {
4515                 dev_kfree_skb_any(skb);
4516                 return NETDEV_TX_OK;
4517         }
4518
4519         if (skb->len <= 0) {
4520                 dev_kfree_skb_any(skb);
4521                 return NETDEV_TX_OK;
4522         }
4523
4524         /*
4525          * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4526          * in order to meet this minimum size requirement.
4527          */
4528         if (skb->len < 17) {
4529                 if (skb_padto(skb, 17))
4530                         return NETDEV_TX_OK;
4531                 skb->len = 17;
4532         }
4533
4534         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4535 }
4536
4537 /**
4538  * igb_tx_timeout - Respond to a Tx Hang
4539  * @netdev: network interface device structure
4540  **/
4541 static void igb_tx_timeout(struct net_device *netdev)
4542 {
4543         struct igb_adapter *adapter = netdev_priv(netdev);
4544         struct e1000_hw *hw = &adapter->hw;
4545
4546         /* Do the reset outside of interrupt context */
4547         adapter->tx_timeout_count++;
4548
4549         if (hw->mac.type >= e1000_82580)
4550                 hw->dev_spec._82575.global_device_reset = true;
4551
4552         schedule_work(&adapter->reset_task);
4553         wr32(E1000_EICS,
4554              (adapter->eims_enable_mask & ~adapter->eims_other));
4555 }
4556
4557 static void igb_reset_task(struct work_struct *work)
4558 {
4559         struct igb_adapter *adapter;
4560         adapter = container_of(work, struct igb_adapter, reset_task);
4561
4562         igb_dump(adapter);
4563         netdev_err(adapter->netdev, "Reset adapter\n");
4564         igb_reinit_locked(adapter);
4565 }
4566
4567 /**
4568  * igb_get_stats64 - Get System Network Statistics
4569  * @netdev: network interface device structure
4570  * @stats: rtnl_link_stats64 pointer
4571  *
4572  **/
4573 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4574                                                  struct rtnl_link_stats64 *stats)
4575 {
4576         struct igb_adapter *adapter = netdev_priv(netdev);
4577
4578         spin_lock(&adapter->stats64_lock);
4579         igb_update_stats(adapter, &adapter->stats64);
4580         memcpy(stats, &adapter->stats64, sizeof(*stats));
4581         spin_unlock(&adapter->stats64_lock);
4582
4583         return stats;
4584 }
4585
4586 /**
4587  * igb_change_mtu - Change the Maximum Transfer Unit
4588  * @netdev: network interface device structure
4589  * @new_mtu: new value for maximum frame size
4590  *
4591  * Returns 0 on success, negative on failure
4592  **/
4593 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4594 {
4595         struct igb_adapter *adapter = netdev_priv(netdev);
4596         struct pci_dev *pdev = adapter->pdev;
4597         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4598
4599         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4600                 dev_err(&pdev->dev, "Invalid MTU setting\n");
4601                 return -EINVAL;
4602         }
4603
4604 #define MAX_STD_JUMBO_FRAME_SIZE 9238
4605         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4606                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4607                 return -EINVAL;
4608         }
4609
4610         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4611                 msleep(1);
4612
4613         /* igb_down has a dependency on max_frame_size */
4614         adapter->max_frame_size = max_frame;
4615
4616         if (netif_running(netdev))
4617                 igb_down(adapter);
4618
4619         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4620                  netdev->mtu, new_mtu);
4621         netdev->mtu = new_mtu;
4622
4623         if (netif_running(netdev))
4624                 igb_up(adapter);
4625         else
4626                 igb_reset(adapter);
4627
4628         clear_bit(__IGB_RESETTING, &adapter->state);
4629
4630         return 0;
4631 }
4632
4633 /**
4634  * igb_update_stats - Update the board statistics counters
4635  * @adapter: board private structure
4636  **/
4637
4638 void igb_update_stats(struct igb_adapter *adapter,
4639                       struct rtnl_link_stats64 *net_stats)
4640 {
4641         struct e1000_hw *hw = &adapter->hw;
4642         struct pci_dev *pdev = adapter->pdev;
4643         u32 reg, mpc;
4644         u16 phy_tmp;
4645         int i;
4646         u64 bytes, packets;
4647         unsigned int start;
4648         u64 _bytes, _packets;
4649
4650 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4651
4652         /*
4653          * Prevent stats update while adapter is being reset, or if the pci
4654          * connection is down.
4655          */
4656         if (adapter->link_speed == 0)
4657                 return;
4658         if (pci_channel_offline(pdev))
4659                 return;
4660
4661         bytes = 0;
4662         packets = 0;
4663         for (i = 0; i < adapter->num_rx_queues; i++) {
4664                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4665                 struct igb_ring *ring = adapter->rx_ring[i];
4666
4667                 ring->rx_stats.drops += rqdpc_tmp;
4668                 net_stats->rx_fifo_errors += rqdpc_tmp;
4669
4670                 do {
4671                         start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4672                         _bytes = ring->rx_stats.bytes;
4673                         _packets = ring->rx_stats.packets;
4674                 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4675                 bytes += _bytes;
4676                 packets += _packets;
4677         }
4678
4679         net_stats->rx_bytes = bytes;
4680         net_stats->rx_packets = packets;
4681
4682         bytes = 0;
4683         packets = 0;
4684         for (i = 0; i < adapter->num_tx_queues; i++) {
4685                 struct igb_ring *ring = adapter->tx_ring[i];
4686                 do {
4687                         start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4688                         _bytes = ring->tx_stats.bytes;
4689                         _packets = ring->tx_stats.packets;
4690                 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4691                 bytes += _bytes;
4692                 packets += _packets;
4693         }
4694         net_stats->tx_bytes = bytes;
4695         net_stats->tx_packets = packets;
4696
4697         /* read stats registers */
4698         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4699         adapter->stats.gprc += rd32(E1000_GPRC);
4700         adapter->stats.gorc += rd32(E1000_GORCL);
4701         rd32(E1000_GORCH); /* clear GORCL */
4702         adapter->stats.bprc += rd32(E1000_BPRC);
4703         adapter->stats.mprc += rd32(E1000_MPRC);
4704         adapter->stats.roc += rd32(E1000_ROC);
4705
4706         adapter->stats.prc64 += rd32(E1000_PRC64);
4707         adapter->stats.prc127 += rd32(E1000_PRC127);
4708         adapter->stats.prc255 += rd32(E1000_PRC255);
4709         adapter->stats.prc511 += rd32(E1000_PRC511);
4710         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4711         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4712         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4713         adapter->stats.sec += rd32(E1000_SEC);
4714
4715         mpc = rd32(E1000_MPC);
4716         adapter->stats.mpc += mpc;
4717         net_stats->rx_fifo_errors += mpc;
4718         adapter->stats.scc += rd32(E1000_SCC);
4719         adapter->stats.ecol += rd32(E1000_ECOL);
4720         adapter->stats.mcc += rd32(E1000_MCC);
4721         adapter->stats.latecol += rd32(E1000_LATECOL);
4722         adapter->stats.dc += rd32(E1000_DC);
4723         adapter->stats.rlec += rd32(E1000_RLEC);
4724         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4725         adapter->stats.xontxc += rd32(E1000_XONTXC);
4726         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4727         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4728         adapter->stats.fcruc += rd32(E1000_FCRUC);
4729         adapter->stats.gptc += rd32(E1000_GPTC);
4730         adapter->stats.gotc += rd32(E1000_GOTCL);
4731         rd32(E1000_GOTCH); /* clear GOTCL */
4732         adapter->stats.rnbc += rd32(E1000_RNBC);
4733         adapter->stats.ruc += rd32(E1000_RUC);
4734         adapter->stats.rfc += rd32(E1000_RFC);
4735         adapter->stats.rjc += rd32(E1000_RJC);
4736         adapter->stats.tor += rd32(E1000_TORH);
4737         adapter->stats.tot += rd32(E1000_TOTH);
4738         adapter->stats.tpr += rd32(E1000_TPR);
4739
4740         adapter->stats.ptc64 += rd32(E1000_PTC64);
4741         adapter->stats.ptc127 += rd32(E1000_PTC127);
4742         adapter->stats.ptc255 += rd32(E1000_PTC255);
4743         adapter->stats.ptc511 += rd32(E1000_PTC511);
4744         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4745         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4746
4747         adapter->stats.mptc += rd32(E1000_MPTC);
4748         adapter->stats.bptc += rd32(E1000_BPTC);
4749
4750         adapter->stats.tpt += rd32(E1000_TPT);
4751         adapter->stats.colc += rd32(E1000_COLC);
4752
4753         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4754         /* read internal phy specific stats */
4755         reg = rd32(E1000_CTRL_EXT);
4756         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4757                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4758                 adapter->stats.tncrs += rd32(E1000_TNCRS);
4759         }
4760
4761         adapter->stats.tsctc += rd32(E1000_TSCTC);
4762         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4763
4764         adapter->stats.iac += rd32(E1000_IAC);
4765         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4766         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4767         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4768         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4769         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4770         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4771         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4772         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4773
4774         /* Fill out the OS statistics structure */
4775         net_stats->multicast = adapter->stats.mprc;
4776         net_stats->collisions = adapter->stats.colc;
4777
4778         /* Rx Errors */
4779
4780         /* RLEC on some newer hardware can be incorrect so build
4781          * our own version based on RUC and ROC */
4782         net_stats->rx_errors = adapter->stats.rxerrc +
4783                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4784                 adapter->stats.ruc + adapter->stats.roc +
4785                 adapter->stats.cexterr;
4786         net_stats->rx_length_errors = adapter->stats.ruc +
4787                                       adapter->stats.roc;
4788         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4789         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4790         net_stats->rx_missed_errors = adapter->stats.mpc;
4791
4792         /* Tx Errors */
4793         net_stats->tx_errors = adapter->stats.ecol +
4794                                adapter->stats.latecol;
4795         net_stats->tx_aborted_errors = adapter->stats.ecol;
4796         net_stats->tx_window_errors = adapter->stats.latecol;
4797         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4798
4799         /* Tx Dropped needs to be maintained elsewhere */
4800
4801         /* Phy Stats */
4802         if (hw->phy.media_type == e1000_media_type_copper) {
4803                 if ((adapter->link_speed == SPEED_1000) &&
4804                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4805                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4806                         adapter->phy_stats.idle_errors += phy_tmp;
4807                 }
4808         }
4809
4810         /* Management Stats */
4811         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4812         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4813         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4814
4815         /* OS2BMC Stats */
4816         reg = rd32(E1000_MANC);
4817         if (reg & E1000_MANC_EN_BMC2OS) {
4818                 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4819                 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4820                 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4821                 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4822         }
4823 }
4824
4825 static irqreturn_t igb_msix_other(int irq, void *data)
4826 {
4827         struct igb_adapter *adapter = data;
4828         struct e1000_hw *hw = &adapter->hw;
4829         u32 icr = rd32(E1000_ICR);
4830         /* reading ICR causes bit 31 of EICR to be cleared */
4831
4832         if (icr & E1000_ICR_DRSTA)
4833                 schedule_work(&adapter->reset_task);
4834
4835         if (icr & E1000_ICR_DOUTSYNC) {
4836                 /* HW is reporting DMA is out of sync */
4837                 adapter->stats.doosync++;
4838                 /* The DMA Out of Sync is also indication of a spoof event
4839                  * in IOV mode. Check the Wrong VM Behavior register to
4840                  * see if it is really a spoof event. */
4841                 igb_check_wvbr(adapter);
4842         }
4843
4844         /* Check for a mailbox event */
4845         if (icr & E1000_ICR_VMMB)
4846                 igb_msg_task(adapter);
4847
4848         if (icr & E1000_ICR_LSC) {
4849                 hw->mac.get_link_status = 1;
4850                 /* guard against interrupt when we're going down */
4851                 if (!test_bit(__IGB_DOWN, &adapter->state))
4852                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4853         }
4854
4855         wr32(E1000_EIMS, adapter->eims_other);
4856
4857         return IRQ_HANDLED;
4858 }
4859
4860 static void igb_write_itr(struct igb_q_vector *q_vector)
4861 {
4862         struct igb_adapter *adapter = q_vector->adapter;
4863         u32 itr_val = q_vector->itr_val & 0x7FFC;
4864
4865         if (!q_vector->set_itr)
4866                 return;
4867
4868         if (!itr_val)
4869                 itr_val = 0x4;
4870
4871         if (adapter->hw.mac.type == e1000_82575)
4872                 itr_val |= itr_val << 16;
4873         else
4874                 itr_val |= E1000_EITR_CNT_IGNR;
4875
4876         writel(itr_val, q_vector->itr_register);
4877         q_vector->set_itr = 0;
4878 }
4879
4880 static irqreturn_t igb_msix_ring(int irq, void *data)
4881 {
4882         struct igb_q_vector *q_vector = data;
4883
4884         /* Write the ITR value calculated from the previous interrupt. */
4885         igb_write_itr(q_vector);
4886
4887         napi_schedule(&q_vector->napi);
4888
4889         return IRQ_HANDLED;
4890 }
4891
4892 #ifdef CONFIG_IGB_DCA
4893 static void igb_update_dca(struct igb_q_vector *q_vector)
4894 {
4895         struct igb_adapter *adapter = q_vector->adapter;
4896         struct e1000_hw *hw = &adapter->hw;
4897         int cpu = get_cpu();
4898
4899         if (q_vector->cpu == cpu)
4900                 goto out_no_update;
4901
4902         if (q_vector->tx.ring) {
4903                 int q = q_vector->tx.ring->reg_idx;
4904                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4905                 if (hw->mac.type == e1000_82575) {
4906                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4907                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4908                 } else {
4909                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4910                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4911                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4912                 }
4913                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4914                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4915         }
4916         if (q_vector->rx.ring) {
4917                 int q = q_vector->rx.ring->reg_idx;
4918                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4919                 if (hw->mac.type == e1000_82575) {
4920                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4921                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4922                 } else {
4923                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4924                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4925                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4926                 }
4927                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4928                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4929                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4930                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4931         }
4932         q_vector->cpu = cpu;
4933 out_no_update:
4934         put_cpu();
4935 }
4936
4937 static void igb_setup_dca(struct igb_adapter *adapter)
4938 {
4939         struct e1000_hw *hw = &adapter->hw;
4940         int i;
4941
4942         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4943                 return;
4944
4945         /* Always use CB2 mode, difference is masked in the CB driver. */
4946         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4947
4948         for (i = 0; i < adapter->num_q_vectors; i++) {
4949                 adapter->q_vector[i]->cpu = -1;
4950                 igb_update_dca(adapter->q_vector[i]);
4951         }
4952 }
4953
4954 static int __igb_notify_dca(struct device *dev, void *data)
4955 {
4956         struct net_device *netdev = dev_get_drvdata(dev);
4957         struct igb_adapter *adapter = netdev_priv(netdev);
4958         struct pci_dev *pdev = adapter->pdev;
4959         struct e1000_hw *hw = &adapter->hw;
4960         unsigned long event = *(unsigned long *)data;
4961
4962         switch (event) {
4963         case DCA_PROVIDER_ADD:
4964                 /* if already enabled, don't do it again */
4965                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4966                         break;
4967                 if (dca_add_requester(dev) == 0) {
4968                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4969                         dev_info(&pdev->dev, "DCA enabled\n");
4970                         igb_setup_dca(adapter);
4971                         break;
4972                 }
4973                 /* Fall Through since DCA is disabled. */
4974         case DCA_PROVIDER_REMOVE:
4975                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4976                         /* without this a class_device is left
4977                          * hanging around in the sysfs model */
4978                         dca_remove_requester(dev);
4979                         dev_info(&pdev->dev, "DCA disabled\n");
4980                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4981                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4982                 }
4983                 break;
4984         }
4985
4986         return 0;
4987 }
4988
4989 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4990                           void *p)
4991 {
4992         int ret_val;
4993
4994         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4995                                          __igb_notify_dca);
4996
4997         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4998 }
4999 #endif /* CONFIG_IGB_DCA */
5000
5001 #ifdef CONFIG_PCI_IOV
5002 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5003 {
5004         unsigned char mac_addr[ETH_ALEN];
5005         struct pci_dev *pdev = adapter->pdev;
5006         struct e1000_hw *hw = &adapter->hw;
5007         struct pci_dev *pvfdev;
5008         unsigned int device_id;
5009         u16 thisvf_devfn;
5010
5011         eth_random_addr(mac_addr);
5012         igb_set_vf_mac(adapter, vf, mac_addr);
5013
5014         switch (adapter->hw.mac.type) {
5015         case e1000_82576:
5016                 device_id = IGB_82576_VF_DEV_ID;
5017                 /* VF Stride for 82576 is 2 */
5018                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5019                         (pdev->devfn & 1);
5020                 break;
5021         case e1000_i350:
5022                 device_id = IGB_I350_VF_DEV_ID;
5023                 /* VF Stride for I350 is 4 */
5024                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5025                                 (pdev->devfn & 3);
5026                 break;
5027         default:
5028                 device_id = 0;
5029                 thisvf_devfn = 0;
5030                 break;
5031         }
5032
5033         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5034         while (pvfdev) {
5035                 if (pvfdev->devfn == thisvf_devfn)
5036                         break;
5037                 pvfdev = pci_get_device(hw->vendor_id,
5038                                         device_id, pvfdev);
5039         }
5040
5041         if (pvfdev)
5042                 adapter->vf_data[vf].vfdev = pvfdev;
5043         else
5044                 dev_err(&pdev->dev,
5045                         "Couldn't find pci dev ptr for VF %4.4x\n",
5046                         thisvf_devfn);
5047         return pvfdev != NULL;
5048 }
5049
5050 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5051 {
5052         struct e1000_hw *hw = &adapter->hw;
5053         struct pci_dev *pdev = adapter->pdev;
5054         struct pci_dev *pvfdev;
5055         u16 vf_devfn = 0;
5056         u16 vf_stride;
5057         unsigned int device_id;
5058         int vfs_found = 0;
5059
5060         switch (adapter->hw.mac.type) {
5061         case e1000_82576:
5062                 device_id = IGB_82576_VF_DEV_ID;
5063                 /* VF Stride for 82576 is 2 */
5064                 vf_stride = 2;
5065                 break;
5066         case e1000_i350:
5067                 device_id = IGB_I350_VF_DEV_ID;
5068                 /* VF Stride for I350 is 4 */
5069                 vf_stride = 4;
5070                 break;
5071         default:
5072                 device_id = 0;
5073                 vf_stride = 0;
5074                 break;
5075         }
5076
5077         vf_devfn = pdev->devfn + 0x80;
5078         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5079         while (pvfdev) {
5080                 if (pvfdev->devfn == vf_devfn &&
5081                     (pvfdev->bus->number >= pdev->bus->number))
5082                         vfs_found++;
5083                 vf_devfn += vf_stride;
5084                 pvfdev = pci_get_device(hw->vendor_id,
5085                                         device_id, pvfdev);
5086         }
5087
5088         return vfs_found;
5089 }
5090
5091 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5092 {
5093         int i;
5094         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5095                 if (adapter->vf_data[i].vfdev) {
5096                         if (adapter->vf_data[i].vfdev->dev_flags &
5097                             PCI_DEV_FLAGS_ASSIGNED)
5098                                 return true;
5099                 }
5100         }
5101         return false;
5102 }
5103
5104 #endif
5105 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5106 {
5107         struct e1000_hw *hw = &adapter->hw;
5108         u32 ping;
5109         int i;
5110
5111         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5112                 ping = E1000_PF_CONTROL_MSG;
5113                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5114                         ping |= E1000_VT_MSGTYPE_CTS;
5115                 igb_write_mbx(hw, &ping, 1, i);
5116         }
5117 }
5118
5119 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5120 {
5121         struct e1000_hw *hw = &adapter->hw;
5122         u32 vmolr = rd32(E1000_VMOLR(vf));
5123         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5124
5125         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5126                             IGB_VF_FLAG_MULTI_PROMISC);
5127         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5128
5129         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5130                 vmolr |= E1000_VMOLR_MPME;
5131                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5132                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5133         } else {
5134                 /*
5135                  * if we have hashes and we are clearing a multicast promisc
5136                  * flag we need to write the hashes to the MTA as this step
5137                  * was previously skipped
5138                  */
5139                 if (vf_data->num_vf_mc_hashes > 30) {
5140                         vmolr |= E1000_VMOLR_MPME;
5141                 } else if (vf_data->num_vf_mc_hashes) {
5142                         int j;
5143                         vmolr |= E1000_VMOLR_ROMPE;
5144                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5145                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5146                 }
5147         }
5148
5149         wr32(E1000_VMOLR(vf), vmolr);
5150
5151         /* there are flags left unprocessed, likely not supported */
5152         if (*msgbuf & E1000_VT_MSGINFO_MASK)
5153                 return -EINVAL;
5154
5155         return 0;
5156
5157 }
5158
5159 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5160                                   u32 *msgbuf, u32 vf)
5161 {
5162         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5163         u16 *hash_list = (u16 *)&msgbuf[1];
5164         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5165         int i;
5166
5167         /* salt away the number of multicast addresses assigned
5168          * to this VF for later use to restore when the PF multi cast
5169          * list changes
5170          */
5171         vf_data->num_vf_mc_hashes = n;
5172
5173         /* only up to 30 hash values supported */
5174         if (n > 30)
5175                 n = 30;
5176
5177         /* store the hashes for later use */
5178         for (i = 0; i < n; i++)
5179                 vf_data->vf_mc_hashes[i] = hash_list[i];
5180
5181         /* Flush and reset the mta with the new values */
5182         igb_set_rx_mode(adapter->netdev);
5183
5184         return 0;
5185 }
5186
5187 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5188 {
5189         struct e1000_hw *hw = &adapter->hw;
5190         struct vf_data_storage *vf_data;
5191         int i, j;
5192
5193         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5194                 u32 vmolr = rd32(E1000_VMOLR(i));
5195                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5196
5197                 vf_data = &adapter->vf_data[i];
5198
5199                 if ((vf_data->num_vf_mc_hashes > 30) ||
5200                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5201                         vmolr |= E1000_VMOLR_MPME;
5202                 } else if (vf_data->num_vf_mc_hashes) {
5203                         vmolr |= E1000_VMOLR_ROMPE;
5204                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5205                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5206                 }
5207                 wr32(E1000_VMOLR(i), vmolr);
5208         }
5209 }
5210
5211 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5212 {
5213         struct e1000_hw *hw = &adapter->hw;
5214         u32 pool_mask, reg, vid;
5215         int i;
5216
5217         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5218
5219         /* Find the vlan filter for this id */
5220         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5221                 reg = rd32(E1000_VLVF(i));
5222
5223                 /* remove the vf from the pool */
5224                 reg &= ~pool_mask;
5225
5226                 /* if pool is empty then remove entry from vfta */
5227                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5228                     (reg & E1000_VLVF_VLANID_ENABLE)) {
5229                         reg = 0;
5230                         vid = reg & E1000_VLVF_VLANID_MASK;
5231                         igb_vfta_set(hw, vid, false);
5232                 }
5233
5234                 wr32(E1000_VLVF(i), reg);
5235         }
5236
5237         adapter->vf_data[vf].vlans_enabled = 0;
5238 }
5239
5240 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5241 {
5242         struct e1000_hw *hw = &adapter->hw;
5243         u32 reg, i;
5244
5245         /* The vlvf table only exists on 82576 hardware and newer */
5246         if (hw->mac.type < e1000_82576)
5247                 return -1;
5248
5249         /* we only need to do this if VMDq is enabled */
5250         if (!adapter->vfs_allocated_count)
5251                 return -1;
5252
5253         /* Find the vlan filter for this id */
5254         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5255                 reg = rd32(E1000_VLVF(i));
5256                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5257                     vid == (reg & E1000_VLVF_VLANID_MASK))
5258                         break;
5259         }
5260
5261         if (add) {
5262                 if (i == E1000_VLVF_ARRAY_SIZE) {
5263                         /* Did not find a matching VLAN ID entry that was
5264                          * enabled.  Search for a free filter entry, i.e.
5265                          * one without the enable bit set
5266                          */
5267                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5268                                 reg = rd32(E1000_VLVF(i));
5269                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5270                                         break;
5271                         }
5272                 }
5273                 if (i < E1000_VLVF_ARRAY_SIZE) {
5274                         /* Found an enabled/available entry */
5275                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5276
5277                         /* if !enabled we need to set this up in vfta */
5278                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5279                                 /* add VID to filter table */
5280                                 igb_vfta_set(hw, vid, true);
5281                                 reg |= E1000_VLVF_VLANID_ENABLE;
5282                         }
5283                         reg &= ~E1000_VLVF_VLANID_MASK;
5284                         reg |= vid;
5285                         wr32(E1000_VLVF(i), reg);
5286
5287                         /* do not modify RLPML for PF devices */
5288                         if (vf >= adapter->vfs_allocated_count)
5289                                 return 0;
5290
5291                         if (!adapter->vf_data[vf].vlans_enabled) {
5292                                 u32 size;
5293                                 reg = rd32(E1000_VMOLR(vf));
5294                                 size = reg & E1000_VMOLR_RLPML_MASK;
5295                                 size += 4;
5296                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5297                                 reg |= size;
5298                                 wr32(E1000_VMOLR(vf), reg);
5299                         }
5300
5301                         adapter->vf_data[vf].vlans_enabled++;
5302                 }
5303         } else {
5304                 if (i < E1000_VLVF_ARRAY_SIZE) {
5305                         /* remove vf from the pool */
5306                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5307                         /* if pool is empty then remove entry from vfta */
5308                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5309                                 reg = 0;
5310                                 igb_vfta_set(hw, vid, false);
5311                         }
5312                         wr32(E1000_VLVF(i), reg);
5313
5314                         /* do not modify RLPML for PF devices */
5315                         if (vf >= adapter->vfs_allocated_count)
5316                                 return 0;
5317
5318                         adapter->vf_data[vf].vlans_enabled--;
5319                         if (!adapter->vf_data[vf].vlans_enabled) {
5320                                 u32 size;
5321                                 reg = rd32(E1000_VMOLR(vf));
5322                                 size = reg & E1000_VMOLR_RLPML_MASK;
5323                                 size -= 4;
5324                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5325                                 reg |= size;
5326                                 wr32(E1000_VMOLR(vf), reg);
5327                         }
5328                 }
5329         }
5330         return 0;
5331 }
5332
5333 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5334 {
5335         struct e1000_hw *hw = &adapter->hw;
5336
5337         if (vid)
5338                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5339         else
5340                 wr32(E1000_VMVIR(vf), 0);
5341 }
5342
5343 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5344                                int vf, u16 vlan, u8 qos)
5345 {
5346         int err = 0;
5347         struct igb_adapter *adapter = netdev_priv(netdev);
5348
5349         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5350                 return -EINVAL;
5351         if (vlan || qos) {
5352                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5353                 if (err)
5354                         goto out;
5355                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5356                 igb_set_vmolr(adapter, vf, !vlan);
5357                 adapter->vf_data[vf].pf_vlan = vlan;
5358                 adapter->vf_data[vf].pf_qos = qos;
5359                 dev_info(&adapter->pdev->dev,
5360                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5361                 if (test_bit(__IGB_DOWN, &adapter->state)) {
5362                         dev_warn(&adapter->pdev->dev,
5363                                  "The VF VLAN has been set,"
5364                                  " but the PF device is not up.\n");
5365                         dev_warn(&adapter->pdev->dev,
5366                                  "Bring the PF device up before"
5367                                  " attempting to use the VF device.\n");
5368                 }
5369         } else {
5370                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5371                                    false, vf);
5372                 igb_set_vmvir(adapter, vlan, vf);
5373                 igb_set_vmolr(adapter, vf, true);
5374                 adapter->vf_data[vf].pf_vlan = 0;
5375                 adapter->vf_data[vf].pf_qos = 0;
5376        }
5377 out:
5378        return err;
5379 }
5380
5381 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5382 {
5383         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5384         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5385
5386         return igb_vlvf_set(adapter, vid, add, vf);
5387 }
5388
5389 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5390 {
5391         /* clear flags - except flag that indicates PF has set the MAC */
5392         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5393         adapter->vf_data[vf].last_nack = jiffies;
5394
5395         /* reset offloads to defaults */
5396         igb_set_vmolr(adapter, vf, true);
5397
5398         /* reset vlans for device */
5399         igb_clear_vf_vfta(adapter, vf);
5400         if (adapter->vf_data[vf].pf_vlan)
5401                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5402                                     adapter->vf_data[vf].pf_vlan,
5403                                     adapter->vf_data[vf].pf_qos);
5404         else
5405                 igb_clear_vf_vfta(adapter, vf);
5406
5407         /* reset multicast table array for vf */
5408         adapter->vf_data[vf].num_vf_mc_hashes = 0;
5409
5410         /* Flush and reset the mta with the new values */
5411         igb_set_rx_mode(adapter->netdev);
5412 }
5413
5414 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5415 {
5416         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5417
5418         /* generate a new mac address as we were hotplug removed/added */
5419         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5420                 eth_random_addr(vf_mac);
5421
5422         /* process remaining reset events */
5423         igb_vf_reset(adapter, vf);
5424 }
5425
5426 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5427 {
5428         struct e1000_hw *hw = &adapter->hw;
5429         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5430         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5431         u32 reg, msgbuf[3];
5432         u8 *addr = (u8 *)(&msgbuf[1]);
5433
5434         /* process all the same items cleared in a function level reset */
5435         igb_vf_reset(adapter, vf);
5436
5437         /* set vf mac address */
5438         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5439
5440         /* enable transmit and receive for vf */
5441         reg = rd32(E1000_VFTE);
5442         wr32(E1000_VFTE, reg | (1 << vf));
5443         reg = rd32(E1000_VFRE);
5444         wr32(E1000_VFRE, reg | (1 << vf));
5445
5446         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5447
5448         /* reply to reset with ack and vf mac address */
5449         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5450         memcpy(addr, vf_mac, 6);
5451         igb_write_mbx(hw, msgbuf, 3, vf);
5452 }
5453
5454 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5455 {
5456         /*
5457          * The VF MAC Address is stored in a packed array of bytes
5458          * starting at the second 32 bit word of the msg array
5459          */
5460         unsigned char *addr = (char *)&msg[1];
5461         int err = -1;
5462
5463         if (is_valid_ether_addr(addr))
5464                 err = igb_set_vf_mac(adapter, vf, addr);
5465
5466         return err;
5467 }
5468
5469 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5470 {
5471         struct e1000_hw *hw = &adapter->hw;
5472         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5473         u32 msg = E1000_VT_MSGTYPE_NACK;
5474
5475         /* if device isn't clear to send it shouldn't be reading either */
5476         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5477             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5478                 igb_write_mbx(hw, &msg, 1, vf);
5479                 vf_data->last_nack = jiffies;
5480         }
5481 }
5482
5483 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5484 {
5485         struct pci_dev *pdev = adapter->pdev;
5486         u32 msgbuf[E1000_VFMAILBOX_SIZE];
5487         struct e1000_hw *hw = &adapter->hw;
5488         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5489         s32 retval;
5490
5491         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5492
5493         if (retval) {
5494                 /* if receive failed revoke VF CTS stats and restart init */
5495                 dev_err(&pdev->dev, "Error receiving message from VF\n");
5496                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5497                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5498                         return;
5499                 goto out;
5500         }
5501
5502         /* this is a message we already processed, do nothing */
5503         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5504                 return;
5505
5506         /*
5507          * until the vf completes a reset it should not be
5508          * allowed to start any configuration.
5509          */
5510
5511         if (msgbuf[0] == E1000_VF_RESET) {
5512                 igb_vf_reset_msg(adapter, vf);
5513                 return;
5514         }
5515
5516         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5517                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5518                         return;
5519                 retval = -1;
5520                 goto out;
5521         }
5522
5523         switch ((msgbuf[0] & 0xFFFF)) {
5524         case E1000_VF_SET_MAC_ADDR:
5525                 retval = -EINVAL;
5526                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5527                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5528                 else
5529                         dev_warn(&pdev->dev,
5530                                  "VF %d attempted to override administratively "
5531                                  "set MAC address\nReload the VF driver to "
5532                                  "resume operations\n", vf);
5533                 break;
5534         case E1000_VF_SET_PROMISC:
5535                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5536                 break;
5537         case E1000_VF_SET_MULTICAST:
5538                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5539                 break;
5540         case E1000_VF_SET_LPE:
5541                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5542                 break;
5543         case E1000_VF_SET_VLAN:
5544                 retval = -1;
5545                 if (vf_data->pf_vlan)
5546                         dev_warn(&pdev->dev,
5547                                  "VF %d attempted to override administratively "
5548                                  "set VLAN tag\nReload the VF driver to "
5549                                  "resume operations\n", vf);
5550                 else
5551                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5552                 break;
5553         default:
5554                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5555                 retval = -1;
5556                 break;
5557         }
5558
5559         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5560 out:
5561         /* notify the VF of the results of what it sent us */
5562         if (retval)
5563                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5564         else
5565                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5566
5567         igb_write_mbx(hw, msgbuf, 1, vf);
5568 }
5569
5570 static void igb_msg_task(struct igb_adapter *adapter)
5571 {
5572         struct e1000_hw *hw = &adapter->hw;
5573         u32 vf;
5574
5575         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5576                 /* process any reset requests */
5577                 if (!igb_check_for_rst(hw, vf))
5578                         igb_vf_reset_event(adapter, vf);
5579
5580                 /* process any messages pending */
5581                 if (!igb_check_for_msg(hw, vf))
5582                         igb_rcv_msg_from_vf(adapter, vf);
5583
5584                 /* process any acks */
5585                 if (!igb_check_for_ack(hw, vf))
5586                         igb_rcv_ack_from_vf(adapter, vf);
5587         }
5588 }
5589
5590 /**
5591  *  igb_set_uta - Set unicast filter table address
5592  *  @adapter: board private structure
5593  *
5594  *  The unicast table address is a register array of 32-bit registers.
5595  *  The table is meant to be used in a way similar to how the MTA is used
5596  *  however due to certain limitations in the hardware it is necessary to
5597  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5598  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
5599  **/
5600 static void igb_set_uta(struct igb_adapter *adapter)
5601 {
5602         struct e1000_hw *hw = &adapter->hw;
5603         int i;
5604
5605         /* The UTA table only exists on 82576 hardware and newer */
5606         if (hw->mac.type < e1000_82576)
5607                 return;
5608
5609         /* we only need to do this if VMDq is enabled */
5610         if (!adapter->vfs_allocated_count)
5611                 return;
5612
5613         for (i = 0; i < hw->mac.uta_reg_count; i++)
5614                 array_wr32(E1000_UTA, i, ~0);
5615 }
5616
5617 /**
5618  * igb_intr_msi - Interrupt Handler
5619  * @irq: interrupt number
5620  * @data: pointer to a network interface device structure
5621  **/
5622 static irqreturn_t igb_intr_msi(int irq, void *data)
5623 {
5624         struct igb_adapter *adapter = data;
5625         struct igb_q_vector *q_vector = adapter->q_vector[0];
5626         struct e1000_hw *hw = &adapter->hw;
5627         /* read ICR disables interrupts using IAM */
5628         u32 icr = rd32(E1000_ICR);
5629
5630         igb_write_itr(q_vector);
5631
5632         if (icr & E1000_ICR_DRSTA)
5633                 schedule_work(&adapter->reset_task);
5634
5635         if (icr & E1000_ICR_DOUTSYNC) {
5636                 /* HW is reporting DMA is out of sync */
5637                 adapter->stats.doosync++;
5638         }
5639
5640         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5641                 hw->mac.get_link_status = 1;
5642                 if (!test_bit(__IGB_DOWN, &adapter->state))
5643                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5644         }
5645
5646         napi_schedule(&q_vector->napi);
5647
5648         return IRQ_HANDLED;
5649 }
5650
5651 /**
5652  * igb_intr - Legacy Interrupt Handler
5653  * @irq: interrupt number
5654  * @data: pointer to a network interface device structure
5655  **/
5656 static irqreturn_t igb_intr(int irq, void *data)
5657 {
5658         struct igb_adapter *adapter = data;
5659         struct igb_q_vector *q_vector = adapter->q_vector[0];
5660         struct e1000_hw *hw = &adapter->hw;
5661         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5662          * need for the IMC write */
5663         u32 icr = rd32(E1000_ICR);
5664
5665         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5666          * not set, then the adapter didn't send an interrupt */
5667         if (!(icr & E1000_ICR_INT_ASSERTED))
5668                 return IRQ_NONE;
5669
5670         igb_write_itr(q_vector);
5671
5672         if (icr & E1000_ICR_DRSTA)
5673                 schedule_work(&adapter->reset_task);
5674
5675         if (icr & E1000_ICR_DOUTSYNC) {
5676                 /* HW is reporting DMA is out of sync */
5677                 adapter->stats.doosync++;
5678         }
5679
5680         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5681                 hw->mac.get_link_status = 1;
5682                 /* guard against interrupt when we're going down */
5683                 if (!test_bit(__IGB_DOWN, &adapter->state))
5684                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5685         }
5686
5687         napi_schedule(&q_vector->napi);
5688
5689         return IRQ_HANDLED;
5690 }
5691
5692 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5693 {
5694         struct igb_adapter *adapter = q_vector->adapter;
5695         struct e1000_hw *hw = &adapter->hw;
5696
5697         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5698             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5699                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5700                         igb_set_itr(q_vector);
5701                 else
5702                         igb_update_ring_itr(q_vector);
5703         }
5704
5705         if (!test_bit(__IGB_DOWN, &adapter->state)) {
5706                 if (adapter->msix_entries)
5707                         wr32(E1000_EIMS, q_vector->eims_value);
5708                 else
5709                         igb_irq_enable(adapter);
5710         }
5711 }
5712
5713 /**
5714  * igb_poll - NAPI Rx polling callback
5715  * @napi: napi polling structure
5716  * @budget: count of how many packets we should handle
5717  **/
5718 static int igb_poll(struct napi_struct *napi, int budget)
5719 {
5720         struct igb_q_vector *q_vector = container_of(napi,
5721                                                      struct igb_q_vector,
5722                                                      napi);
5723         bool clean_complete = true;
5724
5725 #ifdef CONFIG_IGB_DCA
5726         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5727                 igb_update_dca(q_vector);
5728 #endif
5729         if (q_vector->tx.ring)
5730                 clean_complete = igb_clean_tx_irq(q_vector);
5731
5732         if (q_vector->rx.ring)
5733                 clean_complete &= igb_clean_rx_irq(q_vector, budget);
5734
5735         /* If all work not completed, return budget and keep polling */
5736         if (!clean_complete)
5737                 return budget;
5738
5739         /* If not enough Rx work done, exit the polling mode */
5740         napi_complete(napi);
5741         igb_ring_irq_enable(q_vector);
5742
5743         return 0;
5744 }
5745
5746 #ifdef CONFIG_IGB_PTP
5747 /**
5748  * igb_tx_hwtstamp - utility function which checks for TX time stamp
5749  * @q_vector: pointer to q_vector containing needed info
5750  * @buffer: pointer to igb_tx_buffer structure
5751  *
5752  * If we were asked to do hardware stamping and such a time stamp is
5753  * available, then it must have been for this skb here because we only
5754  * allow only one such packet into the queue.
5755  */
5756 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5757                             struct igb_tx_buffer *buffer_info)
5758 {
5759         struct igb_adapter *adapter = q_vector->adapter;
5760         struct e1000_hw *hw = &adapter->hw;
5761         struct skb_shared_hwtstamps shhwtstamps;
5762         u64 regval;
5763
5764         /* if skb does not support hw timestamp or TX stamp not valid exit */
5765         if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
5766             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5767                 return;
5768
5769         regval = rd32(E1000_TXSTMPL);
5770         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5771
5772         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5773         skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5774 }
5775
5776 #endif
5777 /**
5778  * igb_clean_tx_irq - Reclaim resources after transmit completes
5779  * @q_vector: pointer to q_vector containing needed info
5780  *
5781  * returns true if ring is completely cleaned
5782  **/
5783 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5784 {
5785         struct igb_adapter *adapter = q_vector->adapter;
5786         struct igb_ring *tx_ring = q_vector->tx.ring;
5787         struct igb_tx_buffer *tx_buffer;
5788         union e1000_adv_tx_desc *tx_desc, *eop_desc;
5789         unsigned int total_bytes = 0, total_packets = 0;
5790         unsigned int budget = q_vector->tx.work_limit;
5791         unsigned int i = tx_ring->next_to_clean;
5792
5793         if (test_bit(__IGB_DOWN, &adapter->state))
5794                 return true;
5795
5796         tx_buffer = &tx_ring->tx_buffer_info[i];
5797         tx_desc = IGB_TX_DESC(tx_ring, i);
5798         i -= tx_ring->count;
5799
5800         for (; budget; budget--) {
5801                 eop_desc = tx_buffer->next_to_watch;
5802
5803                 /* prevent any other reads prior to eop_desc */
5804                 rmb();
5805
5806                 /* if next_to_watch is not set then there is no work pending */
5807                 if (!eop_desc)
5808                         break;
5809
5810                 /* if DD is not set pending work has not been completed */
5811                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5812                         break;
5813
5814                 /* clear next_to_watch to prevent false hangs */
5815                 tx_buffer->next_to_watch = NULL;
5816
5817                 /* update the statistics for this packet */
5818                 total_bytes += tx_buffer->bytecount;
5819                 total_packets += tx_buffer->gso_segs;
5820
5821 #ifdef CONFIG_IGB_PTP
5822                 /* retrieve hardware timestamp */
5823                 igb_tx_hwtstamp(q_vector, tx_buffer);
5824
5825 #endif
5826                 /* free the skb */
5827                 dev_kfree_skb_any(tx_buffer->skb);
5828                 tx_buffer->skb = NULL;
5829
5830                 /* unmap skb header data */
5831                 dma_unmap_single(tx_ring->dev,
5832                                  tx_buffer->dma,
5833                                  tx_buffer->length,
5834                                  DMA_TO_DEVICE);
5835
5836                 /* clear last DMA location and unmap remaining buffers */
5837                 while (tx_desc != eop_desc) {
5838                         tx_buffer->dma = 0;
5839
5840                         tx_buffer++;
5841                         tx_desc++;
5842                         i++;
5843                         if (unlikely(!i)) {
5844                                 i -= tx_ring->count;
5845                                 tx_buffer = tx_ring->tx_buffer_info;
5846                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
5847                         }
5848
5849                         /* unmap any remaining paged data */
5850                         if (tx_buffer->dma) {
5851                                 dma_unmap_page(tx_ring->dev,
5852                                                tx_buffer->dma,
5853                                                tx_buffer->length,
5854                                                DMA_TO_DEVICE);
5855                         }
5856                 }
5857
5858                 /* clear last DMA location */
5859                 tx_buffer->dma = 0;
5860
5861                 /* move us one more past the eop_desc for start of next pkt */
5862                 tx_buffer++;
5863                 tx_desc++;
5864                 i++;
5865                 if (unlikely(!i)) {
5866                         i -= tx_ring->count;
5867                         tx_buffer = tx_ring->tx_buffer_info;
5868                         tx_desc = IGB_TX_DESC(tx_ring, 0);
5869                 }
5870         }
5871
5872         netdev_tx_completed_queue(txring_txq(tx_ring),
5873                                   total_packets, total_bytes);
5874         i += tx_ring->count;
5875         tx_ring->next_to_clean = i;
5876         u64_stats_update_begin(&tx_ring->tx_syncp);
5877         tx_ring->tx_stats.bytes += total_bytes;
5878         tx_ring->tx_stats.packets += total_packets;
5879         u64_stats_update_end(&tx_ring->tx_syncp);
5880         q_vector->tx.total_bytes += total_bytes;
5881         q_vector->tx.total_packets += total_packets;
5882
5883         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
5884                 struct e1000_hw *hw = &adapter->hw;
5885
5886                 eop_desc = tx_buffer->next_to_watch;
5887
5888                 /* Detect a transmit hang in hardware, this serializes the
5889                  * check with the clearing of time_stamp and movement of i */
5890                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5891                 if (eop_desc &&
5892                     time_after(jiffies, tx_buffer->time_stamp +
5893                                (adapter->tx_timeout_factor * HZ)) &&
5894                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5895
5896                         /* detected Tx unit hang */
5897                         dev_err(tx_ring->dev,
5898                                 "Detected Tx Unit Hang\n"
5899                                 "  Tx Queue             <%d>\n"
5900                                 "  TDH                  <%x>\n"
5901                                 "  TDT                  <%x>\n"
5902                                 "  next_to_use          <%x>\n"
5903                                 "  next_to_clean        <%x>\n"
5904                                 "buffer_info[next_to_clean]\n"
5905                                 "  time_stamp           <%lx>\n"
5906                                 "  next_to_watch        <%p>\n"
5907                                 "  jiffies              <%lx>\n"
5908                                 "  desc.status          <%x>\n",
5909                                 tx_ring->queue_index,
5910                                 rd32(E1000_TDH(tx_ring->reg_idx)),
5911                                 readl(tx_ring->tail),
5912                                 tx_ring->next_to_use,
5913                                 tx_ring->next_to_clean,
5914                                 tx_buffer->time_stamp,
5915                                 eop_desc,
5916                                 jiffies,
5917                                 eop_desc->wb.status);
5918                         netif_stop_subqueue(tx_ring->netdev,
5919                                             tx_ring->queue_index);
5920
5921                         /* we are about to reset, no point in enabling stuff */
5922                         return true;
5923                 }
5924         }
5925
5926         if (unlikely(total_packets &&
5927                      netif_carrier_ok(tx_ring->netdev) &&
5928                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5929                 /* Make sure that anybody stopping the queue after this
5930                  * sees the new next_to_clean.
5931                  */
5932                 smp_mb();
5933                 if (__netif_subqueue_stopped(tx_ring->netdev,
5934                                              tx_ring->queue_index) &&
5935                     !(test_bit(__IGB_DOWN, &adapter->state))) {
5936                         netif_wake_subqueue(tx_ring->netdev,
5937                                             tx_ring->queue_index);
5938
5939                         u64_stats_update_begin(&tx_ring->tx_syncp);
5940                         tx_ring->tx_stats.restart_queue++;
5941                         u64_stats_update_end(&tx_ring->tx_syncp);
5942                 }
5943         }
5944
5945         return !!budget;
5946 }
5947
5948 static inline void igb_rx_checksum(struct igb_ring *ring,
5949                                    union e1000_adv_rx_desc *rx_desc,
5950                                    struct sk_buff *skb)
5951 {
5952         skb_checksum_none_assert(skb);
5953
5954         /* Ignore Checksum bit is set */
5955         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
5956                 return;
5957
5958         /* Rx checksum disabled via ethtool */
5959         if (!(ring->netdev->features & NETIF_F_RXCSUM))
5960                 return;
5961
5962         /* TCP/UDP checksum error bit is set */
5963         if (igb_test_staterr(rx_desc,
5964                              E1000_RXDEXT_STATERR_TCPE |
5965                              E1000_RXDEXT_STATERR_IPE)) {
5966                 /*
5967                  * work around errata with sctp packets where the TCPE aka
5968                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5969                  * packets, (aka let the stack check the crc32c)
5970                  */
5971                 if (!((skb->len == 60) &&
5972                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
5973                         u64_stats_update_begin(&ring->rx_syncp);
5974                         ring->rx_stats.csum_err++;
5975                         u64_stats_update_end(&ring->rx_syncp);
5976                 }
5977                 /* let the stack verify checksum errors */
5978                 return;
5979         }
5980         /* It must be a TCP or UDP packet with a valid checksum */
5981         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5982                                       E1000_RXD_STAT_UDPCS))
5983                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5984
5985         dev_dbg(ring->dev, "cksum success: bits %08X\n",
5986                 le32_to_cpu(rx_desc->wb.upper.status_error));
5987 }
5988
5989 static inline void igb_rx_hash(struct igb_ring *ring,
5990                                union e1000_adv_rx_desc *rx_desc,
5991                                struct sk_buff *skb)
5992 {
5993         if (ring->netdev->features & NETIF_F_RXHASH)
5994                 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5995 }
5996
5997 #ifdef CONFIG_IGB_PTP
5998 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5999                             union e1000_adv_rx_desc *rx_desc,
6000                             struct sk_buff *skb)
6001 {
6002         struct igb_adapter *adapter = q_vector->adapter;
6003         struct e1000_hw *hw = &adapter->hw;
6004         u64 regval;
6005
6006         if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6007                                        E1000_RXDADV_STAT_TS))
6008                 return;
6009
6010         /*
6011          * If this bit is set, then the RX registers contain the time stamp. No
6012          * other packet will be time stamped until we read these registers, so
6013          * read the registers to make them available again. Because only one
6014          * packet can be time stamped at a time, we know that the register
6015          * values must belong to this one here and therefore we don't need to
6016          * compare any of the additional attributes stored for it.
6017          *
6018          * If nothing went wrong, then it should have a shared tx_flags that we
6019          * can turn into a skb_shared_hwtstamps.
6020          */
6021         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
6022                 u32 *stamp = (u32 *)skb->data;
6023                 regval = le32_to_cpu(*(stamp + 2));
6024                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6025                 skb_pull(skb, IGB_TS_HDR_LEN);
6026         } else {
6027                 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6028                         return;
6029
6030                 regval = rd32(E1000_RXSTMPL);
6031                 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
6032         }
6033
6034         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6035 }
6036
6037 #endif
6038 static void igb_rx_vlan(struct igb_ring *ring,
6039                         union e1000_adv_rx_desc *rx_desc,
6040                         struct sk_buff *skb)
6041 {
6042         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6043                 u16 vid;
6044                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6045                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6046                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6047                 else
6048                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6049
6050                 __vlan_hwaccel_put_tag(skb, vid);
6051         }
6052 }
6053
6054 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
6055 {
6056         /* HW will not DMA in data larger than the given buffer, even if it
6057          * parses the (NFS, of course) header to be larger.  In that case, it
6058          * fills the header buffer and spills the rest into the page.
6059          */
6060         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6061                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
6062         if (hlen > IGB_RX_HDR_LEN)
6063                 hlen = IGB_RX_HDR_LEN;
6064         return hlen;
6065 }
6066
6067 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
6068 {
6069         struct igb_ring *rx_ring = q_vector->rx.ring;
6070         union e1000_adv_rx_desc *rx_desc;
6071         const int current_node = numa_node_id();
6072         unsigned int total_bytes = 0, total_packets = 0;
6073         u16 cleaned_count = igb_desc_unused(rx_ring);
6074         u16 i = rx_ring->next_to_clean;
6075
6076         rx_desc = IGB_RX_DESC(rx_ring, i);
6077
6078         while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
6079                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
6080                 struct sk_buff *skb = buffer_info->skb;
6081                 union e1000_adv_rx_desc *next_rxd;
6082
6083                 buffer_info->skb = NULL;
6084                 prefetch(skb->data);
6085
6086                 i++;
6087                 if (i == rx_ring->count)
6088                         i = 0;
6089
6090                 next_rxd = IGB_RX_DESC(rx_ring, i);
6091                 prefetch(next_rxd);
6092
6093                 /*
6094                  * This memory barrier is needed to keep us from reading
6095                  * any other fields out of the rx_desc until we know the
6096                  * RXD_STAT_DD bit is set
6097                  */
6098                 rmb();
6099
6100                 if (!skb_is_nonlinear(skb)) {
6101                         __skb_put(skb, igb_get_hlen(rx_desc));
6102                         dma_unmap_single(rx_ring->dev, buffer_info->dma,
6103                                          IGB_RX_HDR_LEN,
6104                                          DMA_FROM_DEVICE);
6105                         buffer_info->dma = 0;
6106                 }
6107
6108                 if (rx_desc->wb.upper.length) {
6109                         u16 length = le16_to_cpu(rx_desc->wb.upper.length);
6110
6111                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
6112                                                 buffer_info->page,
6113                                                 buffer_info->page_offset,
6114                                                 length);
6115
6116                         skb->len += length;
6117                         skb->data_len += length;
6118                         skb->truesize += PAGE_SIZE / 2;
6119
6120                         if ((page_count(buffer_info->page) != 1) ||
6121                             (page_to_nid(buffer_info->page) != current_node))
6122                                 buffer_info->page = NULL;
6123                         else
6124                                 get_page(buffer_info->page);
6125
6126                         dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6127                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
6128                         buffer_info->page_dma = 0;
6129                 }
6130
6131                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
6132                         struct igb_rx_buffer *next_buffer;
6133                         next_buffer = &rx_ring->rx_buffer_info[i];
6134                         buffer_info->skb = next_buffer->skb;
6135                         buffer_info->dma = next_buffer->dma;
6136                         next_buffer->skb = skb;
6137                         next_buffer->dma = 0;
6138                         goto next_desc;
6139                 }
6140
6141                 if (unlikely((igb_test_staterr(rx_desc,
6142                                                E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6143                              && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
6144                         dev_kfree_skb_any(skb);
6145                         goto next_desc;
6146                 }
6147
6148 #ifdef CONFIG_IGB_PTP
6149                 igb_rx_hwtstamp(q_vector, rx_desc, skb);
6150 #endif
6151                 igb_rx_hash(rx_ring, rx_desc, skb);
6152                 igb_rx_checksum(rx_ring, rx_desc, skb);
6153                 igb_rx_vlan(rx_ring, rx_desc, skb);
6154
6155                 total_bytes += skb->len;
6156                 total_packets++;
6157
6158                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6159
6160                 napi_gro_receive(&q_vector->napi, skb);
6161
6162                 budget--;
6163 next_desc:
6164                 if (!budget)
6165                         break;
6166
6167                 cleaned_count++;
6168                 /* return some buffers to hardware, one at a time is too slow */
6169                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
6170                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
6171                         cleaned_count = 0;
6172                 }
6173
6174                 /* use prefetched values */
6175                 rx_desc = next_rxd;
6176         }
6177
6178         rx_ring->next_to_clean = i;
6179         u64_stats_update_begin(&rx_ring->rx_syncp);
6180         rx_ring->rx_stats.packets += total_packets;
6181         rx_ring->rx_stats.bytes += total_bytes;
6182         u64_stats_update_end(&rx_ring->rx_syncp);
6183         q_vector->rx.total_packets += total_packets;
6184         q_vector->rx.total_bytes += total_bytes;
6185
6186         if (cleaned_count)
6187                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6188
6189         return !!budget;
6190 }
6191
6192 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
6193                                  struct igb_rx_buffer *bi)
6194 {
6195         struct sk_buff *skb = bi->skb;
6196         dma_addr_t dma = bi->dma;
6197
6198         if (dma)
6199                 return true;
6200
6201         if (likely(!skb)) {
6202                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6203                                                 IGB_RX_HDR_LEN);
6204                 bi->skb = skb;
6205                 if (!skb) {
6206                         rx_ring->rx_stats.alloc_failed++;
6207                         return false;
6208                 }
6209
6210                 /* initialize skb for ring */
6211                 skb_record_rx_queue(skb, rx_ring->queue_index);
6212         }
6213
6214         dma = dma_map_single(rx_ring->dev, skb->data,
6215                              IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6216
6217         if (dma_mapping_error(rx_ring->dev, dma)) {
6218                 rx_ring->rx_stats.alloc_failed++;
6219                 return false;
6220         }
6221
6222         bi->dma = dma;
6223         return true;
6224 }
6225
6226 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6227                                   struct igb_rx_buffer *bi)
6228 {
6229         struct page *page = bi->page;
6230         dma_addr_t page_dma = bi->page_dma;
6231         unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6232
6233         if (page_dma)
6234                 return true;
6235
6236         if (!page) {
6237                 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
6238                 bi->page = page;
6239                 if (unlikely(!page)) {
6240                         rx_ring->rx_stats.alloc_failed++;
6241                         return false;
6242                 }
6243         }
6244
6245         page_dma = dma_map_page(rx_ring->dev, page,
6246                                 page_offset, PAGE_SIZE / 2,
6247                                 DMA_FROM_DEVICE);
6248
6249         if (dma_mapping_error(rx_ring->dev, page_dma)) {
6250                 rx_ring->rx_stats.alloc_failed++;
6251                 return false;
6252         }
6253
6254         bi->page_dma = page_dma;
6255         bi->page_offset = page_offset;
6256         return true;
6257 }
6258
6259 /**
6260  * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6261  * @adapter: address of board private structure
6262  **/
6263 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6264 {
6265         union e1000_adv_rx_desc *rx_desc;
6266         struct igb_rx_buffer *bi;
6267         u16 i = rx_ring->next_to_use;
6268
6269         rx_desc = IGB_RX_DESC(rx_ring, i);
6270         bi = &rx_ring->rx_buffer_info[i];
6271         i -= rx_ring->count;
6272
6273         while (cleaned_count--) {
6274                 if (!igb_alloc_mapped_skb(rx_ring, bi))
6275                         break;
6276
6277                 /* Refresh the desc even if buffer_addrs didn't change
6278                  * because each write-back erases this info. */
6279                 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
6280
6281                 if (!igb_alloc_mapped_page(rx_ring, bi))
6282                         break;
6283
6284                 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
6285
6286                 rx_desc++;
6287                 bi++;
6288                 i++;
6289                 if (unlikely(!i)) {
6290                         rx_desc = IGB_RX_DESC(rx_ring, 0);
6291                         bi = rx_ring->rx_buffer_info;
6292                         i -= rx_ring->count;
6293                 }
6294
6295                 /* clear the hdr_addr for the next_to_use descriptor */
6296                 rx_desc->read.hdr_addr = 0;
6297         }
6298
6299         i += rx_ring->count;
6300
6301         if (rx_ring->next_to_use != i) {
6302                 rx_ring->next_to_use = i;
6303
6304                 /* Force memory writes to complete before letting h/w
6305                  * know there are new descriptors to fetch.  (Only
6306                  * applicable for weak-ordered memory model archs,
6307                  * such as IA-64). */
6308                 wmb();
6309                 writel(i, rx_ring->tail);
6310         }
6311 }
6312
6313 /**
6314  * igb_mii_ioctl -
6315  * @netdev:
6316  * @ifreq:
6317  * @cmd:
6318  **/
6319 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6320 {
6321         struct igb_adapter *adapter = netdev_priv(netdev);
6322         struct mii_ioctl_data *data = if_mii(ifr);
6323
6324         if (adapter->hw.phy.media_type != e1000_media_type_copper)
6325                 return -EOPNOTSUPP;
6326
6327         switch (cmd) {
6328         case SIOCGMIIPHY:
6329                 data->phy_id = adapter->hw.phy.addr;
6330                 break;
6331         case SIOCGMIIREG:
6332                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6333                                      &data->val_out))
6334                         return -EIO;
6335                 break;
6336         case SIOCSMIIREG:
6337         default:
6338                 return -EOPNOTSUPP;
6339         }
6340         return 0;
6341 }
6342
6343 /**
6344  * igb_hwtstamp_ioctl - control hardware time stamping
6345  * @netdev:
6346  * @ifreq:
6347  * @cmd:
6348  *
6349  * Outgoing time stamping can be enabled and disabled. Play nice and
6350  * disable it when requested, although it shouldn't case any overhead
6351  * when no packet needs it. At most one packet in the queue may be
6352  * marked for time stamping, otherwise it would be impossible to tell
6353  * for sure to which packet the hardware time stamp belongs.
6354  *
6355  * Incoming time stamping has to be configured via the hardware
6356  * filters. Not all combinations are supported, in particular event
6357  * type has to be specified. Matching the kind of event packet is
6358  * not supported, with the exception of "all V2 events regardless of
6359  * level 2 or 4".
6360  *
6361  **/
6362 static int igb_hwtstamp_ioctl(struct net_device *netdev,
6363                               struct ifreq *ifr, int cmd)
6364 {
6365         struct igb_adapter *adapter = netdev_priv(netdev);
6366         struct e1000_hw *hw = &adapter->hw;
6367         struct hwtstamp_config config;
6368         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6369         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6370         u32 tsync_rx_cfg = 0;
6371         bool is_l4 = false;
6372         bool is_l2 = false;
6373         u32 regval;
6374
6375         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6376                 return -EFAULT;
6377
6378         /* reserved for future extensions */
6379         if (config.flags)
6380                 return -EINVAL;
6381
6382         switch (config.tx_type) {
6383         case HWTSTAMP_TX_OFF:
6384                 tsync_tx_ctl = 0;
6385         case HWTSTAMP_TX_ON:
6386                 break;
6387         default:
6388                 return -ERANGE;
6389         }
6390
6391         switch (config.rx_filter) {
6392         case HWTSTAMP_FILTER_NONE:
6393                 tsync_rx_ctl = 0;
6394                 break;
6395         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6396         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6397         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6398         case HWTSTAMP_FILTER_ALL:
6399                 /*
6400                  * register TSYNCRXCFG must be set, therefore it is not
6401                  * possible to time stamp both Sync and Delay_Req messages
6402                  * => fall back to time stamping all packets
6403                  */
6404                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6405                 config.rx_filter = HWTSTAMP_FILTER_ALL;
6406                 break;
6407         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
6408                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6409                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
6410                 is_l4 = true;
6411                 break;
6412         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
6413                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6414                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
6415                 is_l4 = true;
6416                 break;
6417         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6418         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
6419                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6420                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
6421                 is_l2 = true;
6422                 is_l4 = true;
6423                 config.rx_filter = HWTSTAMP_FILTER_SOME;
6424                 break;
6425         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6426         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
6427                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6428                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
6429                 is_l2 = true;
6430                 is_l4 = true;
6431                 config.rx_filter = HWTSTAMP_FILTER_SOME;
6432                 break;
6433         case HWTSTAMP_FILTER_PTP_V2_EVENT:
6434         case HWTSTAMP_FILTER_PTP_V2_SYNC:
6435         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
6436                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
6437                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
6438                 is_l2 = true;
6439                 is_l4 = true;
6440                 break;
6441         default:
6442                 return -ERANGE;
6443         }
6444
6445         if (hw->mac.type == e1000_82575) {
6446                 if (tsync_rx_ctl | tsync_tx_ctl)
6447                         return -EINVAL;
6448                 return 0;
6449         }
6450
6451         /*
6452          * Per-packet timestamping only works if all packets are
6453          * timestamped, so enable timestamping in all packets as
6454          * long as one rx filter was configured.
6455          */
6456         if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
6457                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6458                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6459         }
6460
6461         /* enable/disable TX */
6462         regval = rd32(E1000_TSYNCTXCTL);
6463         regval &= ~E1000_TSYNCTXCTL_ENABLED;
6464         regval |= tsync_tx_ctl;
6465         wr32(E1000_TSYNCTXCTL, regval);
6466
6467         /* enable/disable RX */
6468         regval = rd32(E1000_TSYNCRXCTL);
6469         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6470         regval |= tsync_rx_ctl;
6471         wr32(E1000_TSYNCRXCTL, regval);
6472
6473         /* define which PTP packets are time stamped */
6474         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6475
6476         /* define ethertype filter for timestamped packets */
6477         if (is_l2)
6478                 wr32(E1000_ETQF(3),
6479                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6480                                  E1000_ETQF_1588 | /* enable timestamping */
6481                                  ETH_P_1588));     /* 1588 eth protocol type */
6482         else
6483                 wr32(E1000_ETQF(3), 0);
6484
6485 #define PTP_PORT 319
6486         /* L4 Queue Filter[3]: filter by destination port and protocol */
6487         if (is_l4) {
6488                 u32 ftqf = (IPPROTO_UDP /* UDP */
6489                         | E1000_FTQF_VF_BP /* VF not compared */
6490                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6491                         | E1000_FTQF_MASK); /* mask all inputs */
6492                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6493
6494                 wr32(E1000_IMIR(3), htons(PTP_PORT));
6495                 wr32(E1000_IMIREXT(3),
6496                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6497                 if (hw->mac.type == e1000_82576) {
6498                         /* enable source port check */
6499                         wr32(E1000_SPQF(3), htons(PTP_PORT));
6500                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6501                 }
6502                 wr32(E1000_FTQF(3), ftqf);
6503         } else {
6504                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6505         }
6506         wrfl();
6507
6508         adapter->hwtstamp_config = config;
6509
6510         /* clear TX/RX time stamp registers, just to be sure */
6511         regval = rd32(E1000_TXSTMPH);
6512         regval = rd32(E1000_RXSTMPH);
6513
6514         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6515                 -EFAULT : 0;
6516 }
6517
6518 /**
6519  * igb_ioctl -
6520  * @netdev:
6521  * @ifreq:
6522  * @cmd:
6523  **/
6524 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6525 {
6526         switch (cmd) {
6527         case SIOCGMIIPHY:
6528         case SIOCGMIIREG:
6529         case SIOCSMIIREG:
6530                 return igb_mii_ioctl(netdev, ifr, cmd);
6531         case SIOCSHWTSTAMP:
6532                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6533         default:
6534                 return -EOPNOTSUPP;
6535         }
6536 }
6537
6538 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6539 {
6540         struct igb_adapter *adapter = hw->back;
6541         u16 cap_offset;
6542
6543         cap_offset = adapter->pdev->pcie_cap;
6544         if (!cap_offset)
6545                 return -E1000_ERR_CONFIG;
6546
6547         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6548
6549         return 0;
6550 }
6551
6552 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6553 {
6554         struct igb_adapter *adapter = hw->back;
6555         u16 cap_offset;
6556
6557         cap_offset = adapter->pdev->pcie_cap;
6558         if (!cap_offset)
6559                 return -E1000_ERR_CONFIG;
6560
6561         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6562
6563         return 0;
6564 }
6565
6566 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6567 {
6568         struct igb_adapter *adapter = netdev_priv(netdev);
6569         struct e1000_hw *hw = &adapter->hw;
6570         u32 ctrl, rctl;
6571         bool enable = !!(features & NETIF_F_HW_VLAN_RX);
6572
6573         if (enable) {
6574                 /* enable VLAN tag insert/strip */
6575                 ctrl = rd32(E1000_CTRL);
6576                 ctrl |= E1000_CTRL_VME;
6577                 wr32(E1000_CTRL, ctrl);
6578
6579                 /* Disable CFI check */
6580                 rctl = rd32(E1000_RCTL);
6581                 rctl &= ~E1000_RCTL_CFIEN;
6582                 wr32(E1000_RCTL, rctl);
6583         } else {
6584                 /* disable VLAN tag insert/strip */
6585                 ctrl = rd32(E1000_CTRL);
6586                 ctrl &= ~E1000_CTRL_VME;
6587                 wr32(E1000_CTRL, ctrl);
6588         }
6589
6590         igb_rlpml_set(adapter);
6591 }
6592
6593 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6594 {
6595         struct igb_adapter *adapter = netdev_priv(netdev);
6596         struct e1000_hw *hw = &adapter->hw;
6597         int pf_id = adapter->vfs_allocated_count;
6598
6599         /* attempt to add filter to vlvf array */
6600         igb_vlvf_set(adapter, vid, true, pf_id);
6601
6602         /* add the filter since PF can receive vlans w/o entry in vlvf */
6603         igb_vfta_set(hw, vid, true);
6604
6605         set_bit(vid, adapter->active_vlans);
6606
6607         return 0;
6608 }
6609
6610 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6611 {
6612         struct igb_adapter *adapter = netdev_priv(netdev);
6613         struct e1000_hw *hw = &adapter->hw;
6614         int pf_id = adapter->vfs_allocated_count;
6615         s32 err;
6616
6617         /* remove vlan from VLVF table array */
6618         err = igb_vlvf_set(adapter, vid, false, pf_id);
6619
6620         /* if vid was not present in VLVF just remove it from table */
6621         if (err)
6622                 igb_vfta_set(hw, vid, false);
6623
6624         clear_bit(vid, adapter->active_vlans);
6625
6626         return 0;
6627 }
6628
6629 static void igb_restore_vlan(struct igb_adapter *adapter)
6630 {
6631         u16 vid;
6632
6633         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6634
6635         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6636                 igb_vlan_rx_add_vid(adapter->netdev, vid);
6637 }
6638
6639 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6640 {
6641         struct pci_dev *pdev = adapter->pdev;
6642         struct e1000_mac_info *mac = &adapter->hw.mac;
6643
6644         mac->autoneg = 0;
6645
6646         /* Make sure dplx is at most 1 bit and lsb of speed is not set
6647          * for the switch() below to work */
6648         if ((spd & 1) || (dplx & ~1))
6649                 goto err_inval;
6650
6651         /* Fiber NIC's only allow 1000 Gbps Full duplex */
6652         if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6653             spd != SPEED_1000 &&
6654             dplx != DUPLEX_FULL)
6655                 goto err_inval;
6656
6657         switch (spd + dplx) {
6658         case SPEED_10 + DUPLEX_HALF:
6659                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6660                 break;
6661         case SPEED_10 + DUPLEX_FULL:
6662                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6663                 break;
6664         case SPEED_100 + DUPLEX_HALF:
6665                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6666                 break;
6667         case SPEED_100 + DUPLEX_FULL:
6668                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6669                 break;
6670         case SPEED_1000 + DUPLEX_FULL:
6671                 mac->autoneg = 1;
6672                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6673                 break;
6674         case SPEED_1000 + DUPLEX_HALF: /* not supported */
6675         default:
6676                 goto err_inval;
6677         }
6678
6679         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6680         adapter->hw.phy.mdix = AUTO_ALL_MODES;
6681
6682         return 0;
6683
6684 err_inval:
6685         dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6686         return -EINVAL;
6687 }
6688
6689 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6690                           bool runtime)
6691 {
6692         struct net_device *netdev = pci_get_drvdata(pdev);
6693         struct igb_adapter *adapter = netdev_priv(netdev);
6694         struct e1000_hw *hw = &adapter->hw;
6695         u32 ctrl, rctl, status;
6696         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
6697 #ifdef CONFIG_PM
6698         int retval = 0;
6699 #endif
6700
6701         netif_device_detach(netdev);
6702
6703         if (netif_running(netdev))
6704                 __igb_close(netdev, true);
6705
6706         igb_clear_interrupt_scheme(adapter);
6707
6708 #ifdef CONFIG_PM
6709         retval = pci_save_state(pdev);
6710         if (retval)
6711                 return retval;
6712 #endif
6713
6714         status = rd32(E1000_STATUS);
6715         if (status & E1000_STATUS_LU)
6716                 wufc &= ~E1000_WUFC_LNKC;
6717
6718         if (wufc) {
6719                 igb_setup_rctl(adapter);
6720                 igb_set_rx_mode(netdev);
6721
6722                 /* turn on all-multi mode if wake on multicast is enabled */
6723                 if (wufc & E1000_WUFC_MC) {
6724                         rctl = rd32(E1000_RCTL);
6725                         rctl |= E1000_RCTL_MPE;
6726                         wr32(E1000_RCTL, rctl);
6727                 }
6728
6729                 ctrl = rd32(E1000_CTRL);
6730                 /* advertise wake from D3Cold */
6731                 #define E1000_CTRL_ADVD3WUC 0x00100000
6732                 /* phy power management enable */
6733                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6734                 ctrl |= E1000_CTRL_ADVD3WUC;
6735                 wr32(E1000_CTRL, ctrl);
6736
6737                 /* Allow time for pending master requests to run */
6738                 igb_disable_pcie_master(hw);
6739
6740                 wr32(E1000_WUC, E1000_WUC_PME_EN);
6741                 wr32(E1000_WUFC, wufc);
6742         } else {
6743                 wr32(E1000_WUC, 0);
6744                 wr32(E1000_WUFC, 0);
6745         }
6746
6747         *enable_wake = wufc || adapter->en_mng_pt;
6748         if (!*enable_wake)
6749                 igb_power_down_link(adapter);
6750         else
6751                 igb_power_up_link(adapter);
6752
6753         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6754          * would have already happened in close and is redundant. */
6755         igb_release_hw_control(adapter);
6756
6757         pci_disable_device(pdev);
6758
6759         return 0;
6760 }
6761
6762 #ifdef CONFIG_PM
6763 #ifdef CONFIG_PM_SLEEP
6764 static int igb_suspend(struct device *dev)
6765 {
6766         int retval;
6767         bool wake;
6768         struct pci_dev *pdev = to_pci_dev(dev);
6769
6770         retval = __igb_shutdown(pdev, &wake, 0);
6771         if (retval)
6772                 return retval;
6773
6774         if (wake) {
6775                 pci_prepare_to_sleep(pdev);
6776         } else {
6777                 pci_wake_from_d3(pdev, false);
6778                 pci_set_power_state(pdev, PCI_D3hot);
6779         }
6780
6781         return 0;
6782 }
6783 #endif /* CONFIG_PM_SLEEP */
6784
6785 static int igb_resume(struct device *dev)
6786 {
6787         struct pci_dev *pdev = to_pci_dev(dev);
6788         struct net_device *netdev = pci_get_drvdata(pdev);
6789         struct igb_adapter *adapter = netdev_priv(netdev);
6790         struct e1000_hw *hw = &adapter->hw;
6791         u32 err;
6792
6793         pci_set_power_state(pdev, PCI_D0);
6794         pci_restore_state(pdev);
6795         pci_save_state(pdev);
6796
6797         err = pci_enable_device_mem(pdev);
6798         if (err) {
6799                 dev_err(&pdev->dev,
6800                         "igb: Cannot enable PCI device from suspend\n");
6801                 return err;
6802         }
6803         pci_set_master(pdev);
6804
6805         pci_enable_wake(pdev, PCI_D3hot, 0);
6806         pci_enable_wake(pdev, PCI_D3cold, 0);
6807
6808         if (igb_init_interrupt_scheme(adapter)) {
6809                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6810                 return -ENOMEM;
6811         }
6812
6813         igb_reset(adapter);
6814
6815         /* let the f/w know that the h/w is now under the control of the
6816          * driver. */
6817         igb_get_hw_control(adapter);
6818
6819         wr32(E1000_WUS, ~0);
6820
6821         if (netdev->flags & IFF_UP) {
6822                 err = __igb_open(netdev, true);
6823                 if (err)
6824                         return err;
6825         }
6826
6827         netif_device_attach(netdev);
6828         return 0;
6829 }
6830
6831 #ifdef CONFIG_PM_RUNTIME
6832 static int igb_runtime_idle(struct device *dev)
6833 {
6834         struct pci_dev *pdev = to_pci_dev(dev);
6835         struct net_device *netdev = pci_get_drvdata(pdev);
6836         struct igb_adapter *adapter = netdev_priv(netdev);
6837
6838         if (!igb_has_link(adapter))
6839                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6840
6841         return -EBUSY;
6842 }
6843
6844 static int igb_runtime_suspend(struct device *dev)
6845 {
6846         struct pci_dev *pdev = to_pci_dev(dev);
6847         int retval;
6848         bool wake;
6849
6850         retval = __igb_shutdown(pdev, &wake, 1);
6851         if (retval)
6852                 return retval;
6853
6854         if (wake) {
6855                 pci_prepare_to_sleep(pdev);
6856         } else {
6857                 pci_wake_from_d3(pdev, false);
6858                 pci_set_power_state(pdev, PCI_D3hot);
6859         }
6860
6861         return 0;
6862 }
6863
6864 static int igb_runtime_resume(struct device *dev)
6865 {
6866         return igb_resume(dev);
6867 }
6868 #endif /* CONFIG_PM_RUNTIME */
6869 #endif
6870
6871 static void igb_shutdown(struct pci_dev *pdev)
6872 {
6873         bool wake;
6874
6875         __igb_shutdown(pdev, &wake, 0);
6876
6877         if (system_state == SYSTEM_POWER_OFF) {
6878                 pci_wake_from_d3(pdev, wake);
6879                 pci_set_power_state(pdev, PCI_D3hot);
6880         }
6881 }
6882
6883 #ifdef CONFIG_NET_POLL_CONTROLLER
6884 /*
6885  * Polling 'interrupt' - used by things like netconsole to send skbs
6886  * without having to re-enable interrupts. It's not called while
6887  * the interrupt routine is executing.
6888  */
6889 static void igb_netpoll(struct net_device *netdev)
6890 {
6891         struct igb_adapter *adapter = netdev_priv(netdev);
6892         struct e1000_hw *hw = &adapter->hw;
6893         struct igb_q_vector *q_vector;
6894         int i;
6895
6896         for (i = 0; i < adapter->num_q_vectors; i++) {
6897                 q_vector = adapter->q_vector[i];
6898                 if (adapter->msix_entries)
6899                         wr32(E1000_EIMC, q_vector->eims_value);
6900                 else
6901                         igb_irq_disable(adapter);
6902                 napi_schedule(&q_vector->napi);
6903         }
6904 }
6905 #endif /* CONFIG_NET_POLL_CONTROLLER */
6906
6907 /**
6908  * igb_io_error_detected - called when PCI error is detected
6909  * @pdev: Pointer to PCI device
6910  * @state: The current pci connection state
6911  *
6912  * This function is called after a PCI bus error affecting
6913  * this device has been detected.
6914  */
6915 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6916                                               pci_channel_state_t state)
6917 {
6918         struct net_device *netdev = pci_get_drvdata(pdev);
6919         struct igb_adapter *adapter = netdev_priv(netdev);
6920
6921         netif_device_detach(netdev);
6922
6923         if (state == pci_channel_io_perm_failure)
6924                 return PCI_ERS_RESULT_DISCONNECT;
6925
6926         if (netif_running(netdev))
6927                 igb_down(adapter);
6928         pci_disable_device(pdev);
6929
6930         /* Request a slot slot reset. */
6931         return PCI_ERS_RESULT_NEED_RESET;
6932 }
6933
6934 /**
6935  * igb_io_slot_reset - called after the pci bus has been reset.
6936  * @pdev: Pointer to PCI device
6937  *
6938  * Restart the card from scratch, as if from a cold-boot. Implementation
6939  * resembles the first-half of the igb_resume routine.
6940  */
6941 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6942 {
6943         struct net_device *netdev = pci_get_drvdata(pdev);
6944         struct igb_adapter *adapter = netdev_priv(netdev);
6945         struct e1000_hw *hw = &adapter->hw;
6946         pci_ers_result_t result;
6947         int err;
6948
6949         if (pci_enable_device_mem(pdev)) {
6950                 dev_err(&pdev->dev,
6951                         "Cannot re-enable PCI device after reset.\n");
6952                 result = PCI_ERS_RESULT_DISCONNECT;
6953         } else {
6954                 pci_set_master(pdev);
6955                 pci_restore_state(pdev);
6956                 pci_save_state(pdev);
6957
6958                 pci_enable_wake(pdev, PCI_D3hot, 0);
6959                 pci_enable_wake(pdev, PCI_D3cold, 0);
6960
6961                 igb_reset(adapter);
6962                 wr32(E1000_WUS, ~0);
6963                 result = PCI_ERS_RESULT_RECOVERED;
6964         }
6965
6966         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6967         if (err) {
6968                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6969                         "failed 0x%0x\n", err);
6970                 /* non-fatal, continue */
6971         }
6972
6973         return result;
6974 }
6975
6976 /**
6977  * igb_io_resume - called when traffic can start flowing again.
6978  * @pdev: Pointer to PCI device
6979  *
6980  * This callback is called when the error recovery driver tells us that
6981  * its OK to resume normal operation. Implementation resembles the
6982  * second-half of the igb_resume routine.
6983  */
6984 static void igb_io_resume(struct pci_dev *pdev)
6985 {
6986         struct net_device *netdev = pci_get_drvdata(pdev);
6987         struct igb_adapter *adapter = netdev_priv(netdev);
6988
6989         if (netif_running(netdev)) {
6990                 if (igb_up(adapter)) {
6991                         dev_err(&pdev->dev, "igb_up failed after reset\n");
6992                         return;
6993                 }
6994         }
6995
6996         netif_device_attach(netdev);
6997
6998         /* let the f/w know that the h/w is now under the control of the
6999          * driver. */
7000         igb_get_hw_control(adapter);
7001 }
7002
7003 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
7004                              u8 qsel)
7005 {
7006         u32 rar_low, rar_high;
7007         struct e1000_hw *hw = &adapter->hw;
7008
7009         /* HW expects these in little endian so we reverse the byte order
7010          * from network order (big endian) to little endian
7011          */
7012         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7013                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7014         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
7015
7016         /* Indicate to hardware the Address is Valid. */
7017         rar_high |= E1000_RAH_AV;
7018
7019         if (hw->mac.type == e1000_82575)
7020                 rar_high |= E1000_RAH_POOL_1 * qsel;
7021         else
7022                 rar_high |= E1000_RAH_POOL_1 << qsel;
7023
7024         wr32(E1000_RAL(index), rar_low);
7025         wrfl();
7026         wr32(E1000_RAH(index), rar_high);
7027         wrfl();
7028 }
7029
7030 static int igb_set_vf_mac(struct igb_adapter *adapter,
7031                           int vf, unsigned char *mac_addr)
7032 {
7033         struct e1000_hw *hw = &adapter->hw;
7034         /* VF MAC addresses start at end of receive addresses and moves
7035          * torwards the first, as a result a collision should not be possible */
7036         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
7037
7038         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
7039
7040         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
7041
7042         return 0;
7043 }
7044
7045 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
7046 {
7047         struct igb_adapter *adapter = netdev_priv(netdev);
7048         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
7049                 return -EINVAL;
7050         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
7051         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7052         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
7053                                       " change effective.");
7054         if (test_bit(__IGB_DOWN, &adapter->state)) {
7055                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
7056                          " but the PF device is not up.\n");
7057                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
7058                          " attempting to use the VF device.\n");
7059         }
7060         return igb_set_vf_mac(adapter, vf, mac);
7061 }
7062
7063 static int igb_link_mbps(int internal_link_speed)
7064 {
7065         switch (internal_link_speed) {
7066         case SPEED_100:
7067                 return 100;
7068         case SPEED_1000:
7069                 return 1000;
7070         default:
7071                 return 0;
7072         }
7073 }
7074
7075 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
7076                                   int link_speed)
7077 {
7078         int rf_dec, rf_int;
7079         u32 bcnrc_val;
7080
7081         if (tx_rate != 0) {
7082                 /* Calculate the rate factor values to set */
7083                 rf_int = link_speed / tx_rate;
7084                 rf_dec = (link_speed - (rf_int * tx_rate));
7085                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
7086
7087                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7088                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
7089                                E1000_RTTBCNRC_RF_INT_MASK);
7090                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
7091         } else {
7092                 bcnrc_val = 0;
7093         }
7094
7095         wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7096         /*
7097          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
7098          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
7099          */
7100         wr32(E1000_RTTBCNRM, 0x14);
7101         wr32(E1000_RTTBCNRC, bcnrc_val);
7102 }
7103
7104 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7105 {
7106         int actual_link_speed, i;
7107         bool reset_rate = false;
7108
7109         /* VF TX rate limit was not set or not supported */
7110         if ((adapter->vf_rate_link_speed == 0) ||
7111             (adapter->hw.mac.type != e1000_82576))
7112                 return;
7113
7114         actual_link_speed = igb_link_mbps(adapter->link_speed);
7115         if (actual_link_speed != adapter->vf_rate_link_speed) {
7116                 reset_rate = true;
7117                 adapter->vf_rate_link_speed = 0;
7118                 dev_info(&adapter->pdev->dev,
7119                          "Link speed has been changed. VF Transmit "
7120                          "rate is disabled\n");
7121         }
7122
7123         for (i = 0; i < adapter->vfs_allocated_count; i++) {
7124                 if (reset_rate)
7125                         adapter->vf_data[i].tx_rate = 0;
7126
7127                 igb_set_vf_rate_limit(&adapter->hw, i,
7128                                       adapter->vf_data[i].tx_rate,
7129                                       actual_link_speed);
7130         }
7131 }
7132
7133 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7134 {
7135         struct igb_adapter *adapter = netdev_priv(netdev);
7136         struct e1000_hw *hw = &adapter->hw;
7137         int actual_link_speed;
7138
7139         if (hw->mac.type != e1000_82576)
7140                 return -EOPNOTSUPP;
7141
7142         actual_link_speed = igb_link_mbps(adapter->link_speed);
7143         if ((vf >= adapter->vfs_allocated_count) ||
7144             (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7145             (tx_rate < 0) || (tx_rate > actual_link_speed))
7146                 return -EINVAL;
7147
7148         adapter->vf_rate_link_speed = actual_link_speed;
7149         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7150         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7151
7152         return 0;
7153 }
7154
7155 static int igb_ndo_get_vf_config(struct net_device *netdev,
7156                                  int vf, struct ifla_vf_info *ivi)
7157 {
7158         struct igb_adapter *adapter = netdev_priv(netdev);
7159         if (vf >= adapter->vfs_allocated_count)
7160                 return -EINVAL;
7161         ivi->vf = vf;
7162         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
7163         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
7164         ivi->vlan = adapter->vf_data[vf].pf_vlan;
7165         ivi->qos = adapter->vf_data[vf].pf_qos;
7166         return 0;
7167 }
7168
7169 static void igb_vmm_control(struct igb_adapter *adapter)
7170 {
7171         struct e1000_hw *hw = &adapter->hw;
7172         u32 reg;
7173
7174         switch (hw->mac.type) {
7175         case e1000_82575:
7176         case e1000_i210:
7177         case e1000_i211:
7178         default:
7179                 /* replication is not supported for 82575 */
7180                 return;
7181         case e1000_82576:
7182                 /* notify HW that the MAC is adding vlan tags */
7183                 reg = rd32(E1000_DTXCTL);
7184                 reg |= E1000_DTXCTL_VLAN_ADDED;
7185                 wr32(E1000_DTXCTL, reg);
7186         case e1000_82580:
7187                 /* enable replication vlan tag stripping */
7188                 reg = rd32(E1000_RPLOLR);
7189                 reg |= E1000_RPLOLR_STRVLAN;
7190                 wr32(E1000_RPLOLR, reg);
7191         case e1000_i350:
7192                 /* none of the above registers are supported by i350 */
7193                 break;
7194         }
7195
7196         if (adapter->vfs_allocated_count) {
7197                 igb_vmdq_set_loopback_pf(hw, true);
7198                 igb_vmdq_set_replication_pf(hw, true);
7199                 igb_vmdq_set_anti_spoofing_pf(hw, true,
7200                                                 adapter->vfs_allocated_count);
7201         } else {
7202                 igb_vmdq_set_loopback_pf(hw, false);
7203                 igb_vmdq_set_replication_pf(hw, false);
7204         }
7205 }
7206
7207 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7208 {
7209         struct e1000_hw *hw = &adapter->hw;
7210         u32 dmac_thr;
7211         u16 hwm;
7212
7213         if (hw->mac.type > e1000_82580) {
7214                 if (adapter->flags & IGB_FLAG_DMAC) {
7215                         u32 reg;
7216
7217                         /* force threshold to 0. */
7218                         wr32(E1000_DMCTXTH, 0);
7219
7220                         /*
7221                          * DMA Coalescing high water mark needs to be greater
7222                          * than the Rx threshold. Set hwm to PBA - max frame
7223                          * size in 16B units, capping it at PBA - 6KB.
7224                          */
7225                         hwm = 64 * pba - adapter->max_frame_size / 16;
7226                         if (hwm < 64 * (pba - 6))
7227                                 hwm = 64 * (pba - 6);
7228                         reg = rd32(E1000_FCRTC);
7229                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7230                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7231                                 & E1000_FCRTC_RTH_COAL_MASK);
7232                         wr32(E1000_FCRTC, reg);
7233
7234                         /*
7235                          * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7236                          * frame size, capping it at PBA - 10KB.
7237                          */
7238                         dmac_thr = pba - adapter->max_frame_size / 512;
7239                         if (dmac_thr < pba - 10)
7240                                 dmac_thr = pba - 10;
7241                         reg = rd32(E1000_DMACR);
7242                         reg &= ~E1000_DMACR_DMACTHR_MASK;
7243                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7244                                 & E1000_DMACR_DMACTHR_MASK);
7245
7246                         /* transition to L0x or L1 if available..*/
7247                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7248
7249                         /* watchdog timer= +-1000 usec in 32usec intervals */
7250                         reg |= (1000 >> 5);
7251
7252                         /* Disable BMC-to-OS Watchdog Enable */
7253                         reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
7254                         wr32(E1000_DMACR, reg);
7255
7256                         /*
7257                          * no lower threshold to disable
7258                          * coalescing(smart fifb)-UTRESH=0
7259                          */
7260                         wr32(E1000_DMCRTRH, 0);
7261
7262                         reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7263
7264                         wr32(E1000_DMCTLX, reg);
7265
7266                         /*
7267                          * free space in tx packet buffer to wake from
7268                          * DMA coal
7269                          */
7270                         wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7271                              (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7272
7273                         /*
7274                          * make low power state decision controlled
7275                          * by DMA coal
7276                          */
7277                         reg = rd32(E1000_PCIEMISC);
7278                         reg &= ~E1000_PCIEMISC_LX_DECISION;
7279                         wr32(E1000_PCIEMISC, reg);
7280                 } /* endif adapter->dmac is not disabled */
7281         } else if (hw->mac.type == e1000_82580) {
7282                 u32 reg = rd32(E1000_PCIEMISC);
7283                 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7284                 wr32(E1000_DMACR, 0);
7285         }
7286 }
7287
7288 /* igb_main.c */