drivers/net/ethernet/intel/e1000/e1000_main.c

   1 /*******************************************************************************
   2
   3   Intel PRO/1000 Linux driver
   4   Copyright(c) 1999 - 2006 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   Linux NICS <linux.nics@intel.com>
  24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #include "e1000.h"
  30 #include <net/ip6_checksum.h>
  31 #include <linux/io.h>
  32 #include <linux/prefetch.h>
  33 #include <linux/bitops.h>
  34 #include <linux/if_vlan.h>
  35
  36 char e1000_driver_name[] = "e1000";
  37 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  38 #define DRV_VERSION "7.3.21-k8-NAPI"
  39 const char e1000_driver_version[] = DRV_VERSION;
  40 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  41
  42 /* e1000_pci_tbl - PCI Device ID Table
  43  *
  44  * Last entry must be all 0s
  45  *
  46  * Macro expands to...
  47  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  48  */
  49 static const struct pci_device_id e1000_pci_tbl[] = {
  50         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  53         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  54         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  55         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  56         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  57         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  58         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  59         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  60         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  61         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  62         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  63         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  64         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  65         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  66         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  67         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  68         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  69         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  70         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  71         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  72         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  73         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  74         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  75         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  76         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  77         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  78         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  79         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  80         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  81         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  82         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  83         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  84         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  85         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  86         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  87         /* required last entry */
  88         {0,}
  89 };
  90
  91 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  92
  93 int e1000_up(struct e1000_adapter *adapter);
  94 void e1000_down(struct e1000_adapter *adapter);
  95 void e1000_reinit_locked(struct e1000_adapter *adapter);
  96 void e1000_reset(struct e1000_adapter *adapter);
  97 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  98 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  99 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
 100 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
 101 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
 102                              struct e1000_tx_ring *txdr);
 103 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
 104                              struct e1000_rx_ring *rxdr);
 105 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
 106                              struct e1000_tx_ring *tx_ring);
 107 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
 108                              struct e1000_rx_ring *rx_ring);
 109 void e1000_update_stats(struct e1000_adapter *adapter);
 110
 111 static int e1000_init_module(void);
 112 static void e1000_exit_module(void);
 113 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 114 static void e1000_remove(struct pci_dev *pdev);
 115 static int e1000_alloc_queues(struct e1000_adapter *adapter);
 116 static int e1000_sw_init(struct e1000_adapter *adapter);
 117 static int e1000_open(struct net_device *netdev);
 118 static int e1000_close(struct net_device *netdev);
 119 static void e1000_configure_tx(struct e1000_adapter *adapter);
 120 static void e1000_configure_rx(struct e1000_adapter *adapter);
 121 static void e1000_setup_rctl(struct e1000_adapter *adapter);
 122 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
 123 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
 124 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
 125                                 struct e1000_tx_ring *tx_ring);
 126 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 127                                 struct e1000_rx_ring *rx_ring);
 128 static void e1000_set_rx_mode(struct net_device *netdev);
 129 static void e1000_update_phy_info_task(struct work_struct *work);
 130 static void e1000_watchdog(struct work_struct *work);
 131 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 132 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 133                                     struct net_device *netdev);
 134 static struct net_device_stats * e1000_get_stats(struct net_device *netdev);
 135 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 136 static int e1000_set_mac(struct net_device *netdev, void *p);
 137 static irqreturn_t e1000_intr(int irq, void *data);
 138 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 139                                struct e1000_tx_ring *tx_ring);
 140 static int e1000_clean(struct napi_struct *napi, int budget);
 141 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 142                                struct e1000_rx_ring *rx_ring,
 143                                int *work_done, int work_to_do);
 144 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 145                                      struct e1000_rx_ring *rx_ring,
 146                                      int *work_done, int work_to_do);
 147 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 148                                    struct e1000_rx_ring *rx_ring,
 149                                    int cleaned_count);
 150 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 151                                          struct e1000_rx_ring *rx_ring,
 152                                          int cleaned_count);
 153 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 154 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 155                            int cmd);
 156 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 157 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 158 static void e1000_tx_timeout(struct net_device *dev);
 159 static void e1000_reset_task(struct work_struct *work);
 160 static void e1000_smartspeed(struct e1000_adapter *adapter);
 161 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 162                                        struct sk_buff *skb);
 163
 164 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 165 static void e1000_vlan_mode(struct net_device *netdev,
 166                             netdev_features_t features);
 167 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 168                                      bool filter_on);
 169 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 170                                  __be16 proto, u16 vid);
 171 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 172                                   __be16 proto, u16 vid);
 173 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 174
 175 #ifdef CONFIG_PM
 176 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
 177 static int e1000_resume(struct pci_dev *pdev);
 178 #endif
 179 static void e1000_shutdown(struct pci_dev *pdev);
 180
 181 #ifdef CONFIG_NET_POLL_CONTROLLER
 182 /* for netdump / net console */
 183 static void e1000_netpoll (struct net_device *netdev);
 184 #endif
 185
 186 #define COPYBREAK_DEFAULT 256
 187 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 188 module_param(copybreak, uint, 0644);
 189 MODULE_PARM_DESC(copybreak,
 190         "Maximum size of packet that is copied to a new buffer on receive");
 191
 192 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 193                      pci_channel_state_t state);
 194 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 195 static void e1000_io_resume(struct pci_dev *pdev);
 196
 197 static const struct pci_error_handlers e1000_err_handler = {
 198         .error_detected = e1000_io_error_detected,
 199         .slot_reset = e1000_io_slot_reset,
 200         .resume = e1000_io_resume,
 201 };
 202
 203 static struct pci_driver e1000_driver = {
 204         .name     = e1000_driver_name,
 205         .id_table = e1000_pci_tbl,
 206         .probe    = e1000_probe,
 207         .remove   = e1000_remove,
 208 #ifdef CONFIG_PM
 209         /* Power Management Hooks */
 210         .suspend  = e1000_suspend,
 211         .resume   = e1000_resume,
 212 #endif
 213         .shutdown = e1000_shutdown,
 214         .err_handler = &e1000_err_handler
 215 };
 216
 217 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 218 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 219 MODULE_LICENSE("GPL");
 220 MODULE_VERSION(DRV_VERSION);
 221
 222 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 223 static int debug = -1;
 224 module_param(debug, int, 0);
 225 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 226
 227 /**
 228  * e1000_get_hw_dev - return device
 229  * used by hardware layer to print debugging information
 230  *
 231  **/
 232 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 233 {
 234         struct e1000_adapter *adapter = hw->back;
 235         return adapter->netdev;
 236 }
 237
 238 /**
 239  * e1000_init_module - Driver Registration Routine
 240  *
 241  * e1000_init_module is the first routine called when the driver is
 242  * loaded. All it does is register with the PCI subsystem.
 243  **/
 244 static int __init e1000_init_module(void)
 245 {
 246         int ret;
 247         pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
 248
 249         pr_info("%s\n", e1000_copyright);
 250
 251         ret = pci_register_driver(&e1000_driver);
 252         if (copybreak != COPYBREAK_DEFAULT) {
 253                 if (copybreak == 0)
 254                         pr_info("copybreak disabled\n");
 255                 else
 256                         pr_info("copybreak enabled for "
 257                                    "packets <= %u bytes\n", copybreak);
 258         }
 259         return ret;
 260 }
 261
 262 module_init(e1000_init_module);
 263
 264 /**
 265  * e1000_exit_module - Driver Exit Cleanup Routine
 266  *
 267  * e1000_exit_module is called just before the driver is removed
 268  * from memory.
 269  **/
 270 static void __exit e1000_exit_module(void)
 271 {
 272         pci_unregister_driver(&e1000_driver);
 273 }
 274
 275 module_exit(e1000_exit_module);
 276
 277 static int e1000_request_irq(struct e1000_adapter *adapter)
 278 {
 279         struct net_device *netdev = adapter->netdev;
 280         irq_handler_t handler = e1000_intr;
 281         int irq_flags = IRQF_SHARED;
 282         int err;
 283
 284         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 285                           netdev);
 286         if (err) {
 287                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 288         }
 289
 290         return err;
 291 }
 292
 293 static void e1000_free_irq(struct e1000_adapter *adapter)
 294 {
 295         struct net_device *netdev = adapter->netdev;
 296
 297         free_irq(adapter->pdev->irq, netdev);
 298 }
 299
 300 /**
 301  * e1000_irq_disable - Mask off interrupt generation on the NIC
 302  * @adapter: board private structure
 303  **/
 304 static void e1000_irq_disable(struct e1000_adapter *adapter)
 305 {
 306         struct e1000_hw *hw = &adapter->hw;
 307
 308         ew32(IMC, ~0);
 309         E1000_WRITE_FLUSH();
 310         synchronize_irq(adapter->pdev->irq);
 311 }
 312
 313 /**
 314  * e1000_irq_enable - Enable default interrupt generation settings
 315  * @adapter: board private structure
 316  **/
 317 static void e1000_irq_enable(struct e1000_adapter *adapter)
 318 {
 319         struct e1000_hw *hw = &adapter->hw;
 320
 321         ew32(IMS, IMS_ENABLE_MASK);
 322         E1000_WRITE_FLUSH();
 323 }
 324
 325 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 326 {
 327         struct e1000_hw *hw = &adapter->hw;
 328         struct net_device *netdev = adapter->netdev;
 329         u16 vid = hw->mng_cookie.vlan_id;
 330         u16 old_vid = adapter->mng_vlan_id;
 331
 332         if (!e1000_vlan_used(adapter))
 333                 return;
 334
 335         if (!test_bit(vid, adapter->active_vlans)) {
 336                 if (hw->mng_cookie.status &
 337                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 338                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 339                         adapter->mng_vlan_id = vid;
 340                 } else {
 341                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 342                 }
 343                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 344                     (vid != old_vid) &&
 345                     !test_bit(old_vid, adapter->active_vlans))
 346                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 347                                                old_vid);
 348         } else {
 349                 adapter->mng_vlan_id = vid;
 350         }
 351 }
 352
 353 static void e1000_init_manageability(struct e1000_adapter *adapter)
 354 {
 355         struct e1000_hw *hw = &adapter->hw;
 356
 357         if (adapter->en_mng_pt) {
 358                 u32 manc = er32(MANC);
 359
 360                 /* disable hardware interception of ARP */
 361                 manc &= ~(E1000_MANC_ARP_EN);
 362
 363                 ew32(MANC, manc);
 364         }
 365 }
 366
 367 static void e1000_release_manageability(struct e1000_adapter *adapter)
 368 {
 369         struct e1000_hw *hw = &adapter->hw;
 370
 371         if (adapter->en_mng_pt) {
 372                 u32 manc = er32(MANC);
 373
 374                 /* re-enable hardware interception of ARP */
 375                 manc |= E1000_MANC_ARP_EN;
 376
 377                 ew32(MANC, manc);
 378         }
 379 }
 380
 381 /**
 382  * e1000_configure - configure the hardware for RX and TX
 383  * @adapter = private board structure
 384  **/
 385 static void e1000_configure(struct e1000_adapter *adapter)
 386 {
 387         struct net_device *netdev = adapter->netdev;
 388         int i;
 389
 390         e1000_set_rx_mode(netdev);
 391
 392         e1000_restore_vlan(adapter);
 393         e1000_init_manageability(adapter);
 394
 395         e1000_configure_tx(adapter);
 396         e1000_setup_rctl(adapter);
 397         e1000_configure_rx(adapter);
 398         /* call E1000_DESC_UNUSED which always leaves
 399          * at least 1 descriptor unused to make sure
 400          * next_to_use != next_to_clean
 401          */
 402         for (i = 0; i < adapter->num_rx_queues; i++) {
 403                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 404                 adapter->alloc_rx_buf(adapter, ring,
 405                                       E1000_DESC_UNUSED(ring));
 406         }
 407 }
 408
 409 int e1000_up(struct e1000_adapter *adapter)
 410 {
 411         struct e1000_hw *hw = &adapter->hw;
 412
 413         /* hardware has been reset, we need to reload some things */
 414         e1000_configure(adapter);
 415
 416         clear_bit(__E1000_DOWN, &adapter->flags);
 417
 418         napi_enable(&adapter->napi);
 419
 420         e1000_irq_enable(adapter);
 421
 422         netif_wake_queue(adapter->netdev);
 423
 424         /* fire a link change interrupt to start the watchdog */
 425         ew32(ICS, E1000_ICS_LSC);
 426         return 0;
 427 }
 428
 429 /**
 430  * e1000_power_up_phy - restore link in case the phy was powered down
 431  * @adapter: address of board private structure
 432  *
 433  * The phy may be powered down to save power and turn off link when the
 434  * driver is unloaded and wake on lan is not enabled (among others)
 435  * *** this routine MUST be followed by a call to e1000_reset ***
 436  **/
 437 void e1000_power_up_phy(struct e1000_adapter *adapter)
 438 {
 439         struct e1000_hw *hw = &adapter->hw;
 440         u16 mii_reg = 0;
 441
 442         /* Just clear the power down bit to wake the phy back up */
 443         if (hw->media_type == e1000_media_type_copper) {
 444                 /* according to the manual, the phy will retain its
 445                  * settings across a power-down/up cycle
 446                  */
 447                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 448                 mii_reg &= ~MII_CR_POWER_DOWN;
 449                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 450         }
 451 }
 452
 453 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 454 {
 455         struct e1000_hw *hw = &adapter->hw;
 456
 457         /* Power down the PHY so no link is implied when interface is down *
 458          * The PHY cannot be powered down if any of the following is true *
 459          * (a) WoL is enabled
 460          * (b) AMT is active
 461          * (c) SoL/IDER session is active
 462          */
 463         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 464            hw->media_type == e1000_media_type_copper) {
 465                 u16 mii_reg = 0;
 466
 467                 switch (hw->mac_type) {
 468                 case e1000_82540:
 469                 case e1000_82545:
 470                 case e1000_82545_rev_3:
 471                 case e1000_82546:
 472                 case e1000_ce4100:
 473                 case e1000_82546_rev_3:
 474                 case e1000_82541:
 475                 case e1000_82541_rev_2:
 476                 case e1000_82547:
 477                 case e1000_82547_rev_2:
 478                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 479                                 goto out;
 480                         break;
 481                 default:
 482                         goto out;
 483                 }
 484                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 485                 mii_reg |= MII_CR_POWER_DOWN;
 486                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 487                 msleep(1);
 488         }
 489 out:
 490         return;
 491 }
 492
 493 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 494 {
 495         set_bit(__E1000_DOWN, &adapter->flags);
 496
 497         cancel_delayed_work_sync(&adapter->watchdog_task);
 498
 499         /*
 500          * Since the watchdog task can reschedule other tasks, we should cancel
 501          * it first, otherwise we can run into the situation when a work is
 502          * still running after the adapter has been turned down.
 503          */
 504
 505         cancel_delayed_work_sync(&adapter->phy_info_task);
 506         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 507
 508         /* Only kill reset task if adapter is not resetting */
 509         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 510                 cancel_work_sync(&adapter->reset_task);
 511 }
 512
 513 void e1000_down(struct e1000_adapter *adapter)
 514 {
 515         struct e1000_hw *hw = &adapter->hw;
 516         struct net_device *netdev = adapter->netdev;
 517         u32 rctl, tctl;
 518
 519
 520         /* disable receives in the hardware */
 521         rctl = er32(RCTL);
 522         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 523         /* flush and sleep below */
 524
 525         netif_tx_disable(netdev);
 526
 527         /* disable transmits in the hardware */
 528         tctl = er32(TCTL);
 529         tctl &= ~E1000_TCTL_EN;
 530         ew32(TCTL, tctl);
 531         /* flush both disables and wait for them to finish */
 532         E1000_WRITE_FLUSH();
 533         msleep(10);
 534
 535         napi_disable(&adapter->napi);
 536
 537         e1000_irq_disable(adapter);
 538
 539         /* Setting DOWN must be after irq_disable to prevent
 540          * a screaming interrupt.  Setting DOWN also prevents
 541          * tasks from rescheduling.
 542          */
 543         e1000_down_and_stop(adapter);
 544
 545         adapter->link_speed = 0;
 546         adapter->link_duplex = 0;
 547         netif_carrier_off(netdev);
 548
 549         e1000_reset(adapter);
 550         e1000_clean_all_tx_rings(adapter);
 551         e1000_clean_all_rx_rings(adapter);
 552 }
 553
 554 void e1000_reinit_locked(struct e1000_adapter *adapter)
 555 {
 556         WARN_ON(in_interrupt());
 557         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 558                 msleep(1);
 559         e1000_down(adapter);
 560         e1000_up(adapter);
 561         clear_bit(__E1000_RESETTING, &adapter->flags);
 562 }
 563
 564 void e1000_reset(struct e1000_adapter *adapter)
 565 {
 566         struct e1000_hw *hw = &adapter->hw;
 567         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 568         bool legacy_pba_adjust = false;
 569         u16 hwm;
 570
 571         /* Repartition Pba for greater than 9k mtu
 572          * To take effect CTRL.RST is required.
 573          */
 574
 575         switch (hw->mac_type) {
 576         case e1000_82542_rev2_0:
 577         case e1000_82542_rev2_1:
 578         case e1000_82543:
 579         case e1000_82544:
 580         case e1000_82540:
 581         case e1000_82541:
 582         case e1000_82541_rev_2:
 583                 legacy_pba_adjust = true;
 584                 pba = E1000_PBA_48K;
 585                 break;
 586         case e1000_82545:
 587         case e1000_82545_rev_3:
 588         case e1000_82546:
 589         case e1000_ce4100:
 590         case e1000_82546_rev_3:
 591                 pba = E1000_PBA_48K;
 592                 break;
 593         case e1000_82547:
 594         case e1000_82547_rev_2:
 595                 legacy_pba_adjust = true;
 596                 pba = E1000_PBA_30K;
 597                 break;
 598         case e1000_undefined:
 599         case e1000_num_macs:
 600                 break;
 601         }
 602
 603         if (legacy_pba_adjust) {
 604                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 605                         pba -= 8; /* allocate more FIFO for Tx */
 606
 607                 if (hw->mac_type == e1000_82547) {
 608                         adapter->tx_fifo_head = 0;
 609                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 610                         adapter->tx_fifo_size =
 611                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 612                         atomic_set(&adapter->tx_fifo_stall, 0);
 613                 }
 614         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 615                 /* adjust PBA for jumbo frames */
 616                 ew32(PBA, pba);
 617
 618                 /* To maintain wire speed transmits, the Tx FIFO should be
 619                  * large enough to accommodate two full transmit packets,
 620                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 621                  * the Rx FIFO should be large enough to accommodate at least
 622                  * one full receive packet and is similarly rounded up and
 623                  * expressed in KB.
 624                  */
 625                 pba = er32(PBA);
 626                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 627                 tx_space = pba >> 16;
 628                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 629                 pba &= 0xffff;
 630                 /* the Tx fifo also stores 16 bytes of information about the Tx
 631                  * but don't include ethernet FCS because hardware appends it
 632                  */
 633                 min_tx_space = (hw->max_frame_size +
 634                                 sizeof(struct e1000_tx_desc) -
 635                                 ETH_FCS_LEN) * 2;
 636                 min_tx_space = ALIGN(min_tx_space, 1024);
 637                 min_tx_space >>= 10;
 638                 /* software strips receive CRC, so leave room for it */
 639                 min_rx_space = hw->max_frame_size;
 640                 min_rx_space = ALIGN(min_rx_space, 1024);
 641                 min_rx_space >>= 10;
 642
 643                 /* If current Tx allocation is less than the min Tx FIFO size,
 644                  * and the min Tx FIFO size is less than the current Rx FIFO
 645                  * allocation, take space away from current Rx allocation
 646                  */
 647                 if (tx_space < min_tx_space &&
 648                     ((min_tx_space - tx_space) < pba)) {
 649                         pba = pba - (min_tx_space - tx_space);
 650
 651                         /* PCI/PCIx hardware has PBA alignment constraints */
 652                         switch (hw->mac_type) {
 653                         case e1000_82545 ... e1000_82546_rev_3:
 654                                 pba &= ~(E1000_PBA_8K - 1);
 655                                 break;
 656                         default:
 657                                 break;
 658                         }
 659
 660                         /* if short on Rx space, Rx wins and must trump Tx
 661                          * adjustment or use Early Receive if available
 662                          */
 663                         if (pba < min_rx_space)
 664                                 pba = min_rx_space;
 665                 }
 666         }
 667
 668         ew32(PBA, pba);
 669
 670         /* flow control settings:
 671          * The high water mark must be low enough to fit one full frame
 672          * (or the size used for early receive) above it in the Rx FIFO.
 673          * Set it to the lower of:
 674          * - 90% of the Rx FIFO size, and
 675          * - the full Rx FIFO size minus the early receive size (for parts
 676          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 677          * - the full Rx FIFO size minus one full frame
 678          */
 679         hwm = min(((pba << 10) * 9 / 10),
 680                   ((pba << 10) - hw->max_frame_size));
 681
 682         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 683         hw->fc_low_water = hw->fc_high_water - 8;
 684         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 685         hw->fc_send_xon = 1;
 686         hw->fc = hw->original_fc;
 687
 688         /* Allow time for pending master requests to run */
 689         e1000_reset_hw(hw);
 690         if (hw->mac_type >= e1000_82544)
 691                 ew32(WUC, 0);
 692
 693         if (e1000_init_hw(hw))
 694                 e_dev_err("Hardware Error\n");
 695         e1000_update_mng_vlan(adapter);
 696
 697         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 698         if (hw->mac_type >= e1000_82544 &&
 699             hw->autoneg == 1 &&
 700             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 701                 u32 ctrl = er32(CTRL);
 702                 /* clear phy power management bit if we are in gig only mode,
 703                  * which if enabled will attempt negotiation to 100Mb, which
 704                  * can cause a loss of link at power off or driver unload
 705                  */
 706                 ctrl &= ~E1000_CTRL_SWDPIN3;
 707                 ew32(CTRL, ctrl);
 708         }
 709
 710         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 711         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 712
 713         e1000_reset_adaptive(hw);
 714         e1000_phy_get_info(hw, &adapter->phy_info);
 715
 716         e1000_release_manageability(adapter);
 717 }
 718
 719 /* Dump the eeprom for users having checksum issues */
 720 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 721 {
 722         struct net_device *netdev = adapter->netdev;
 723         struct ethtool_eeprom eeprom;
 724         const struct ethtool_ops *ops = netdev->ethtool_ops;
 725         u8 *data;
 726         int i;
 727         u16 csum_old, csum_new = 0;
 728
 729         eeprom.len = ops->get_eeprom_len(netdev);
 730         eeprom.offset = 0;
 731
 732         data = kmalloc(eeprom.len, GFP_KERNEL);
 733         if (!data)
 734                 return;
 735
 736         ops->get_eeprom(netdev, &eeprom, data);
 737
 738         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 739                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 740         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 741                 csum_new += data[i] + (data[i + 1] << 8);
 742         csum_new = EEPROM_SUM - csum_new;
 743
 744         pr_err("/*********************/\n");
 745         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 746         pr_err("Calculated              : 0x%04x\n", csum_new);
 747
 748         pr_err("Offset    Values\n");
 749         pr_err("========  ======\n");
 750         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 751
 752         pr_err("Include this output when contacting your support provider.\n");
 753         pr_err("This is not a software error! Something bad happened to\n");
 754         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 755         pr_err("result in further problems, possibly loss of data,\n");
 756         pr_err("corruption or system hangs!\n");
 757         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 758         pr_err("which is invalid and requires you to set the proper MAC\n");
 759         pr_err("address manually before continuing to enable this network\n");
 760         pr_err("device. Please inspect the EEPROM dump and report the\n");
 761         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 762         pr_err("/*********************/\n");
 763
 764         kfree(data);
 765 }
 766
 767 /**
 768  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 769  * @pdev: PCI device information struct
 770  *
 771  * Return true if an adapter needs ioport resources
 772  **/
 773 static int e1000_is_need_ioport(struct pci_dev *pdev)
 774 {
 775         switch (pdev->device) {
 776         case E1000_DEV_ID_82540EM:
 777         case E1000_DEV_ID_82540EM_LOM:
 778         case E1000_DEV_ID_82540EP:
 779         case E1000_DEV_ID_82540EP_LOM:
 780         case E1000_DEV_ID_82540EP_LP:
 781         case E1000_DEV_ID_82541EI:
 782         case E1000_DEV_ID_82541EI_MOBILE:
 783         case E1000_DEV_ID_82541ER:
 784         case E1000_DEV_ID_82541ER_LOM:
 785         case E1000_DEV_ID_82541GI:
 786         case E1000_DEV_ID_82541GI_LF:
 787         case E1000_DEV_ID_82541GI_MOBILE:
 788         case E1000_DEV_ID_82544EI_COPPER:
 789         case E1000_DEV_ID_82544EI_FIBER:
 790         case E1000_DEV_ID_82544GC_COPPER:
 791         case E1000_DEV_ID_82544GC_LOM:
 792         case E1000_DEV_ID_82545EM_COPPER:
 793         case E1000_DEV_ID_82545EM_FIBER:
 794         case E1000_DEV_ID_82546EB_COPPER:
 795         case E1000_DEV_ID_82546EB_FIBER:
 796         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 797                 return true;
 798         default:
 799                 return false;
 800         }
 801 }
 802
 803 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 804         netdev_features_t features)
 805 {
 806         /* Since there is no support for separate Rx/Tx vlan accel
 807          * enable/disable make sure Tx flag is always in same state as Rx.
 808          */
 809         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 810                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 811         else
 812                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 813
 814         return features;
 815 }
 816
 817 static int e1000_set_features(struct net_device *netdev,
 818         netdev_features_t features)
 819 {
 820         struct e1000_adapter *adapter = netdev_priv(netdev);
 821         netdev_features_t changed = features ^ netdev->features;
 822
 823         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 824                 e1000_vlan_mode(netdev, features);
 825
 826         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 827                 return 0;
 828
 829         netdev->features = features;
 830         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 831
 832         if (netif_running(netdev))
 833                 e1000_reinit_locked(adapter);
 834         else
 835                 e1000_reset(adapter);
 836
 837         return 0;
 838 }
 839
 840 static const struct net_device_ops e1000_netdev_ops = {
 841         .ndo_open               = e1000_open,
 842         .ndo_stop               = e1000_close,
 843         .ndo_start_xmit         = e1000_xmit_frame,
 844         .ndo_get_stats          = e1000_get_stats,
 845         .ndo_set_rx_mode        = e1000_set_rx_mode,
 846         .ndo_set_mac_address    = e1000_set_mac,
 847         .ndo_tx_timeout         = e1000_tx_timeout,
 848         .ndo_change_mtu         = e1000_change_mtu,
 849         .ndo_do_ioctl           = e1000_ioctl,
 850         .ndo_validate_addr      = eth_validate_addr,
 851         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 852         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 853 #ifdef CONFIG_NET_POLL_CONTROLLER
 854         .ndo_poll_controller    = e1000_netpoll,
 855 #endif
 856         .ndo_fix_features       = e1000_fix_features,
 857         .ndo_set_features       = e1000_set_features,
 858 };
 859
 860 /**
 861  * e1000_init_hw_struct - initialize members of hw struct
 862  * @adapter: board private struct
 863  * @hw: structure used by e1000_hw.c
 864  *
 865  * Factors out initialization of the e1000_hw struct to its own function
 866  * that can be called very early at init (just after struct allocation).
 867  * Fields are initialized based on PCI device information and
 868  * OS network device settings (MTU size).
 869  * Returns negative error codes if MAC type setup fails.
 870  */
 871 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 872                                 struct e1000_hw *hw)
 873 {
 874         struct pci_dev *pdev = adapter->pdev;
 875
 876         /* PCI config space info */
 877         hw->vendor_id = pdev->vendor;
 878         hw->device_id = pdev->device;
 879         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 880         hw->subsystem_id = pdev->subsystem_device;
 881         hw->revision_id = pdev->revision;
 882
 883         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 884
 885         hw->max_frame_size = adapter->netdev->mtu +
 886                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 887         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 888
 889         /* identify the MAC */
 890         if (e1000_set_mac_type(hw)) {
 891                 e_err(probe, "Unknown MAC Type\n");
 892                 return -EIO;
 893         }
 894
 895         switch (hw->mac_type) {
 896         default:
 897                 break;
 898         case e1000_82541:
 899         case e1000_82547:
 900         case e1000_82541_rev_2:
 901         case e1000_82547_rev_2:
 902                 hw->phy_init_script = 1;
 903                 break;
 904         }
 905
 906         e1000_set_media_type(hw);
 907         e1000_get_bus_info(hw);
 908
 909         hw->wait_autoneg_complete = false;
 910         hw->tbi_compatibility_en = true;
 911         hw->adaptive_ifs = true;
 912
 913         /* Copper options */
 914
 915         if (hw->media_type == e1000_media_type_copper) {
 916                 hw->mdix = AUTO_ALL_MODES;
 917                 hw->disable_polarity_correction = false;
 918                 hw->master_slave = E1000_MASTER_SLAVE;
 919         }
 920
 921         return 0;
 922 }
 923
 924 /**
 925  * e1000_probe - Device Initialization Routine
 926  * @pdev: PCI device information struct
 927  * @ent: entry in e1000_pci_tbl
 928  *
 929  * Returns 0 on success, negative on failure
 930  *
 931  * e1000_probe initializes an adapter identified by a pci_dev structure.
 932  * The OS initialization, configuring of the adapter private structure,
 933  * and a hardware reset occur.
 934  **/
 935 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 936 {
 937         struct net_device *netdev;
 938         struct e1000_adapter *adapter;
 939         struct e1000_hw *hw;
 940
 941         static int cards_found = 0;
 942         static int global_quad_port_a = 0; /* global ksp3 port a indication */
 943         int i, err, pci_using_dac;
 944         u16 eeprom_data = 0;
 945         u16 tmp = 0;
 946         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 947         int bars, need_ioport;
 948
 949         /* do not allocate ioport bars when not needed */
 950         need_ioport = e1000_is_need_ioport(pdev);
 951         if (need_ioport) {
 952                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 953                 err = pci_enable_device(pdev);
 954         } else {
 955                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 956                 err = pci_enable_device_mem(pdev);
 957         }
 958         if (err)
 959                 return err;
 960
 961         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 962         if (err)
 963                 goto err_pci_reg;
 964
 965         pci_set_master(pdev);
 966         err = pci_save_state(pdev);
 967         if (err)
 968                 goto err_alloc_etherdev;
 969
 970         err = -ENOMEM;
 971         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 972         if (!netdev)
 973                 goto err_alloc_etherdev;
 974
 975         SET_NETDEV_DEV(netdev, &pdev->dev);
 976
 977         pci_set_drvdata(pdev, netdev);
 978         adapter = netdev_priv(netdev);
 979         adapter->netdev = netdev;
 980         adapter->pdev = pdev;
 981         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 982         adapter->bars = bars;
 983         adapter->need_ioport = need_ioport;
 984
 985         hw = &adapter->hw;
 986         hw->back = adapter;
 987
 988         err = -EIO;
 989         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
 990         if (!hw->hw_addr)
 991                 goto err_ioremap;
 992
 993         if (adapter->need_ioport) {
 994                 for (i = BAR_1; i <= BAR_5; i++) {
 995                         if (pci_resource_len(pdev, i) == 0)
 996                                 continue;
 997                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 998                                 hw->io_base = pci_resource_start(pdev, i);
 999                                 break;
1000                         }
1001                 }
1002         }
1003
1004         /* make ready for any if (hw->...) below */
1005         err = e1000_init_hw_struct(adapter, hw);
1006         if (err)
1007                 goto err_sw_init;
1008
1009         /* there is a workaround being applied below that limits
1010          * 64-bit DMA addresses to 64-bit hardware.  There are some
1011          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
1012          */
1013         pci_using_dac = 0;
1014         if ((hw->bus_type == e1000_bus_type_pcix) &&
1015             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1016                 pci_using_dac = 1;
1017         } else {
1018                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1019                 if (err) {
1020                         pr_err("No usable DMA config, aborting\n");
1021                         goto err_dma;
1022                 }
1023         }
1024
1025         netdev->netdev_ops = &e1000_netdev_ops;
1026         e1000_set_ethtool_ops(netdev);
1027         netdev->watchdog_timeo = 5 * HZ;
1028         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1029
1030         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1031
1032         adapter->bd_number = cards_found;
1033
1034         /* setup the private structure */
1035
1036         err = e1000_sw_init(adapter);
1037         if (err)
1038                 goto err_sw_init;
1039
1040         err = -EIO;
1041         if (hw->mac_type == e1000_ce4100) {
1042                 hw->ce4100_gbe_mdio_base_virt =
1043                                         ioremap(pci_resource_start(pdev, BAR_1),
1044                                                 pci_resource_len(pdev, BAR_1));
1045
1046                 if (!hw->ce4100_gbe_mdio_base_virt)
1047                         goto err_mdio_ioremap;
1048         }
1049
1050         if (hw->mac_type >= e1000_82543) {
1051                 netdev->hw_features = NETIF_F_SG |
1052                                    NETIF_F_HW_CSUM |
1053                                    NETIF_F_HW_VLAN_CTAG_RX;
1054                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1055                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1056         }
1057
1058         if ((hw->mac_type >= e1000_82544) &&
1059            (hw->mac_type != e1000_82547))
1060                 netdev->hw_features |= NETIF_F_TSO;
1061
1062         netdev->priv_flags |= IFF_SUPP_NOFCS;
1063
1064         netdev->features |= netdev->hw_features;
1065         netdev->hw_features |= (NETIF_F_RXCSUM |
1066                                 NETIF_F_RXALL |
1067                                 NETIF_F_RXFCS);
1068
1069         if (pci_using_dac) {
1070                 netdev->features |= NETIF_F_HIGHDMA;
1071                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1072         }
1073
1074         netdev->vlan_features |= (NETIF_F_TSO |
1075                                   NETIF_F_HW_CSUM |
1076                                   NETIF_F_SG);
1077
1078         netdev->priv_flags |= IFF_UNICAST_FLT;
1079
1080         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1081
1082         /* initialize eeprom parameters */
1083         if (e1000_init_eeprom_params(hw)) {
1084                 e_err(probe, "EEPROM initialization failed\n");
1085                 goto err_eeprom;
1086         }
1087
1088         /* before reading the EEPROM, reset the controller to
1089          * put the device in a known good starting state
1090          */
1091
1092         e1000_reset_hw(hw);
1093
1094         /* make sure the EEPROM is good */
1095         if (e1000_validate_eeprom_checksum(hw) < 0) {
1096                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1097                 e1000_dump_eeprom(adapter);
1098                 /* set MAC address to all zeroes to invalidate and temporary
1099                  * disable this device for the user. This blocks regular
1100                  * traffic while still permitting ethtool ioctls from reaching
1101                  * the hardware as well as allowing the user to run the
1102                  * interface after manually setting a hw addr using
1103                  * `ip set address`
1104                  */
1105                 memset(hw->mac_addr, 0, netdev->addr_len);
1106         } else {
1107                 /* copy the MAC address out of the EEPROM */
1108                 if (e1000_read_mac_addr(hw))
1109                         e_err(probe, "EEPROM Read Error\n");
1110         }
1111         /* don't block initalization here due to bad MAC address */
1112         memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1113
1114         if (!is_valid_ether_addr(netdev->dev_addr))
1115                 e_err(probe, "Invalid MAC Address\n");
1116
1117
1118         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1119         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1120                           e1000_82547_tx_fifo_stall_task);
1121         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1122         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1123
1124         e1000_check_options(adapter);
1125
1126         /* Initial Wake on LAN setting
1127          * If APM wake is enabled in the EEPROM,
1128          * enable the ACPI Magic Packet filter
1129          */
1130
1131         switch (hw->mac_type) {
1132         case e1000_82542_rev2_0:
1133         case e1000_82542_rev2_1:
1134         case e1000_82543:
1135                 break;
1136         case e1000_82544:
1137                 e1000_read_eeprom(hw,
1138                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1139                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1140                 break;
1141         case e1000_82546:
1142         case e1000_82546_rev_3:
1143                 if (er32(STATUS) & E1000_STATUS_FUNC_1){
1144                         e1000_read_eeprom(hw,
1145                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1146                         break;
1147                 }
1148                 /* Fall Through */
1149         default:
1150                 e1000_read_eeprom(hw,
1151                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1152                 break;
1153         }
1154         if (eeprom_data & eeprom_apme_mask)
1155                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1156
1157         /* now that we have the eeprom settings, apply the special cases
1158          * where the eeprom may be wrong or the board simply won't support
1159          * wake on lan on a particular port
1160          */
1161         switch (pdev->device) {
1162         case E1000_DEV_ID_82546GB_PCIE:
1163                 adapter->eeprom_wol = 0;
1164                 break;
1165         case E1000_DEV_ID_82546EB_FIBER:
1166         case E1000_DEV_ID_82546GB_FIBER:
1167                 /* Wake events only supported on port A for dual fiber
1168                  * regardless of eeprom setting
1169                  */
1170                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1171                         adapter->eeprom_wol = 0;
1172                 break;
1173         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1174                 /* if quad port adapter, disable WoL on all but port A */
1175                 if (global_quad_port_a != 0)
1176                         adapter->eeprom_wol = 0;
1177                 else
1178                         adapter->quad_port_a = true;
1179                 /* Reset for multiple quad port adapters */
1180                 if (++global_quad_port_a == 4)
1181                         global_quad_port_a = 0;
1182                 break;
1183         }
1184
1185         /* initialize the wol settings based on the eeprom settings */
1186         adapter->wol = adapter->eeprom_wol;
1187         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1188
1189         /* Auto detect PHY address */
1190         if (hw->mac_type == e1000_ce4100) {
1191                 for (i = 0; i < 32; i++) {
1192                         hw->phy_addr = i;
1193                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1194                         if (tmp == 0 || tmp == 0xFF) {
1195                                 if (i == 31)
1196                                         goto err_eeprom;
1197                                 continue;
1198                         } else
1199                                 break;
1200                 }
1201         }
1202
1203         /* reset the hardware with the new settings */
1204         e1000_reset(adapter);
1205
1206         strcpy(netdev->name, "eth%d");
1207         err = register_netdev(netdev);
1208         if (err)
1209                 goto err_register;
1210
1211         e1000_vlan_filter_on_off(adapter, false);
1212
1213         /* print bus type/speed/width info */
1214         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1215                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1216                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1217                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1218                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1219                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1220                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1221                netdev->dev_addr);
1222
1223         /* carrier off reporting is important to ethtool even BEFORE open */
1224         netif_carrier_off(netdev);
1225
1226         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1227
1228         cards_found++;
1229         return 0;
1230
1231 err_register:
1232 err_eeprom:
1233         e1000_phy_hw_reset(hw);
1234
1235         if (hw->flash_address)
1236                 iounmap(hw->flash_address);
1237         kfree(adapter->tx_ring);
1238         kfree(adapter->rx_ring);
1239 err_dma:
1240 err_sw_init:
1241 err_mdio_ioremap:
1242         iounmap(hw->ce4100_gbe_mdio_base_virt);
1243         iounmap(hw->hw_addr);
1244 err_ioremap:
1245         free_netdev(netdev);
1246 err_alloc_etherdev:
1247         pci_release_selected_regions(pdev, bars);
1248 err_pci_reg:
1249         pci_disable_device(pdev);
1250         return err;
1251 }
1252
1253 /**
1254  * e1000_remove - Device Removal Routine
1255  * @pdev: PCI device information struct
1256  *
1257  * e1000_remove is called by the PCI subsystem to alert the driver
1258  * that it should release a PCI device.  The could be caused by a
1259  * Hot-Plug event, or because the driver is going to be removed from
1260  * memory.
1261  **/
1262 static void e1000_remove(struct pci_dev *pdev)
1263 {
1264         struct net_device *netdev = pci_get_drvdata(pdev);
1265         struct e1000_adapter *adapter = netdev_priv(netdev);
1266         struct e1000_hw *hw = &adapter->hw;
1267
1268         e1000_down_and_stop(adapter);
1269         e1000_release_manageability(adapter);
1270
1271         unregister_netdev(netdev);
1272
1273         e1000_phy_hw_reset(hw);
1274
1275         kfree(adapter->tx_ring);
1276         kfree(adapter->rx_ring);
1277
1278         if (hw->mac_type == e1000_ce4100)
1279                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1280         iounmap(hw->hw_addr);
1281         if (hw->flash_address)
1282                 iounmap(hw->flash_address);
1283         pci_release_selected_regions(pdev, adapter->bars);
1284
1285         free_netdev(netdev);
1286
1287         pci_disable_device(pdev);
1288 }
1289
1290 /**
1291  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1292  * @adapter: board private structure to initialize
1293  *
1294  * e1000_sw_init initializes the Adapter private data structure.
1295  * e1000_init_hw_struct MUST be called before this function
1296  **/
1297 static int e1000_sw_init(struct e1000_adapter *adapter)
1298 {
1299         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1300
1301         adapter->num_tx_queues = 1;
1302         adapter->num_rx_queues = 1;
1303
1304         if (e1000_alloc_queues(adapter)) {
1305                 e_err(probe, "Unable to allocate memory for queues\n");
1306                 return -ENOMEM;
1307         }
1308
1309         /* Explicitly disable IRQ since the NIC can be in any state. */
1310         e1000_irq_disable(adapter);
1311
1312         spin_lock_init(&adapter->stats_lock);
1313
1314         set_bit(__E1000_DOWN, &adapter->flags);
1315
1316         return 0;
1317 }
1318
1319 /**
1320  * e1000_alloc_queues - Allocate memory for all rings
1321  * @adapter: board private structure to initialize
1322  *
1323  * We allocate one ring per queue at run-time since we don't know the
1324  * number of queues at compile-time.
1325  **/
1326 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1327 {
1328         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1329                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1330         if (!adapter->tx_ring)
1331                 return -ENOMEM;
1332
1333         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1334                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1335         if (!adapter->rx_ring) {
1336                 kfree(adapter->tx_ring);
1337                 return -ENOMEM;
1338         }
1339
1340         return E1000_SUCCESS;
1341 }
1342
1343 /**
1344  * e1000_open - Called when a network interface is made active
1345  * @netdev: network interface device structure
1346  *
1347  * Returns 0 on success, negative value on failure
1348  *
1349  * The open entry point is called when a network interface is made
1350  * active by the system (IFF_UP).  At this point all resources needed
1351  * for transmit and receive operations are allocated, the interrupt
1352  * handler is registered with the OS, the watchdog task is started,
1353  * and the stack is notified that the interface is ready.
1354  **/
1355 static int e1000_open(struct net_device *netdev)
1356 {
1357         struct e1000_adapter *adapter = netdev_priv(netdev);
1358         struct e1000_hw *hw = &adapter->hw;
1359         int err;
1360
1361         /* disallow open during test */
1362         if (test_bit(__E1000_TESTING, &adapter->flags))
1363                 return -EBUSY;
1364
1365         netif_carrier_off(netdev);
1366
1367         /* allocate transmit descriptors */
1368         err = e1000_setup_all_tx_resources(adapter);
1369         if (err)
1370                 goto err_setup_tx;
1371
1372         /* allocate receive descriptors */
1373         err = e1000_setup_all_rx_resources(adapter);
1374         if (err)
1375                 goto err_setup_rx;
1376
1377         e1000_power_up_phy(adapter);
1378
1379         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1380         if ((hw->mng_cookie.status &
1381                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1382                 e1000_update_mng_vlan(adapter);
1383         }
1384
1385         /* before we allocate an interrupt, we must be ready to handle it.
1386          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1387          * as soon as we call pci_request_irq, so we have to setup our
1388          * clean_rx handler before we do so.
1389          */
1390         e1000_configure(adapter);
1391
1392         err = e1000_request_irq(adapter);
1393         if (err)
1394                 goto err_req_irq;
1395
1396         /* From here on the code is the same as e1000_up() */
1397         clear_bit(__E1000_DOWN, &adapter->flags);
1398
1399         napi_enable(&adapter->napi);
1400
1401         e1000_irq_enable(adapter);
1402
1403         netif_start_queue(netdev);
1404
1405         /* fire a link status change interrupt to start the watchdog */
1406         ew32(ICS, E1000_ICS_LSC);
1407
1408         return E1000_SUCCESS;
1409
1410 err_req_irq:
1411         e1000_power_down_phy(adapter);
1412         e1000_free_all_rx_resources(adapter);
1413 err_setup_rx:
1414         e1000_free_all_tx_resources(adapter);
1415 err_setup_tx:
1416         e1000_reset(adapter);
1417
1418         return err;
1419 }
1420
1421 /**
1422  * e1000_close - Disables a network interface
1423  * @netdev: network interface device structure
1424  *
1425  * Returns 0, this is not allowed to fail
1426  *
1427  * The close entry point is called when an interface is de-activated
1428  * by the OS.  The hardware is still under the drivers control, but
1429  * needs to be disabled.  A global MAC reset is issued to stop the
1430  * hardware, and all transmit and receive resources are freed.
1431  **/
1432 static int e1000_close(struct net_device *netdev)
1433 {
1434         struct e1000_adapter *adapter = netdev_priv(netdev);
1435         struct e1000_hw *hw = &adapter->hw;
1436         int count = E1000_CHECK_RESET_COUNT;
1437
1438         while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
1439                 usleep_range(10000, 20000);
1440
1441         WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
1442         e1000_down(adapter);
1443         e1000_power_down_phy(adapter);
1444         e1000_free_irq(adapter);
1445
1446         e1000_free_all_tx_resources(adapter);
1447         e1000_free_all_rx_resources(adapter);
1448
1449         /* kill manageability vlan ID if supported, but not if a vlan with
1450          * the same ID is registered on the host OS (let 8021q kill it)
1451          */
1452         if ((hw->mng_cookie.status &
1453              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1454             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1455                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1456                                        adapter->mng_vlan_id);
1457         }
1458
1459         return 0;
1460 }
1461
1462 /**
1463  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1464  * @adapter: address of board private structure
1465  * @start: address of beginning of memory
1466  * @len: length of memory
1467  **/
1468 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1469                                   unsigned long len)
1470 {
1471         struct e1000_hw *hw = &adapter->hw;
1472         unsigned long begin = (unsigned long)start;
1473         unsigned long end = begin + len;
1474
1475         /* First rev 82545 and 82546 need to not allow any memory
1476          * write location to cross 64k boundary due to errata 23
1477          */
1478         if (hw->mac_type == e1000_82545 ||
1479             hw->mac_type == e1000_ce4100 ||
1480             hw->mac_type == e1000_82546) {
1481                 return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
1482         }
1483
1484         return true;
1485 }
1486
1487 /**
1488  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1489  * @adapter: board private structure
1490  * @txdr:    tx descriptor ring (for a specific queue) to setup
1491  *
1492  * Return 0 on success, negative on failure
1493  **/
1494 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1495                                     struct e1000_tx_ring *txdr)
1496 {
1497         struct pci_dev *pdev = adapter->pdev;
1498         int size;
1499
1500         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1501         txdr->buffer_info = vzalloc(size);
1502         if (!txdr->buffer_info)
1503                 return -ENOMEM;
1504
1505         /* round up to nearest 4K */
1506
1507         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1508         txdr->size = ALIGN(txdr->size, 4096);
1509
1510         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1511                                         GFP_KERNEL);
1512         if (!txdr->desc) {
1513 setup_tx_desc_die:
1514                 vfree(txdr->buffer_info);
1515                 return -ENOMEM;
1516         }
1517
1518         /* Fix for errata 23, can't cross 64kB boundary */
1519         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1520                 void *olddesc = txdr->desc;
1521                 dma_addr_t olddma = txdr->dma;
1522                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1523                       txdr->size, txdr->desc);
1524                 /* Try again, without freeing the previous */
1525                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1526                                                 &txdr->dma, GFP_KERNEL);
1527                 /* Failed allocation, critical failure */
1528                 if (!txdr->desc) {
1529                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1530                                           olddma);
1531                         goto setup_tx_desc_die;
1532                 }
1533
1534                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1535                         /* give up */
1536                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1537                                           txdr->dma);
1538                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1539                                           olddma);
1540                         e_err(probe, "Unable to allocate aligned memory "
1541                               "for the transmit descriptor ring\n");
1542                         vfree(txdr->buffer_info);
1543                         return -ENOMEM;
1544                 } else {
1545                         /* Free old allocation, new allocation was successful */
1546                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1547                                           olddma);
1548                 }
1549         }
1550         memset(txdr->desc, 0, txdr->size);
1551
1552         txdr->next_to_use = 0;
1553         txdr->next_to_clean = 0;
1554
1555         return 0;
1556 }
1557
1558 /**
1559  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1560  *                                (Descriptors) for all queues
1561  * @adapter: board private structure
1562  *
1563  * Return 0 on success, negative on failure
1564  **/
1565 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1566 {
1567         int i, err = 0;
1568
1569         for (i = 0; i < adapter->num_tx_queues; i++) {
1570                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1571                 if (err) {
1572                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1573                         for (i-- ; i >= 0; i--)
1574                                 e1000_free_tx_resources(adapter,
1575                                                         &adapter->tx_ring[i]);
1576                         break;
1577                 }
1578         }
1579
1580         return err;
1581 }
1582
1583 /**
1584  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1585  * @adapter: board private structure
1586  *
1587  * Configure the Tx unit of the MAC after a reset.
1588  **/
1589 static void e1000_configure_tx(struct e1000_adapter *adapter)
1590 {
1591         u64 tdba;
1592         struct e1000_hw *hw = &adapter->hw;
1593         u32 tdlen, tctl, tipg;
1594         u32 ipgr1, ipgr2;
1595
1596         /* Setup the HW Tx Head and Tail descriptor pointers */
1597
1598         switch (adapter->num_tx_queues) {
1599         case 1:
1600         default:
1601                 tdba = adapter->tx_ring[0].dma;
1602                 tdlen = adapter->tx_ring[0].count *
1603                         sizeof(struct e1000_tx_desc);
1604                 ew32(TDLEN, tdlen);
1605                 ew32(TDBAH, (tdba >> 32));
1606                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1607                 ew32(TDT, 0);
1608                 ew32(TDH, 0);
1609                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1610                                            E1000_TDH : E1000_82542_TDH);
1611                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1612                                            E1000_TDT : E1000_82542_TDT);
1613                 break;
1614         }
1615
1616         /* Set the default values for the Tx Inter Packet Gap timer */
1617         if ((hw->media_type == e1000_media_type_fiber ||
1618              hw->media_type == e1000_media_type_internal_serdes))
1619                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1620         else
1621                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1622
1623         switch (hw->mac_type) {
1624         case e1000_82542_rev2_0:
1625         case e1000_82542_rev2_1:
1626                 tipg = DEFAULT_82542_TIPG_IPGT;
1627                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1628                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1629                 break;
1630         default:
1631                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1632                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1633                 break;
1634         }
1635         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1636         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1637         ew32(TIPG, tipg);
1638
1639         /* Set the Tx Interrupt Delay register */
1640
1641         ew32(TIDV, adapter->tx_int_delay);
1642         if (hw->mac_type >= e1000_82540)
1643                 ew32(TADV, adapter->tx_abs_int_delay);
1644
1645         /* Program the Transmit Control Register */
1646
1647         tctl = er32(TCTL);
1648         tctl &= ~E1000_TCTL_CT;
1649         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1650                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1651
1652         e1000_config_collision_dist(hw);
1653
1654         /* Setup Transmit Descriptor Settings for eop descriptor */
1655         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1656
1657         /* only set IDE if we are delaying interrupts using the timers */
1658         if (adapter->tx_int_delay)
1659                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1660
1661         if (hw->mac_type < e1000_82543)
1662                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1663         else
1664                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1665
1666         /* Cache if we're 82544 running in PCI-X because we'll
1667          * need this to apply a workaround later in the send path.
1668          */
1669         if (hw->mac_type == e1000_82544 &&
1670             hw->bus_type == e1000_bus_type_pcix)
1671                 adapter->pcix_82544 = true;
1672
1673         ew32(TCTL, tctl);
1674
1675 }
1676
1677 /**
1678  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1679  * @adapter: board private structure
1680  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1681  *
1682  * Returns 0 on success, negative on failure
1683  **/
1684 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1685                                     struct e1000_rx_ring *rxdr)
1686 {
1687         struct pci_dev *pdev = adapter->pdev;
1688         int size, desc_len;
1689
1690         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1691         rxdr->buffer_info = vzalloc(size);
1692         if (!rxdr->buffer_info)
1693                 return -ENOMEM;
1694
1695         desc_len = sizeof(struct e1000_rx_desc);
1696
1697         /* Round up to nearest 4K */
1698
1699         rxdr->size = rxdr->count * desc_len;
1700         rxdr->size = ALIGN(rxdr->size, 4096);
1701
1702         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1703                                         GFP_KERNEL);
1704         if (!rxdr->desc) {
1705 setup_rx_desc_die:
1706                 vfree(rxdr->buffer_info);
1707                 return -ENOMEM;
1708         }
1709
1710         /* Fix for errata 23, can't cross 64kB boundary */
1711         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1712                 void *olddesc = rxdr->desc;
1713                 dma_addr_t olddma = rxdr->dma;
1714                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1715                       rxdr->size, rxdr->desc);
1716                 /* Try again, without freeing the previous */
1717                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1718                                                 &rxdr->dma, GFP_KERNEL);
1719                 /* Failed allocation, critical failure */
1720                 if (!rxdr->desc) {
1721                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1722                                           olddma);
1723                         goto setup_rx_desc_die;
1724                 }
1725
1726                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1727                         /* give up */
1728                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1729                                           rxdr->dma);
1730                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1731                                           olddma);
1732                         e_err(probe, "Unable to allocate aligned memory for "
1733                               "the Rx descriptor ring\n");
1734                         goto setup_rx_desc_die;
1735                 } else {
1736                         /* Free old allocation, new allocation was successful */
1737                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1738                                           olddma);
1739                 }
1740         }
1741         memset(rxdr->desc, 0, rxdr->size);
1742
1743         rxdr->next_to_clean = 0;
1744         rxdr->next_to_use = 0;
1745         rxdr->rx_skb_top = NULL;
1746
1747         return 0;
1748 }
1749
1750 /**
1751  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1752  *                                (Descriptors) for all queues
1753  * @adapter: board private structure
1754  *
1755  * Return 0 on success, negative on failure
1756  **/
1757 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1758 {
1759         int i, err = 0;
1760
1761         for (i = 0; i < adapter->num_rx_queues; i++) {
1762                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1763                 if (err) {
1764                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1765                         for (i-- ; i >= 0; i--)
1766                                 e1000_free_rx_resources(adapter,
1767                                                         &adapter->rx_ring[i]);
1768                         break;
1769                 }
1770         }
1771
1772         return err;
1773 }
1774
1775 /**
1776  * e1000_setup_rctl - configure the receive control registers
1777  * @adapter: Board private structure
1778  **/
1779 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1780 {
1781         struct e1000_hw *hw = &adapter->hw;
1782         u32 rctl;
1783
1784         rctl = er32(RCTL);
1785
1786         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1787
1788         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1789                 E1000_RCTL_RDMTS_HALF |
1790                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1791
1792         if (hw->tbi_compatibility_on == 1)
1793                 rctl |= E1000_RCTL_SBP;
1794         else
1795                 rctl &= ~E1000_RCTL_SBP;
1796
1797         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1798                 rctl &= ~E1000_RCTL_LPE;
1799         else
1800                 rctl |= E1000_RCTL_LPE;
1801
1802         /* Setup buffer sizes */
1803         rctl &= ~E1000_RCTL_SZ_4096;
1804         rctl |= E1000_RCTL_BSEX;
1805         switch (adapter->rx_buffer_len) {
1806                 case E1000_RXBUFFER_2048:
1807                 default:
1808                         rctl |= E1000_RCTL_SZ_2048;
1809                         rctl &= ~E1000_RCTL_BSEX;
1810                         break;
1811                 case E1000_RXBUFFER_4096:
1812                         rctl |= E1000_RCTL_SZ_4096;
1813                         break;
1814                 case E1000_RXBUFFER_8192:
1815                         rctl |= E1000_RCTL_SZ_8192;
1816                         break;
1817                 case E1000_RXBUFFER_16384:
1818                         rctl |= E1000_RCTL_SZ_16384;
1819                         break;
1820         }
1821
1822         /* This is useful for sniffing bad packets. */
1823         if (adapter->netdev->features & NETIF_F_RXALL) {
1824                 /* UPE and MPE will be handled by normal PROMISC logic
1825                  * in e1000e_set_rx_mode
1826                  */
1827                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1828                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1829                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1830
1831                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1832                           E1000_RCTL_DPF | /* Allow filtered pause */
1833                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1834                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1835                  * and that breaks VLANs.
1836                  */
1837         }
1838
1839         ew32(RCTL, rctl);
1840 }
1841
1842 /**
1843  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1844  * @adapter: board private structure
1845  *
1846  * Configure the Rx unit of the MAC after a reset.
1847  **/
1848 static void e1000_configure_rx(struct e1000_adapter *adapter)
1849 {
1850         u64 rdba;
1851         struct e1000_hw *hw = &adapter->hw;
1852         u32 rdlen, rctl, rxcsum;
1853
1854         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1855                 rdlen = adapter->rx_ring[0].count *
1856                         sizeof(struct e1000_rx_desc);
1857                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1858                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1859         } else {
1860                 rdlen = adapter->rx_ring[0].count *
1861                         sizeof(struct e1000_rx_desc);
1862                 adapter->clean_rx = e1000_clean_rx_irq;
1863                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1864         }
1865
1866         /* disable receives while setting up the descriptors */
1867         rctl = er32(RCTL);
1868         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1869
1870         /* set the Receive Delay Timer Register */
1871         ew32(RDTR, adapter->rx_int_delay);
1872
1873         if (hw->mac_type >= e1000_82540) {
1874                 ew32(RADV, adapter->rx_abs_int_delay);
1875                 if (adapter->itr_setting != 0)
1876                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1877         }
1878
1879         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1880          * the Base and Length of the Rx Descriptor Ring
1881          */
1882         switch (adapter->num_rx_queues) {
1883         case 1:
1884         default:
1885                 rdba = adapter->rx_ring[0].dma;
1886                 ew32(RDLEN, rdlen);
1887                 ew32(RDBAH, (rdba >> 32));
1888                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1889                 ew32(RDT, 0);
1890                 ew32(RDH, 0);
1891                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1892                                            E1000_RDH : E1000_82542_RDH);
1893                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1894                                            E1000_RDT : E1000_82542_RDT);
1895                 break;
1896         }
1897
1898         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1899         if (hw->mac_type >= e1000_82543) {
1900                 rxcsum = er32(RXCSUM);
1901                 if (adapter->rx_csum)
1902                         rxcsum |= E1000_RXCSUM_TUOFL;
1903                 else
1904                         /* don't need to clear IPPCSE as it defaults to 0 */
1905                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1906                 ew32(RXCSUM, rxcsum);
1907         }
1908
1909         /* Enable Receives */
1910         ew32(RCTL, rctl | E1000_RCTL_EN);
1911 }
1912
1913 /**
1914  * e1000_free_tx_resources - Free Tx Resources per Queue
1915  * @adapter: board private structure
1916  * @tx_ring: Tx descriptor ring for a specific queue
1917  *
1918  * Free all transmit software resources
1919  **/
1920 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1921                                     struct e1000_tx_ring *tx_ring)
1922 {
1923         struct pci_dev *pdev = adapter->pdev;
1924
1925         e1000_clean_tx_ring(adapter, tx_ring);
1926
1927         vfree(tx_ring->buffer_info);
1928         tx_ring->buffer_info = NULL;
1929
1930         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1931                           tx_ring->dma);
1932
1933         tx_ring->desc = NULL;
1934 }
1935
1936 /**
1937  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1938  * @adapter: board private structure
1939  *
1940  * Free all transmit software resources
1941  **/
1942 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1943 {
1944         int i;
1945
1946         for (i = 0; i < adapter->num_tx_queues; i++)
1947                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1948 }
1949
1950 static void
1951 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1952                                  struct e1000_tx_buffer *buffer_info)
1953 {
1954         if (buffer_info->dma) {
1955                 if (buffer_info->mapped_as_page)
1956                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1957                                        buffer_info->length, DMA_TO_DEVICE);
1958                 else
1959                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1960                                          buffer_info->length,
1961                                          DMA_TO_DEVICE);
1962                 buffer_info->dma = 0;
1963         }
1964         if (buffer_info->skb) {
1965                 dev_kfree_skb_any(buffer_info->skb);
1966                 buffer_info->skb = NULL;
1967         }
1968         buffer_info->time_stamp = 0;
1969         /* buffer_info must be completely set up in the transmit path */
1970 }
1971
1972 /**
1973  * e1000_clean_tx_ring - Free Tx Buffers
1974  * @adapter: board private structure
1975  * @tx_ring: ring to be cleaned
1976  **/
1977 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1978                                 struct e1000_tx_ring *tx_ring)
1979 {
1980         struct e1000_hw *hw = &adapter->hw;
1981         struct e1000_tx_buffer *buffer_info;
1982         unsigned long size;
1983         unsigned int i;
1984
1985         /* Free all the Tx ring sk_buffs */
1986
1987         for (i = 0; i < tx_ring->count; i++) {
1988                 buffer_info = &tx_ring->buffer_info[i];
1989                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1990         }
1991
1992         netdev_reset_queue(adapter->netdev);
1993         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1994         memset(tx_ring->buffer_info, 0, size);
1995
1996         /* Zero out the descriptor ring */
1997
1998         memset(tx_ring->desc, 0, tx_ring->size);
1999
2000         tx_ring->next_to_use = 0;
2001         tx_ring->next_to_clean = 0;
2002         tx_ring->last_tx_tso = false;
2003
2004         writel(0, hw->hw_addr + tx_ring->tdh);
2005         writel(0, hw->hw_addr + tx_ring->tdt);
2006 }
2007
2008 /**
2009  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2010  * @adapter: board private structure
2011  **/
2012 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2013 {
2014         int i;
2015
2016         for (i = 0; i < adapter->num_tx_queues; i++)
2017                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2018 }
2019
2020 /**
2021  * e1000_free_rx_resources - Free Rx Resources
2022  * @adapter: board private structure
2023  * @rx_ring: ring to clean the resources from
2024  *
2025  * Free all receive software resources
2026  **/
2027 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2028                                     struct e1000_rx_ring *rx_ring)
2029 {
2030         struct pci_dev *pdev = adapter->pdev;
2031
2032         e1000_clean_rx_ring(adapter, rx_ring);
2033
2034         vfree(rx_ring->buffer_info);
2035         rx_ring->buffer_info = NULL;
2036
2037         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2038                           rx_ring->dma);
2039
2040         rx_ring->desc = NULL;
2041 }
2042
2043 /**
2044  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2045  * @adapter: board private structure
2046  *
2047  * Free all receive software resources
2048  **/
2049 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2050 {
2051         int i;
2052
2053         for (i = 0; i < adapter->num_rx_queues; i++)
2054                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2055 }
2056
2057 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2058 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2059 {
2060         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2061                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2062 }
2063
2064 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2065 {
2066         unsigned int len = e1000_frag_len(a);
2067         u8 *data = netdev_alloc_frag(len);
2068
2069         if (likely(data))
2070                 data += E1000_HEADROOM;
2071         return data;
2072 }
2073
2074 static void e1000_free_frag(const void *data)
2075 {
2076         put_page(virt_to_head_page(data));
2077 }
2078
2079 /**
2080  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2081  * @adapter: board private structure
2082  * @rx_ring: ring to free buffers from
2083  **/
2084 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2085                                 struct e1000_rx_ring *rx_ring)
2086 {
2087         struct e1000_hw *hw = &adapter->hw;
2088         struct e1000_rx_buffer *buffer_info;
2089         struct pci_dev *pdev = adapter->pdev;
2090         unsigned long size;
2091         unsigned int i;
2092
2093         /* Free all the Rx netfrags */
2094         for (i = 0; i < rx_ring->count; i++) {
2095                 buffer_info = &rx_ring->buffer_info[i];
2096                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2097                         if (buffer_info->dma)
2098                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2099                                                  adapter->rx_buffer_len,
2100                                                  DMA_FROM_DEVICE);
2101                         if (buffer_info->rxbuf.data) {
2102                                 e1000_free_frag(buffer_info->rxbuf.data);
2103                                 buffer_info->rxbuf.data = NULL;
2104                         }
2105                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2106                         if (buffer_info->dma)
2107                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2108                                                adapter->rx_buffer_len,
2109                                                DMA_FROM_DEVICE);
2110                         if (buffer_info->rxbuf.page) {
2111                                 put_page(buffer_info->rxbuf.page);
2112                                 buffer_info->rxbuf.page = NULL;
2113                         }
2114                 }
2115
2116                 buffer_info->dma = 0;
2117         }
2118
2119         /* there also may be some cached data from a chained receive */
2120         napi_free_frags(&adapter->napi);
2121         rx_ring->rx_skb_top = NULL;
2122
2123         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2124         memset(rx_ring->buffer_info, 0, size);
2125
2126         /* Zero out the descriptor ring */
2127         memset(rx_ring->desc, 0, rx_ring->size);
2128
2129         rx_ring->next_to_clean = 0;
2130         rx_ring->next_to_use = 0;
2131
2132         writel(0, hw->hw_addr + rx_ring->rdh);
2133         writel(0, hw->hw_addr + rx_ring->rdt);
2134 }
2135
2136 /**
2137  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2138  * @adapter: board private structure
2139  **/
2140 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2141 {
2142         int i;
2143
2144         for (i = 0; i < adapter->num_rx_queues; i++)
2145                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2146 }
2147
2148 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2149  * and memory write and invalidate disabled for certain operations
2150  */
2151 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2152 {
2153         struct e1000_hw *hw = &adapter->hw;
2154         struct net_device *netdev = adapter->netdev;
2155         u32 rctl;
2156
2157         e1000_pci_clear_mwi(hw);
2158
2159         rctl = er32(RCTL);
2160         rctl |= E1000_RCTL_RST;
2161         ew32(RCTL, rctl);
2162         E1000_WRITE_FLUSH();
2163         mdelay(5);
2164
2165         if (netif_running(netdev))
2166                 e1000_clean_all_rx_rings(adapter);
2167 }
2168
2169 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2170 {
2171         struct e1000_hw *hw = &adapter->hw;
2172         struct net_device *netdev = adapter->netdev;
2173         u32 rctl;
2174
2175         rctl = er32(RCTL);
2176         rctl &= ~E1000_RCTL_RST;
2177         ew32(RCTL, rctl);
2178         E1000_WRITE_FLUSH();
2179         mdelay(5);
2180
2181         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2182                 e1000_pci_set_mwi(hw);
2183
2184         if (netif_running(netdev)) {
2185                 /* No need to loop, because 82542 supports only 1 queue */
2186                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2187                 e1000_configure_rx(adapter);
2188                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2189         }
2190 }
2191
2192 /**
2193  * e1000_set_mac - Change the Ethernet Address of the NIC
2194  * @netdev: network interface device structure
2195  * @p: pointer to an address structure
2196  *
2197  * Returns 0 on success, negative on failure
2198  **/
2199 static int e1000_set_mac(struct net_device *netdev, void *p)
2200 {
2201         struct e1000_adapter *adapter = netdev_priv(netdev);
2202         struct e1000_hw *hw = &adapter->hw;
2203         struct sockaddr *addr = p;
2204
2205         if (!is_valid_ether_addr(addr->sa_data))
2206                 return -EADDRNOTAVAIL;
2207
2208         /* 82542 2.0 needs to be in reset to write receive address registers */
2209
2210         if (hw->mac_type == e1000_82542_rev2_0)
2211                 e1000_enter_82542_rst(adapter);
2212
2213         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2214         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2215
2216         e1000_rar_set(hw, hw->mac_addr, 0);
2217
2218         if (hw->mac_type == e1000_82542_rev2_0)
2219                 e1000_leave_82542_rst(adapter);
2220
2221         return 0;
2222 }
2223
2224 /**
2225  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2226  * @netdev: network interface device structure
2227  *
2228  * The set_rx_mode entry point is called whenever the unicast or multicast
2229  * address lists or the network interface flags are updated. This routine is
2230  * responsible for configuring the hardware for proper unicast, multicast,
2231  * promiscuous mode, and all-multi behavior.
2232  **/
2233 static void e1000_set_rx_mode(struct net_device *netdev)
2234 {
2235         struct e1000_adapter *adapter = netdev_priv(netdev);
2236         struct e1000_hw *hw = &adapter->hw;
2237         struct netdev_hw_addr *ha;
2238         bool use_uc = false;
2239         u32 rctl;
2240         u32 hash_value;
2241         int i, rar_entries = E1000_RAR_ENTRIES;
2242         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2243         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2244
2245         if (!mcarray)
2246                 return;
2247
2248         /* Check for Promiscuous and All Multicast modes */
2249
2250         rctl = er32(RCTL);
2251
2252         if (netdev->flags & IFF_PROMISC) {
2253                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2254                 rctl &= ~E1000_RCTL_VFE;
2255         } else {
2256                 if (netdev->flags & IFF_ALLMULTI)
2257                         rctl |= E1000_RCTL_MPE;
2258                 else
2259                         rctl &= ~E1000_RCTL_MPE;
2260                 /* Enable VLAN filter if there is a VLAN */
2261                 if (e1000_vlan_used(adapter))
2262                         rctl |= E1000_RCTL_VFE;
2263         }
2264
2265         if (netdev_uc_count(netdev) > rar_entries - 1) {
2266                 rctl |= E1000_RCTL_UPE;
2267         } else if (!(netdev->flags & IFF_PROMISC)) {
2268                 rctl &= ~E1000_RCTL_UPE;
2269                 use_uc = true;
2270         }
2271
2272         ew32(RCTL, rctl);
2273
2274         /* 82542 2.0 needs to be in reset to write receive address registers */
2275
2276         if (hw->mac_type == e1000_82542_rev2_0)
2277                 e1000_enter_82542_rst(adapter);
2278
2279         /* load the first 14 addresses into the exact filters 1-14. Unicast
2280          * addresses take precedence to avoid disabling unicast filtering
2281          * when possible.
2282          *
2283          * RAR 0 is used for the station MAC address
2284          * if there are not 14 addresses, go ahead and clear the filters
2285          */
2286         i = 1;
2287         if (use_uc)
2288                 netdev_for_each_uc_addr(ha, netdev) {
2289                         if (i == rar_entries)
2290                                 break;
2291                         e1000_rar_set(hw, ha->addr, i++);
2292                 }
2293
2294         netdev_for_each_mc_addr(ha, netdev) {
2295                 if (i == rar_entries) {
2296                         /* load any remaining addresses into the hash table */
2297                         u32 hash_reg, hash_bit, mta;
2298                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2299                         hash_reg = (hash_value >> 5) & 0x7F;
2300                         hash_bit = hash_value & 0x1F;
2301                         mta = (1 << hash_bit);
2302                         mcarray[hash_reg] |= mta;
2303                 } else {
2304                         e1000_rar_set(hw, ha->addr, i++);
2305                 }
2306         }
2307
2308         for (; i < rar_entries; i++) {
2309                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2310                 E1000_WRITE_FLUSH();
2311                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2312                 E1000_WRITE_FLUSH();
2313         }
2314
2315         /* write the hash table completely, write from bottom to avoid
2316          * both stupid write combining chipsets, and flushing each write
2317          */
2318         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2319                 /* If we are on an 82544 has an errata where writing odd
2320                  * offsets overwrites the previous even offset, but writing
2321                  * backwards over the range solves the issue by always
2322                  * writing the odd offset first
2323                  */
2324                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2325         }
2326         E1000_WRITE_FLUSH();
2327
2328         if (hw->mac_type == e1000_82542_rev2_0)
2329                 e1000_leave_82542_rst(adapter);
2330
2331         kfree(mcarray);
2332 }
2333
2334 /**
2335  * e1000_update_phy_info_task - get phy info
2336  * @work: work struct contained inside adapter struct
2337  *
2338  * Need to wait a few seconds after link up to get diagnostic information from
2339  * the phy
2340  */
2341 static void e1000_update_phy_info_task(struct work_struct *work)
2342 {
2343         struct e1000_adapter *adapter = container_of(work,
2344                                                      struct e1000_adapter,
2345                                                      phy_info_task.work);
2346
2347         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2348 }
2349
2350 /**
2351  * e1000_82547_tx_fifo_stall_task - task to complete work
2352  * @work: work struct contained inside adapter struct
2353  **/
2354 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2355 {
2356         struct e1000_adapter *adapter = container_of(work,
2357                                                      struct e1000_adapter,
2358                                                      fifo_stall_task.work);
2359         struct e1000_hw *hw = &adapter->hw;
2360         struct net_device *netdev = adapter->netdev;
2361         u32 tctl;
2362
2363         if (atomic_read(&adapter->tx_fifo_stall)) {
2364                 if ((er32(TDT) == er32(TDH)) &&
2365                    (er32(TDFT) == er32(TDFH)) &&
2366                    (er32(TDFTS) == er32(TDFHS))) {
2367                         tctl = er32(TCTL);
2368                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2369                         ew32(TDFT, adapter->tx_head_addr);
2370                         ew32(TDFH, adapter->tx_head_addr);
2371                         ew32(TDFTS, adapter->tx_head_addr);
2372                         ew32(TDFHS, adapter->tx_head_addr);
2373                         ew32(TCTL, tctl);
2374                         E1000_WRITE_FLUSH();
2375
2376                         adapter->tx_fifo_head = 0;
2377                         atomic_set(&adapter->tx_fifo_stall, 0);
2378                         netif_wake_queue(netdev);
2379                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2380                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2381                 }
2382         }
2383 }
2384
2385 bool e1000_has_link(struct e1000_adapter *adapter)
2386 {
2387         struct e1000_hw *hw = &adapter->hw;
2388         bool link_active = false;
2389
2390         /* get_link_status is set on LSC (link status) interrupt or rx
2391          * sequence error interrupt (except on intel ce4100).
2392          * get_link_status will stay false until the
2393          * e1000_check_for_link establishes link for copper adapters
2394          * ONLY
2395          */
2396         switch (hw->media_type) {
2397         case e1000_media_type_copper:
2398                 if (hw->mac_type == e1000_ce4100)
2399                         hw->get_link_status = 1;
2400                 if (hw->get_link_status) {
2401                         e1000_check_for_link(hw);
2402                         link_active = !hw->get_link_status;
2403                 } else {
2404                         link_active = true;
2405                 }
2406                 break;
2407         case e1000_media_type_fiber:
2408                 e1000_check_for_link(hw);
2409                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2410                 break;
2411         case e1000_media_type_internal_serdes:
2412                 e1000_check_for_link(hw);
2413                 link_active = hw->serdes_has_link;
2414                 break;
2415         default:
2416                 break;
2417         }
2418
2419         return link_active;
2420 }
2421
2422 /**
2423  * e1000_watchdog - work function
2424  * @work: work struct contained inside adapter struct
2425  **/
2426 static void e1000_watchdog(struct work_struct *work)
2427 {
2428         struct e1000_adapter *adapter = container_of(work,
2429                                                      struct e1000_adapter,
2430                                                      watchdog_task.work);
2431         struct e1000_hw *hw = &adapter->hw;
2432         struct net_device *netdev = adapter->netdev;
2433         struct e1000_tx_ring *txdr = adapter->tx_ring;
2434         u32 link, tctl;
2435
2436         link = e1000_has_link(adapter);
2437         if ((netif_carrier_ok(netdev)) && link)
2438                 goto link_up;
2439
2440         if (link) {
2441                 if (!netif_carrier_ok(netdev)) {
2442                         u32 ctrl;
2443                         bool txb2b = true;
2444                         /* update snapshot of PHY registers on LSC */
2445                         e1000_get_speed_and_duplex(hw,
2446                                                    &adapter->link_speed,
2447                                                    &adapter->link_duplex);
2448
2449                         ctrl = er32(CTRL);
2450                         pr_info("%s NIC Link is Up %d Mbps %s, "
2451                                 "Flow Control: %s\n",
2452                                 netdev->name,
2453                                 adapter->link_speed,
2454                                 adapter->link_duplex == FULL_DUPLEX ?
2455                                 "Full Duplex" : "Half Duplex",
2456                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2457                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2458                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2459                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2460
2461                         /* adjust timeout factor according to speed/duplex */
2462                         adapter->tx_timeout_factor = 1;
2463                         switch (adapter->link_speed) {
2464                         case SPEED_10:
2465                                 txb2b = false;
2466                                 adapter->tx_timeout_factor = 16;
2467                                 break;
2468                         case SPEED_100:
2469                                 txb2b = false;
2470                                 /* maybe add some timeout factor ? */
2471                                 break;
2472                         }
2473
2474                         /* enable transmits in the hardware */
2475                         tctl = er32(TCTL);
2476                         tctl |= E1000_TCTL_EN;
2477                         ew32(TCTL, tctl);
2478
2479                         netif_carrier_on(netdev);
2480                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2481                                 schedule_delayed_work(&adapter->phy_info_task,
2482                                                       2 * HZ);
2483                         adapter->smartspeed = 0;
2484                 }
2485         } else {
2486                 if (netif_carrier_ok(netdev)) {
2487                         adapter->link_speed = 0;
2488                         adapter->link_duplex = 0;
2489                         pr_info("%s NIC Link is Down\n",
2490                                 netdev->name);
2491                         netif_carrier_off(netdev);
2492
2493                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2494                                 schedule_delayed_work(&adapter->phy_info_task,
2495                                                       2 * HZ);
2496                 }
2497
2498                 e1000_smartspeed(adapter);
2499         }
2500
2501 link_up:
2502         e1000_update_stats(adapter);
2503
2504         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2505         adapter->tpt_old = adapter->stats.tpt;
2506         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2507         adapter->colc_old = adapter->stats.colc;
2508
2509         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2510         adapter->gorcl_old = adapter->stats.gorcl;
2511         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2512         adapter->gotcl_old = adapter->stats.gotcl;
2513
2514         e1000_update_adaptive(hw);
2515
2516         if (!netif_carrier_ok(netdev)) {
2517                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2518                         /* We've lost link, so the controller stops DMA,
2519                          * but we've got queued Tx work that's never going
2520                          * to get done, so reset controller to flush Tx.
2521                          * (Do the reset outside of interrupt context).
2522                          */
2523                         adapter->tx_timeout_count++;
2524                         schedule_work(&adapter->reset_task);
2525                         /* exit immediately since reset is imminent */
2526                         return;
2527                 }
2528         }
2529
2530         /* Simple mode for Interrupt Throttle Rate (ITR) */
2531         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2532                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2533                  * Total asymmetrical Tx or Rx gets ITR=8000;
2534                  * everyone else is between 2000-8000.
2535                  */
2536                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2537                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2538                             adapter->gotcl - adapter->gorcl :
2539                             adapter->gorcl - adapter->gotcl) / 10000;
2540                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2541
2542                 ew32(ITR, 1000000000 / (itr * 256));
2543         }
2544
2545         /* Cause software interrupt to ensure rx ring is cleaned */
2546         ew32(ICS, E1000_ICS_RXDMT0);
2547
2548         /* Force detection of hung controller every watchdog period */
2549         adapter->detect_tx_hung = true;
2550
2551         /* Reschedule the task */
2552         if (!test_bit(__E1000_DOWN, &adapter->flags))
2553                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2554 }
2555
2556 enum latency_range {
2557         lowest_latency = 0,
2558         low_latency = 1,
2559         bulk_latency = 2,
2560         latency_invalid = 255
2561 };
2562
2563 /**
2564  * e1000_update_itr - update the dynamic ITR value based on statistics
2565  * @adapter: pointer to adapter
2566  * @itr_setting: current adapter->itr
2567  * @packets: the number of packets during this measurement interval
2568  * @bytes: the number of bytes during this measurement interval
2569  *
2570  *      Stores a new ITR value based on packets and byte
2571  *      counts during the last interrupt.  The advantage of per interrupt
2572  *      computation is faster updates and more accurate ITR for the current
2573  *      traffic pattern.  Constants in this function were computed
2574  *      based on theoretical maximum wire speed and thresholds were set based
2575  *      on testing data as well as attempting to minimize response time
2576  *      while increasing bulk throughput.
2577  *      this functionality is controlled by the InterruptThrottleRate module
2578  *      parameter (see e1000_param.c)
2579  **/
2580 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2581                                      u16 itr_setting, int packets, int bytes)
2582 {
2583         unsigned int retval = itr_setting;
2584         struct e1000_hw *hw = &adapter->hw;
2585
2586         if (unlikely(hw->mac_type < e1000_82540))
2587                 goto update_itr_done;
2588
2589         if (packets == 0)
2590                 goto update_itr_done;
2591
2592         switch (itr_setting) {
2593         case lowest_latency:
2594                 /* jumbo frames get bulk treatment*/
2595                 if (bytes/packets > 8000)
2596                         retval = bulk_latency;
2597                 else if ((packets < 5) && (bytes > 512))
2598                         retval = low_latency;
2599                 break;
2600         case low_latency:  /* 50 usec aka 20000 ints/s */
2601                 if (bytes > 10000) {
2602                         /* jumbo frames need bulk latency setting */
2603                         if (bytes/packets > 8000)
2604                                 retval = bulk_latency;
2605                         else if ((packets < 10) || ((bytes/packets) > 1200))
2606                                 retval = bulk_latency;
2607                         else if ((packets > 35))
2608                                 retval = lowest_latency;
2609                 } else if (bytes/packets > 2000)
2610                         retval = bulk_latency;
2611                 else if (packets <= 2 && bytes < 512)
2612                         retval = lowest_latency;
2613                 break;
2614         case bulk_latency: /* 250 usec aka 4000 ints/s */
2615                 if (bytes > 25000) {
2616                         if (packets > 35)
2617                                 retval = low_latency;
2618                 } else if (bytes < 6000) {
2619                         retval = low_latency;
2620                 }
2621                 break;
2622         }
2623
2624 update_itr_done:
2625         return retval;
2626 }
2627
2628 static void e1000_set_itr(struct e1000_adapter *adapter)
2629 {
2630         struct e1000_hw *hw = &adapter->hw;
2631         u16 current_itr;
2632         u32 new_itr = adapter->itr;
2633
2634         if (unlikely(hw->mac_type < e1000_82540))
2635                 return;
2636
2637         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2638         if (unlikely(adapter->link_speed != SPEED_1000)) {
2639                 current_itr = 0;
2640                 new_itr = 4000;
2641                 goto set_itr_now;
2642         }
2643
2644         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2645                                            adapter->total_tx_packets,
2646                                            adapter->total_tx_bytes);
2647         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2648         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2649                 adapter->tx_itr = low_latency;
2650
2651         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2652                                            adapter->total_rx_packets,
2653                                            adapter->total_rx_bytes);
2654         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2655         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2656                 adapter->rx_itr = low_latency;
2657
2658         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2659
2660         switch (current_itr) {
2661         /* counts and packets in update_itr are dependent on these numbers */
2662         case lowest_latency:
2663                 new_itr = 70000;
2664                 break;
2665         case low_latency:
2666                 new_itr = 20000; /* aka hwitr = ~200 */
2667                 break;
2668         case bulk_latency:
2669                 new_itr = 4000;
2670                 break;
2671         default:
2672                 break;
2673         }
2674
2675 set_itr_now:
2676         if (new_itr != adapter->itr) {
2677                 /* this attempts to bias the interrupt rate towards Bulk
2678                  * by adding intermediate steps when interrupt rate is
2679                  * increasing
2680                  */
2681                 new_itr = new_itr > adapter->itr ?
2682                           min(adapter->itr + (new_itr >> 2), new_itr) :
2683                           new_itr;
2684                 adapter->itr = new_itr;
2685                 ew32(ITR, 1000000000 / (new_itr * 256));
2686         }
2687 }
2688
2689 #define E1000_TX_FLAGS_CSUM             0x00000001
2690 #define E1000_TX_FLAGS_VLAN             0x00000002
2691 #define E1000_TX_FLAGS_TSO              0x00000004
2692 #define E1000_TX_FLAGS_IPV4             0x00000008
2693 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2694 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2695 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2696
2697 static int e1000_tso(struct e1000_adapter *adapter,
2698                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2699                      __be16 protocol)
2700 {
2701         struct e1000_context_desc *context_desc;
2702         struct e1000_tx_buffer *buffer_info;
2703         unsigned int i;
2704         u32 cmd_length = 0;
2705         u16 ipcse = 0, tucse, mss;
2706         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2707
2708         if (skb_is_gso(skb)) {
2709                 int err;
2710
2711                 err = skb_cow_head(skb, 0);
2712                 if (err < 0)
2713                         return err;
2714
2715                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2716                 mss = skb_shinfo(skb)->gso_size;
2717                 if (protocol == htons(ETH_P_IP)) {
2718                         struct iphdr *iph = ip_hdr(skb);
2719                         iph->tot_len = 0;
2720                         iph->check = 0;
2721                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2722                                                                  iph->daddr, 0,
2723                                                                  IPPROTO_TCP,
2724                                                                  0);
2725                         cmd_length = E1000_TXD_CMD_IP;
2726                         ipcse = skb_transport_offset(skb) - 1;
2727                 } else if (skb_is_gso_v6(skb)) {
2728                         ipv6_hdr(skb)->payload_len = 0;
2729                         tcp_hdr(skb)->check =
2730                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2731                                                  &ipv6_hdr(skb)->daddr,
2732                                                  0, IPPROTO_TCP, 0);
2733                         ipcse = 0;
2734                 }
2735                 ipcss = skb_network_offset(skb);
2736                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2737                 tucss = skb_transport_offset(skb);
2738                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2739                 tucse = 0;
2740
2741                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2742                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2743
2744                 i = tx_ring->next_to_use;
2745                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2746                 buffer_info = &tx_ring->buffer_info[i];
2747
2748                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2749                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2750                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2751                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2752                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2753                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2754                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2755                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2756                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2757
2758                 buffer_info->time_stamp = jiffies;
2759                 buffer_info->next_to_watch = i;
2760
2761                 if (++i == tx_ring->count) i = 0;
2762                 tx_ring->next_to_use = i;
2763
2764                 return true;
2765         }
2766         return false;
2767 }
2768
2769 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2770                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2771                           __be16 protocol)
2772 {
2773         struct e1000_context_desc *context_desc;
2774         struct e1000_tx_buffer *buffer_info;
2775         unsigned int i;
2776         u8 css;
2777         u32 cmd_len = E1000_TXD_CMD_DEXT;
2778
2779         if (skb->ip_summed != CHECKSUM_PARTIAL)
2780                 return false;
2781
2782         switch (protocol) {
2783         case cpu_to_be16(ETH_P_IP):
2784                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2785                         cmd_len |= E1000_TXD_CMD_TCP;
2786                 break;
2787         case cpu_to_be16(ETH_P_IPV6):
2788                 /* XXX not handling all IPV6 headers */
2789                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2790                         cmd_len |= E1000_TXD_CMD_TCP;
2791                 break;
2792         default:
2793                 if (unlikely(net_ratelimit()))
2794                         e_warn(drv, "checksum_partial proto=%x!\n",
2795                                skb->protocol);
2796                 break;
2797         }
2798
2799         css = skb_checksum_start_offset(skb);
2800
2801         i = tx_ring->next_to_use;
2802         buffer_info = &tx_ring->buffer_info[i];
2803         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2804
2805         context_desc->lower_setup.ip_config = 0;
2806         context_desc->upper_setup.tcp_fields.tucss = css;
2807         context_desc->upper_setup.tcp_fields.tucso =
2808                 css + skb->csum_offset;
2809         context_desc->upper_setup.tcp_fields.tucse = 0;
2810         context_desc->tcp_seg_setup.data = 0;
2811         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2812
2813         buffer_info->time_stamp = jiffies;
2814         buffer_info->next_to_watch = i;
2815
2816         if (unlikely(++i == tx_ring->count)) i = 0;
2817         tx_ring->next_to_use = i;
2818
2819         return true;
2820 }
2821
2822 #define E1000_MAX_TXD_PWR       12
2823 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2824
2825 static int e1000_tx_map(struct e1000_adapter *adapter,
2826                         struct e1000_tx_ring *tx_ring,
2827                         struct sk_buff *skb, unsigned int first,
2828                         unsigned int max_per_txd, unsigned int nr_frags,
2829                         unsigned int mss)
2830 {
2831         struct e1000_hw *hw = &adapter->hw;
2832         struct pci_dev *pdev = adapter->pdev;
2833         struct e1000_tx_buffer *buffer_info;
2834         unsigned int len = skb_headlen(skb);
2835         unsigned int offset = 0, size, count = 0, i;
2836         unsigned int f, bytecount, segs;
2837
2838         i = tx_ring->next_to_use;
2839
2840         while (len) {
2841                 buffer_info = &tx_ring->buffer_info[i];
2842                 size = min(len, max_per_txd);
2843                 /* Workaround for Controller erratum --
2844                  * descriptor for non-tso packet in a linear SKB that follows a
2845                  * tso gets written back prematurely before the data is fully
2846                  * DMA'd to the controller
2847                  */
2848                 if (!skb->data_len && tx_ring->last_tx_tso &&
2849                     !skb_is_gso(skb)) {
2850                         tx_ring->last_tx_tso = false;
2851                         size -= 4;
2852                 }
2853
2854                 /* Workaround for premature desc write-backs
2855                  * in TSO mode.  Append 4-byte sentinel desc
2856                  */
2857                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2858                         size -= 4;
2859                 /* work-around for errata 10 and it applies
2860                  * to all controllers in PCI-X mode
2861                  * The fix is to make sure that the first descriptor of a
2862                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2863                  */
2864                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2865                                 (size > 2015) && count == 0))
2866                         size = 2015;
2867
2868                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2869                  * terminating buffers within evenly-aligned dwords.
2870                  */
2871                 if (unlikely(adapter->pcix_82544 &&
2872                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2873                    size > 4))
2874                         size -= 4;
2875
2876                 buffer_info->length = size;
2877                 /* set time_stamp *before* dma to help avoid a possible race */
2878                 buffer_info->time_stamp = jiffies;
2879                 buffer_info->mapped_as_page = false;
2880                 buffer_info->dma = dma_map_single(&pdev->dev,
2881                                                   skb->data + offset,
2882                                                   size, DMA_TO_DEVICE);
2883                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2884                         goto dma_error;
2885                 buffer_info->next_to_watch = i;
2886
2887                 len -= size;
2888                 offset += size;
2889                 count++;
2890                 if (len) {
2891                         i++;
2892                         if (unlikely(i == tx_ring->count))
2893                                 i = 0;
2894                 }
2895         }
2896
2897         for (f = 0; f < nr_frags; f++) {
2898                 const struct skb_frag_struct *frag;
2899
2900                 frag = &skb_shinfo(skb)->frags[f];
2901                 len = skb_frag_size(frag);
2902                 offset = 0;
2903
2904                 while (len) {
2905                         unsigned long bufend;
2906                         i++;
2907                         if (unlikely(i == tx_ring->count))
2908                                 i = 0;
2909
2910                         buffer_info = &tx_ring->buffer_info[i];
2911                         size = min(len, max_per_txd);
2912                         /* Workaround for premature desc write-backs
2913                          * in TSO mode.  Append 4-byte sentinel desc
2914                          */
2915                         if (unlikely(mss && f == (nr_frags-1) &&
2916                             size == len && size > 8))
2917                                 size -= 4;
2918                         /* Workaround for potential 82544 hang in PCI-X.
2919                          * Avoid terminating buffers within evenly-aligned
2920                          * dwords.
2921                          */
2922                         bufend = (unsigned long)
2923                                 page_to_phys(skb_frag_page(frag));
2924                         bufend += offset + size - 1;
2925                         if (unlikely(adapter->pcix_82544 &&
2926                                      !(bufend & 4) &&
2927                                      size > 4))
2928                                 size -= 4;
2929
2930                         buffer_info->length = size;
2931                         buffer_info->time_stamp = jiffies;
2932                         buffer_info->mapped_as_page = true;
2933                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2934                                                 offset, size, DMA_TO_DEVICE);
2935                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2936                                 goto dma_error;
2937                         buffer_info->next_to_watch = i;
2938
2939                         len -= size;
2940                         offset += size;
2941                         count++;
2942                 }
2943         }
2944
2945         segs = skb_shinfo(skb)->gso_segs ?: 1;
2946         /* multiply data chunks by size of headers */
2947         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2948
2949         tx_ring->buffer_info[i].skb = skb;
2950         tx_ring->buffer_info[i].segs = segs;
2951         tx_ring->buffer_info[i].bytecount = bytecount;
2952         tx_ring->buffer_info[first].next_to_watch = i;
2953
2954         return count;
2955
2956 dma_error:
2957         dev_err(&pdev->dev, "TX DMA map failed\n");
2958         buffer_info->dma = 0;
2959         if (count)
2960                 count--;
2961
2962         while (count--) {
2963                 if (i==0)
2964                         i += tx_ring->count;
2965                 i--;
2966                 buffer_info = &tx_ring->buffer_info[i];
2967                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2968         }
2969
2970         return 0;
2971 }
2972
2973 static void e1000_tx_queue(struct e1000_adapter *adapter,
2974                            struct e1000_tx_ring *tx_ring, int tx_flags,
2975                            int count)
2976 {
2977         struct e1000_hw *hw = &adapter->hw;
2978         struct e1000_tx_desc *tx_desc = NULL;
2979         struct e1000_tx_buffer *buffer_info;
2980         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2981         unsigned int i;
2982
2983         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2984                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2985                              E1000_TXD_CMD_TSE;
2986                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2987
2988                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2989                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2990         }
2991
2992         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2993                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2994                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2995         }
2996
2997         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2998                 txd_lower |= E1000_TXD_CMD_VLE;
2999                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
3000         }
3001
3002         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3003                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
3004
3005         i = tx_ring->next_to_use;
3006
3007         while (count--) {
3008                 buffer_info = &tx_ring->buffer_info[i];
3009                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3010                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3011                 tx_desc->lower.data =
3012                         cpu_to_le32(txd_lower | buffer_info->length);
3013                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3014                 if (unlikely(++i == tx_ring->count)) i = 0;
3015         }
3016
3017         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3018
3019         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3020         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3021                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3022
3023         /* Force memory writes to complete before letting h/w
3024          * know there are new descriptors to fetch.  (Only
3025          * applicable for weak-ordered memory model archs,
3026          * such as IA-64).
3027          */
3028         wmb();
3029
3030         tx_ring->next_to_use = i;
3031         writel(i, hw->hw_addr + tx_ring->tdt);
3032         /* we need this if more than one processor can write to our tail
3033          * at a time, it synchronizes IO on IA64/Altix systems
3034          */
3035         mmiowb();
3036 }
3037
3038 /* 82547 workaround to avoid controller hang in half-duplex environment.
3039  * The workaround is to avoid queuing a large packet that would span
3040  * the internal Tx FIFO ring boundary by notifying the stack to resend
3041  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3042  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3043  * to the beginning of the Tx FIFO.
3044  */
3045
3046 #define E1000_FIFO_HDR                  0x10
3047 #define E1000_82547_PAD_LEN             0x3E0
3048
3049 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3050                                        struct sk_buff *skb)
3051 {
3052         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3053         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3054
3055         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3056
3057         if (adapter->link_duplex != HALF_DUPLEX)
3058                 goto no_fifo_stall_required;
3059
3060         if (atomic_read(&adapter->tx_fifo_stall))
3061                 return 1;
3062
3063         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3064                 atomic_set(&adapter->tx_fifo_stall, 1);
3065                 return 1;
3066         }
3067
3068 no_fifo_stall_required:
3069         adapter->tx_fifo_head += skb_fifo_len;
3070         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3071                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3072         return 0;
3073 }
3074
3075 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3076 {
3077         struct e1000_adapter *adapter = netdev_priv(netdev);
3078         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3079
3080         netif_stop_queue(netdev);
3081         /* Herbert's original patch had:
3082          *  smp_mb__after_netif_stop_queue();
3083          * but since that doesn't exist yet, just open code it.
3084          */
3085         smp_mb();
3086
3087         /* We need to check again in a case another CPU has just
3088          * made room available.
3089          */
3090         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3091                 return -EBUSY;
3092
3093         /* A reprieve! */
3094         netif_start_queue(netdev);
3095         ++adapter->restart_queue;
3096         return 0;
3097 }
3098
3099 static int e1000_maybe_stop_tx(struct net_device *netdev,
3100                                struct e1000_tx_ring *tx_ring, int size)
3101 {
3102         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3103                 return 0;
3104         return __e1000_maybe_stop_tx(netdev, size);
3105 }
3106
3107 #define TXD_USE_COUNT(S, X) (((S) >> (X)) + 1 )
3108 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3109                                     struct net_device *netdev)
3110 {
3111         struct e1000_adapter *adapter = netdev_priv(netdev);
3112         struct e1000_hw *hw = &adapter->hw;
3113         struct e1000_tx_ring *tx_ring;
3114         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3115         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3116         unsigned int tx_flags = 0;
3117         unsigned int len = skb_headlen(skb);
3118         unsigned int nr_frags;
3119         unsigned int mss;
3120         int count = 0;
3121         int tso;
3122         unsigned int f;
3123         __be16 protocol = vlan_get_protocol(skb);
3124
3125         /* This goes back to the question of how to logically map a Tx queue
3126          * to a flow.  Right now, performance is impacted slightly negatively
3127          * if using multiple Tx queues.  If the stack breaks away from a
3128          * single qdisc implementation, we can look at this again.
3129          */
3130         tx_ring = adapter->tx_ring;
3131
3132         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3133          * packets may get corrupted during padding by HW.
3134          * To WA this issue, pad all small packets manually.
3135          */
3136         if (skb->len < ETH_ZLEN) {
3137                 if (skb_pad(skb, ETH_ZLEN - skb->len))
3138                         return NETDEV_TX_OK;
3139                 skb->len = ETH_ZLEN;
3140                 skb_set_tail_pointer(skb, ETH_ZLEN);
3141         }
3142
3143         mss = skb_shinfo(skb)->gso_size;
3144         /* The controller does a simple calculation to
3145          * make sure there is enough room in the FIFO before
3146          * initiating the DMA for each buffer.  The calc is:
3147          * 4 = ceil(buffer len/mss).  To make sure we don't
3148          * overrun the FIFO, adjust the max buffer len if mss
3149          * drops.
3150          */
3151         if (mss) {
3152                 u8 hdr_len;
3153                 max_per_txd = min(mss << 2, max_per_txd);
3154                 max_txd_pwr = fls(max_per_txd) - 1;
3155
3156                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3157                 if (skb->data_len && hdr_len == len) {
3158                         switch (hw->mac_type) {
3159                                 unsigned int pull_size;
3160                         case e1000_82544:
3161                                 /* Make sure we have room to chop off 4 bytes,
3162                                  * and that the end alignment will work out to
3163                                  * this hardware's requirements
3164                                  * NOTE: this is a TSO only workaround
3165                                  * if end byte alignment not correct move us
3166                                  * into the next dword
3167                                  */
3168                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3169                                     & 4)
3170                                         break;
3171                                 /* fall through */
3172                                 pull_size = min((unsigned int)4, skb->data_len);
3173                                 if (!__pskb_pull_tail(skb, pull_size)) {
3174                                         e_err(drv, "__pskb_pull_tail "
3175                                               "failed.\n");
3176                                         dev_kfree_skb_any(skb);
3177                                         return NETDEV_TX_OK;
3178                                 }
3179                                 len = skb_headlen(skb);
3180                                 break;
3181                         default:
3182                                 /* do nothing */
3183                                 break;
3184                         }
3185                 }
3186         }
3187
3188         /* reserve a descriptor for the offload context */
3189         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3190                 count++;
3191         count++;
3192
3193         /* Controller Erratum workaround */
3194         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3195                 count++;
3196
3197         count += TXD_USE_COUNT(len, max_txd_pwr);
3198
3199         if (adapter->pcix_82544)
3200                 count++;
3201
3202         /* work-around for errata 10 and it applies to all controllers
3203          * in PCI-X mode, so add one more descriptor to the count
3204          */
3205         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3206                         (len > 2015)))
3207                 count++;
3208
3209         nr_frags = skb_shinfo(skb)->nr_frags;
3210         for (f = 0; f < nr_frags; f++)
3211                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3212                                        max_txd_pwr);
3213         if (adapter->pcix_82544)
3214                 count += nr_frags;
3215
3216         /* need: count + 2 desc gap to keep tail from touching
3217          * head, otherwise try next time
3218          */
3219         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3220                 return NETDEV_TX_BUSY;
3221
3222         if (unlikely((hw->mac_type == e1000_82547) &&
3223                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3224                 netif_stop_queue(netdev);
3225                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3226                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3227                 return NETDEV_TX_BUSY;
3228         }
3229
3230         if (vlan_tx_tag_present(skb)) {
3231                 tx_flags |= E1000_TX_FLAGS_VLAN;
3232                 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
3233         }
3234
3235         first = tx_ring->next_to_use;
3236
3237         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3238         if (tso < 0) {
3239                 dev_kfree_skb_any(skb);
3240                 return NETDEV_TX_OK;
3241         }
3242
3243         if (likely(tso)) {
3244                 if (likely(hw->mac_type != e1000_82544))
3245                         tx_ring->last_tx_tso = true;
3246                 tx_flags |= E1000_TX_FLAGS_TSO;
3247         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3248                 tx_flags |= E1000_TX_FLAGS_CSUM;
3249
3250         if (protocol == htons(ETH_P_IP))
3251                 tx_flags |= E1000_TX_FLAGS_IPV4;
3252
3253         if (unlikely(skb->no_fcs))
3254                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3255
3256         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3257                              nr_frags, mss);
3258
3259         if (count) {
3260                 netdev_sent_queue(netdev, skb->len);
3261                 skb_tx_timestamp(skb);
3262
3263                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3264                 /* Make sure there is space in the ring for the next send. */
3265                 e1000_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 2);
3266
3267         } else {
3268                 dev_kfree_skb_any(skb);
3269                 tx_ring->buffer_info[first].time_stamp = 0;
3270                 tx_ring->next_to_use = first;
3271         }
3272
3273         return NETDEV_TX_OK;
3274 }
3275
3276 #define NUM_REGS 38 /* 1 based count */
3277 static void e1000_regdump(struct e1000_adapter *adapter)
3278 {
3279         struct e1000_hw *hw = &adapter->hw;
3280         u32 regs[NUM_REGS];
3281         u32 *regs_buff = regs;
3282         int i = 0;
3283
3284         static const char * const reg_name[] = {
3285                 "CTRL",  "STATUS",
3286                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3287                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3288                 "TIDV", "TXDCTL", "TADV", "TARC0",
3289                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3290                 "TXDCTL1", "TARC1",
3291                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3292                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3293                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3294         };
3295
3296         regs_buff[0]  = er32(CTRL);
3297         regs_buff[1]  = er32(STATUS);
3298
3299         regs_buff[2]  = er32(RCTL);
3300         regs_buff[3]  = er32(RDLEN);
3301         regs_buff[4]  = er32(RDH);
3302         regs_buff[5]  = er32(RDT);
3303         regs_buff[6]  = er32(RDTR);
3304
3305         regs_buff[7]  = er32(TCTL);
3306         regs_buff[8]  = er32(TDBAL);
3307         regs_buff[9]  = er32(TDBAH);
3308         regs_buff[10] = er32(TDLEN);
3309         regs_buff[11] = er32(TDH);
3310         regs_buff[12] = er32(TDT);
3311         regs_buff[13] = er32(TIDV);
3312         regs_buff[14] = er32(TXDCTL);
3313         regs_buff[15] = er32(TADV);
3314         regs_buff[16] = er32(TARC0);
3315
3316         regs_buff[17] = er32(TDBAL1);
3317         regs_buff[18] = er32(TDBAH1);
3318         regs_buff[19] = er32(TDLEN1);
3319         regs_buff[20] = er32(TDH1);
3320         regs_buff[21] = er32(TDT1);
3321         regs_buff[22] = er32(TXDCTL1);
3322         regs_buff[23] = er32(TARC1);
3323         regs_buff[24] = er32(CTRL_EXT);
3324         regs_buff[25] = er32(ERT);
3325         regs_buff[26] = er32(RDBAL0);
3326         regs_buff[27] = er32(RDBAH0);
3327         regs_buff[28] = er32(TDFH);
3328         regs_buff[29] = er32(TDFT);
3329         regs_buff[30] = er32(TDFHS);
3330         regs_buff[31] = er32(TDFTS);
3331         regs_buff[32] = er32(TDFPC);
3332         regs_buff[33] = er32(RDFH);
3333         regs_buff[34] = er32(RDFT);
3334         regs_buff[35] = er32(RDFHS);
3335         regs_buff[36] = er32(RDFTS);
3336         regs_buff[37] = er32(RDFPC);
3337
3338         pr_info("Register dump\n");
3339         for (i = 0; i < NUM_REGS; i++)
3340                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3341 }
3342
3343 /*
3344  * e1000_dump: Print registers, tx ring and rx ring
3345  */
3346 static void e1000_dump(struct e1000_adapter *adapter)
3347 {
3348         /* this code doesn't handle multiple rings */
3349         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3350         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3351         int i;
3352
3353         if (!netif_msg_hw(adapter))
3354                 return;
3355
3356         /* Print Registers */
3357         e1000_regdump(adapter);
3358
3359         /* transmit dump */
3360         pr_info("TX Desc ring0 dump\n");
3361
3362         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3363          *
3364          * Legacy Transmit Descriptor
3365          *   +--------------------------------------------------------------+
3366          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3367          *   +--------------------------------------------------------------+
3368          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3369          *   +--------------------------------------------------------------+
3370          *   63       48 47        36 35    32 31     24 23    16 15        0
3371          *
3372          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3373          *   63      48 47    40 39       32 31             16 15    8 7      0
3374          *   +----------------------------------------------------------------+
3375          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3376          *   +----------------------------------------------------------------+
3377          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3378          *   +----------------------------------------------------------------+
3379          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3380          *
3381          * Extended Data Descriptor (DTYP=0x1)
3382          *   +----------------------------------------------------------------+
3383          * 0 |                     Buffer Address [63:0]                      |
3384          *   +----------------------------------------------------------------+
3385          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3386          *   +----------------------------------------------------------------+
3387          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3388          */
3389         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3390         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3391
3392         if (!netif_msg_tx_done(adapter))
3393                 goto rx_ring_summary;
3394
3395         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3396                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3397                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3398                 struct my_u { __le64 a; __le64 b; };
3399                 struct my_u *u = (struct my_u *)tx_desc;
3400                 const char *type;
3401
3402                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3403                         type = "NTC/U";
3404                 else if (i == tx_ring->next_to_use)
3405                         type = "NTU";
3406                 else if (i == tx_ring->next_to_clean)
3407                         type = "NTC";
3408                 else
3409                         type = "";
3410
3411                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3412                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3413                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3414                         (u64)buffer_info->dma, buffer_info->length,
3415                         buffer_info->next_to_watch,
3416                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3417         }
3418
3419 rx_ring_summary:
3420         /* receive dump */
3421         pr_info("\nRX Desc ring dump\n");
3422
3423         /* Legacy Receive Descriptor Format
3424          *
3425          * +-----------------------------------------------------+
3426          * |                Buffer Address [63:0]                |
3427          * +-----------------------------------------------------+
3428          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3429          * +-----------------------------------------------------+
3430          * 63       48 47    40 39      32 31         16 15      0
3431          */
3432         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3433
3434         if (!netif_msg_rx_status(adapter))
3435                 goto exit;
3436
3437         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3438                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3439                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3440                 struct my_u { __le64 a; __le64 b; };
3441                 struct my_u *u = (struct my_u *)rx_desc;
3442                 const char *type;
3443
3444                 if (i == rx_ring->next_to_use)
3445                         type = "NTU";
3446                 else if (i == rx_ring->next_to_clean)
3447                         type = "NTC";
3448                 else
3449                         type = "";
3450
3451                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3452                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3453                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3454         } /* for */
3455
3456         /* dump the descriptor caches */
3457         /* rx */
3458         pr_info("Rx descriptor cache in 64bit format\n");
3459         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3460                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3461                         i,
3462                         readl(adapter->hw.hw_addr + i+4),
3463                         readl(adapter->hw.hw_addr + i),
3464                         readl(adapter->hw.hw_addr + i+12),
3465                         readl(adapter->hw.hw_addr + i+8));
3466         }
3467         /* tx */
3468         pr_info("Tx descriptor cache in 64bit format\n");
3469         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3470                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3471                         i,
3472                         readl(adapter->hw.hw_addr + i+4),
3473                         readl(adapter->hw.hw_addr + i),
3474                         readl(adapter->hw.hw_addr + i+12),
3475                         readl(adapter->hw.hw_addr + i+8));
3476         }
3477 exit:
3478         return;
3479 }
3480
3481 /**
3482  * e1000_tx_timeout - Respond to a Tx Hang
3483  * @netdev: network interface device structure
3484  **/
3485 static void e1000_tx_timeout(struct net_device *netdev)
3486 {
3487         struct e1000_adapter *adapter = netdev_priv(netdev);
3488
3489         /* Do the reset outside of interrupt context */
3490         adapter->tx_timeout_count++;
3491         schedule_work(&adapter->reset_task);
3492 }
3493
3494 static void e1000_reset_task(struct work_struct *work)
3495 {
3496         struct e1000_adapter *adapter =
3497                 container_of(work, struct e1000_adapter, reset_task);
3498
3499         e_err(drv, "Reset adapter\n");
3500         e1000_reinit_locked(adapter);
3501 }
3502
3503 /**
3504  * e1000_get_stats - Get System Network Statistics
3505  * @netdev: network interface device structure
3506  *
3507  * Returns the address of the device statistics structure.
3508  * The statistics are actually updated from the watchdog.
3509  **/
3510 static struct net_device_stats *e1000_get_stats(struct net_device *netdev)
3511 {
3512         /* only return the current stats */
3513         return &netdev->stats;
3514 }
3515
3516 /**
3517  * e1000_change_mtu - Change the Maximum Transfer Unit
3518  * @netdev: network interface device structure
3519  * @new_mtu: new value for maximum frame size
3520  *
3521  * Returns 0 on success, negative on failure
3522  **/
3523 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3524 {
3525         struct e1000_adapter *adapter = netdev_priv(netdev);
3526         struct e1000_hw *hw = &adapter->hw;
3527         int max_frame = new_mtu + ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
3528
3529         if ((max_frame < MINIMUM_ETHERNET_FRAME_SIZE) ||
3530             (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3531                 e_err(probe, "Invalid MTU setting\n");
3532                 return -EINVAL;
3533         }
3534
3535         /* Adapter-specific max frame size limits. */
3536         switch (hw->mac_type) {
3537         case e1000_undefined ... e1000_82542_rev2_1:
3538                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3539                         e_err(probe, "Jumbo Frames not supported.\n");
3540                         return -EINVAL;
3541                 }
3542                 break;
3543         default:
3544                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3545                 break;
3546         }
3547
3548         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3549                 msleep(1);
3550         /* e1000_down has a dependency on max_frame_size */
3551         hw->max_frame_size = max_frame;
3552         if (netif_running(netdev))
3553                 e1000_down(adapter);
3554
3555         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3556          * means we reserve 2 more, this pushes us to allocate from the next
3557          * larger slab size.
3558          * i.e. RXBUFFER_2048 --> size-4096 slab
3559          * however with the new *_jumbo_rx* routines, jumbo receives will use
3560          * fragmented skbs
3561          */
3562
3563         if (max_frame <= E1000_RXBUFFER_2048)
3564                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3565         else
3566 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3567                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3568 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3569                 adapter->rx_buffer_len = PAGE_SIZE;
3570 #endif
3571
3572         /* adjust allocation if LPE protects us, and we aren't using SBP */
3573         if (!hw->tbi_compatibility_on &&
3574             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3575              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3576                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3577
3578         pr_info("%s changing MTU from %d to %d\n",
3579                 netdev->name, netdev->mtu, new_mtu);
3580         netdev->mtu = new_mtu;
3581
3582         if (netif_running(netdev))
3583                 e1000_up(adapter);
3584         else
3585                 e1000_reset(adapter);
3586
3587         clear_bit(__E1000_RESETTING, &adapter->flags);
3588
3589         return 0;
3590 }
3591
3592 /**
3593  * e1000_update_stats - Update the board statistics counters
3594  * @adapter: board private structure
3595  **/
3596 void e1000_update_stats(struct e1000_adapter *adapter)
3597 {
3598         struct net_device *netdev = adapter->netdev;
3599         struct e1000_hw *hw = &adapter->hw;
3600         struct pci_dev *pdev = adapter->pdev;
3601         unsigned long flags;
3602         u16 phy_tmp;
3603
3604 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3605
3606         /* Prevent stats update while adapter is being reset, or if the pci
3607          * connection is down.
3608          */
3609         if (adapter->link_speed == 0)
3610                 return;
3611         if (pci_channel_offline(pdev))
3612                 return;
3613
3614         spin_lock_irqsave(&adapter->stats_lock, flags);
3615
3616         /* these counters are modified from e1000_tbi_adjust_stats,
3617          * called from the interrupt context, so they must only
3618          * be written while holding adapter->stats_lock
3619          */
3620
3621         adapter->stats.crcerrs += er32(CRCERRS);
3622         adapter->stats.gprc += er32(GPRC);
3623         adapter->stats.gorcl += er32(GORCL);
3624         adapter->stats.gorch += er32(GORCH);
3625         adapter->stats.bprc += er32(BPRC);
3626         adapter->stats.mprc += er32(MPRC);
3627         adapter->stats.roc += er32(ROC);
3628
3629         adapter->stats.prc64 += er32(PRC64);
3630         adapter->stats.prc127 += er32(PRC127);
3631         adapter->stats.prc255 += er32(PRC255);
3632         adapter->stats.prc511 += er32(PRC511);
3633         adapter->stats.prc1023 += er32(PRC1023);
3634         adapter->stats.prc1522 += er32(PRC1522);
3635
3636         adapter->stats.symerrs += er32(SYMERRS);
3637         adapter->stats.mpc += er32(MPC);
3638         adapter->stats.scc += er32(SCC);
3639         adapter->stats.ecol += er32(ECOL);
3640         adapter->stats.mcc += er32(MCC);
3641         adapter->stats.latecol += er32(LATECOL);
3642         adapter->stats.dc += er32(DC);
3643         adapter->stats.sec += er32(SEC);
3644         adapter->stats.rlec += er32(RLEC);
3645         adapter->stats.xonrxc += er32(XONRXC);
3646         adapter->stats.xontxc += er32(XONTXC);
3647         adapter->stats.xoffrxc += er32(XOFFRXC);
3648         adapter->stats.xofftxc += er32(XOFFTXC);
3649         adapter->stats.fcruc += er32(FCRUC);
3650         adapter->stats.gptc += er32(GPTC);
3651         adapter->stats.gotcl += er32(GOTCL);
3652         adapter->stats.gotch += er32(GOTCH);
3653         adapter->stats.rnbc += er32(RNBC);
3654         adapter->stats.ruc += er32(RUC);
3655         adapter->stats.rfc += er32(RFC);
3656         adapter->stats.rjc += er32(RJC);
3657         adapter->stats.torl += er32(TORL);
3658         adapter->stats.torh += er32(TORH);
3659         adapter->stats.totl += er32(TOTL);
3660         adapter->stats.toth += er32(TOTH);
3661         adapter->stats.tpr += er32(TPR);
3662
3663         adapter->stats.ptc64 += er32(PTC64);
3664         adapter->stats.ptc127 += er32(PTC127);
3665         adapter->stats.ptc255 += er32(PTC255);
3666         adapter->stats.ptc511 += er32(PTC511);
3667         adapter->stats.ptc1023 += er32(PTC1023);
3668         adapter->stats.ptc1522 += er32(PTC1522);
3669
3670         adapter->stats.mptc += er32(MPTC);
3671         adapter->stats.bptc += er32(BPTC);
3672
3673         /* used for adaptive IFS */
3674
3675         hw->tx_packet_delta = er32(TPT);
3676         adapter->stats.tpt += hw->tx_packet_delta;
3677         hw->collision_delta = er32(COLC);
3678         adapter->stats.colc += hw->collision_delta;
3679
3680         if (hw->mac_type >= e1000_82543) {
3681                 adapter->stats.algnerrc += er32(ALGNERRC);
3682                 adapter->stats.rxerrc += er32(RXERRC);
3683                 adapter->stats.tncrs += er32(TNCRS);
3684                 adapter->stats.cexterr += er32(CEXTERR);
3685                 adapter->stats.tsctc += er32(TSCTC);
3686                 adapter->stats.tsctfc += er32(TSCTFC);
3687         }
3688
3689         /* Fill out the OS statistics structure */
3690         netdev->stats.multicast = adapter->stats.mprc;
3691         netdev->stats.collisions = adapter->stats.colc;
3692
3693         /* Rx Errors */
3694
3695         /* RLEC on some newer hardware can be incorrect so build
3696          * our own version based on RUC and ROC
3697          */
3698         netdev->stats.rx_errors = adapter->stats.rxerrc +
3699                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3700                 adapter->stats.ruc + adapter->stats.roc +
3701                 adapter->stats.cexterr;
3702         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3703         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3704         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3705         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3706         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3707
3708         /* Tx Errors */
3709         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3710         netdev->stats.tx_errors = adapter->stats.txerrc;
3711         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3712         netdev->stats.tx_window_errors = adapter->stats.latecol;
3713         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3714         if (hw->bad_tx_carr_stats_fd &&
3715             adapter->link_duplex == FULL_DUPLEX) {
3716                 netdev->stats.tx_carrier_errors = 0;
3717                 adapter->stats.tncrs = 0;
3718         }
3719
3720         /* Tx Dropped needs to be maintained elsewhere */
3721
3722         /* Phy Stats */
3723         if (hw->media_type == e1000_media_type_copper) {
3724                 if ((adapter->link_speed == SPEED_1000) &&
3725                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3726                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3727                         adapter->phy_stats.idle_errors += phy_tmp;
3728                 }
3729
3730                 if ((hw->mac_type <= e1000_82546) &&
3731                    (hw->phy_type == e1000_phy_m88) &&
3732                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3733                         adapter->phy_stats.receive_errors += phy_tmp;
3734         }
3735
3736         /* Management Stats */
3737         if (hw->has_smbus) {
3738                 adapter->stats.mgptc += er32(MGTPTC);
3739                 adapter->stats.mgprc += er32(MGTPRC);
3740                 adapter->stats.mgpdc += er32(MGTPDC);
3741         }
3742
3743         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3744 }
3745
3746 /**
3747  * e1000_intr - Interrupt Handler
3748  * @irq: interrupt number
3749  * @data: pointer to a network interface device structure
3750  **/
3751 static irqreturn_t e1000_intr(int irq, void *data)
3752 {
3753         struct net_device *netdev = data;
3754         struct e1000_adapter *adapter = netdev_priv(netdev);
3755         struct e1000_hw *hw = &adapter->hw;
3756         u32 icr = er32(ICR);
3757
3758         if (unlikely((!icr)))
3759                 return IRQ_NONE;  /* Not our interrupt */
3760
3761         /* we might have caused the interrupt, but the above
3762          * read cleared it, and just in case the driver is
3763          * down there is nothing to do so return handled
3764          */
3765         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3766                 return IRQ_HANDLED;
3767
3768         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3769                 hw->get_link_status = 1;
3770                 /* guard against interrupt when we're going down */
3771                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3772                         schedule_delayed_work(&adapter->watchdog_task, 1);
3773         }
3774
3775         /* disable interrupts, without the synchronize_irq bit */
3776         ew32(IMC, ~0);
3777         E1000_WRITE_FLUSH();
3778
3779         if (likely(napi_schedule_prep(&adapter->napi))) {
3780                 adapter->total_tx_bytes = 0;
3781                 adapter->total_tx_packets = 0;
3782                 adapter->total_rx_bytes = 0;
3783                 adapter->total_rx_packets = 0;
3784                 __napi_schedule(&adapter->napi);
3785         } else {
3786                 /* this really should not happen! if it does it is basically a
3787                  * bug, but not a hard error, so enable ints and continue
3788                  */
3789                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3790                         e1000_irq_enable(adapter);
3791         }
3792
3793         return IRQ_HANDLED;
3794 }
3795
3796 /**
3797  * e1000_clean - NAPI Rx polling callback
3798  * @adapter: board private structure
3799  **/
3800 static int e1000_clean(struct napi_struct *napi, int budget)
3801 {
3802         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3803                                                      napi);
3804         int tx_clean_complete = 0, work_done = 0;
3805
3806         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3807
3808         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3809
3810         if (!tx_clean_complete)
3811                 work_done = budget;
3812
3813         /* If budget not fully consumed, exit the polling mode */
3814         if (work_done < budget) {
3815                 if (likely(adapter->itr_setting & 3))
3816                         e1000_set_itr(adapter);
3817                 napi_complete(napi);
3818                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3819                         e1000_irq_enable(adapter);
3820         }
3821
3822         return work_done;
3823 }
3824
3825 /**
3826  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3827  * @adapter: board private structure
3828  **/
3829 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3830                                struct e1000_tx_ring *tx_ring)
3831 {
3832         struct e1000_hw *hw = &adapter->hw;
3833         struct net_device *netdev = adapter->netdev;
3834         struct e1000_tx_desc *tx_desc, *eop_desc;
3835         struct e1000_tx_buffer *buffer_info;
3836         unsigned int i, eop;
3837         unsigned int count = 0;
3838         unsigned int total_tx_bytes=0, total_tx_packets=0;
3839         unsigned int bytes_compl = 0, pkts_compl = 0;
3840
3841         i = tx_ring->next_to_clean;
3842         eop = tx_ring->buffer_info[i].next_to_watch;
3843         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3844
3845         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3846                (count < tx_ring->count)) {
3847                 bool cleaned = false;
3848                 rmb();  /* read buffer_info after eop_desc */
3849                 for ( ; !cleaned; count++) {
3850                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3851                         buffer_info = &tx_ring->buffer_info[i];
3852                         cleaned = (i == eop);
3853
3854                         if (cleaned) {
3855                                 total_tx_packets += buffer_info->segs;
3856                                 total_tx_bytes += buffer_info->bytecount;
3857                                 if (buffer_info->skb) {
3858                                         bytes_compl += buffer_info->skb->len;
3859                                         pkts_compl++;
3860                                 }
3861
3862                         }
3863                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3864                         tx_desc->upper.data = 0;
3865
3866                         if (unlikely(++i == tx_ring->count)) i = 0;
3867                 }
3868
3869                 eop = tx_ring->buffer_info[i].next_to_watch;
3870                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3871         }
3872
3873         tx_ring->next_to_clean = i;
3874
3875         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3876
3877 #define TX_WAKE_THRESHOLD 32
3878         if (unlikely(count && netif_carrier_ok(netdev) &&
3879                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3880                 /* Make sure that anybody stopping the queue after this
3881                  * sees the new next_to_clean.
3882                  */
3883                 smp_mb();
3884
3885                 if (netif_queue_stopped(netdev) &&
3886                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3887                         netif_wake_queue(netdev);
3888                         ++adapter->restart_queue;
3889                 }
3890         }
3891
3892         if (adapter->detect_tx_hung) {
3893                 /* Detect a transmit hang in hardware, this serializes the
3894                  * check with the clearing of time_stamp and movement of i
3895                  */
3896                 adapter->detect_tx_hung = false;
3897                 if (tx_ring->buffer_info[eop].time_stamp &&
3898                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3899                                (adapter->tx_timeout_factor * HZ)) &&
3900                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3901
3902                         /* detected Tx unit hang */
3903                         e_err(drv, "Detected Tx Unit Hang\n"
3904                               "  Tx Queue             <%lu>\n"
3905                               "  TDH                  <%x>\n"
3906                               "  TDT                  <%x>\n"
3907                               "  next_to_use          <%x>\n"
3908                               "  next_to_clean        <%x>\n"
3909                               "buffer_info[next_to_clean]\n"
3910                               "  time_stamp           <%lx>\n"
3911                               "  next_to_watch        <%x>\n"
3912                               "  jiffies              <%lx>\n"
3913                               "  next_to_watch.status <%x>\n",
3914                                 (unsigned long)(tx_ring - adapter->tx_ring),
3915                                 readl(hw->hw_addr + tx_ring->tdh),
3916                                 readl(hw->hw_addr + tx_ring->tdt),
3917                                 tx_ring->next_to_use,
3918                                 tx_ring->next_to_clean,
3919                                 tx_ring->buffer_info[eop].time_stamp,
3920                                 eop,
3921                                 jiffies,
3922                                 eop_desc->upper.fields.status);
3923                         e1000_dump(adapter);
3924                         netif_stop_queue(netdev);
3925                 }
3926         }
3927         adapter->total_tx_bytes += total_tx_bytes;
3928         adapter->total_tx_packets += total_tx_packets;
3929         netdev->stats.tx_bytes += total_tx_bytes;
3930         netdev->stats.tx_packets += total_tx_packets;
3931         return count < tx_ring->count;
3932 }
3933
3934 /**
3935  * e1000_rx_checksum - Receive Checksum Offload for 82543
3936  * @adapter:     board private structure
3937  * @status_err:  receive descriptor status and error fields
3938  * @csum:        receive descriptor csum field
3939  * @sk_buff:     socket buffer with received data
3940  **/
3941 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3942                               u32 csum, struct sk_buff *skb)
3943 {
3944         struct e1000_hw *hw = &adapter->hw;
3945         u16 status = (u16)status_err;
3946         u8 errors = (u8)(status_err >> 24);
3947
3948         skb_checksum_none_assert(skb);
3949
3950         /* 82543 or newer only */
3951         if (unlikely(hw->mac_type < e1000_82543)) return;
3952         /* Ignore Checksum bit is set */
3953         if (unlikely(status & E1000_RXD_STAT_IXSM)) return;
3954         /* TCP/UDP checksum error bit is set */
3955         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3956                 /* let the stack verify checksum errors */
3957                 adapter->hw_csum_err++;
3958                 return;
3959         }
3960         /* TCP/UDP Checksum has not been calculated */
3961         if (!(status & E1000_RXD_STAT_TCPCS))
3962                 return;
3963
3964         /* It must be a TCP or UDP packet with a valid checksum */
3965         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3966                 /* TCP checksum is good */
3967                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3968         }
3969         adapter->hw_csum_good++;
3970 }
3971
3972 /**
3973  * e1000_consume_page - helper function for jumbo Rx path
3974  **/
3975 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3976                                u16 length)
3977 {
3978         bi->rxbuf.page = NULL;
3979         skb->len += length;
3980         skb->data_len += length;
3981         skb->truesize += PAGE_SIZE;
3982 }
3983
3984 /**
3985  * e1000_receive_skb - helper function to handle rx indications
3986  * @adapter: board private structure
3987  * @status: descriptor status field as written by hardware
3988  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3989  * @skb: pointer to sk_buff to be indicated to stack
3990  */
3991 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3992                               __le16 vlan, struct sk_buff *skb)
3993 {
3994         skb->protocol = eth_type_trans(skb, adapter->netdev);
3995
3996         if (status & E1000_RXD_STAT_VP) {
3997                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
3998
3999                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4000         }
4001         napi_gro_receive(&adapter->napi, skb);
4002 }
4003
4004 /**
4005  * e1000_tbi_adjust_stats
4006  * @hw: Struct containing variables accessed by shared code
4007  * @frame_len: The length of the frame in question
4008  * @mac_addr: The Ethernet destination address of the frame in question
4009  *
4010  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4011  */
4012 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4013                                    struct e1000_hw_stats *stats,
4014                                    u32 frame_len, const u8 *mac_addr)
4015 {
4016         u64 carry_bit;
4017
4018         /* First adjust the frame length. */
4019         frame_len--;
4020         /* We need to adjust the statistics counters, since the hardware
4021          * counters overcount this packet as a CRC error and undercount
4022          * the packet as a good packet
4023          */
4024         /* This packet should not be counted as a CRC error. */
4025         stats->crcerrs--;
4026         /* This packet does count as a Good Packet Received. */
4027         stats->gprc++;
4028
4029         /* Adjust the Good Octets received counters */
4030         carry_bit = 0x80000000 & stats->gorcl;
4031         stats->gorcl += frame_len;
4032         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4033          * Received Count) was one before the addition,
4034          * AND it is zero after, then we lost the carry out,
4035          * need to add one to Gorch (Good Octets Received Count High).
4036          * This could be simplified if all environments supported
4037          * 64-bit integers.
4038          */
4039         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4040                 stats->gorch++;
4041         /* Is this a broadcast or multicast?  Check broadcast first,
4042          * since the test for a multicast frame will test positive on
4043          * a broadcast frame.
4044          */
4045         if (is_broadcast_ether_addr(mac_addr))
4046                 stats->bprc++;
4047         else if (is_multicast_ether_addr(mac_addr))
4048                 stats->mprc++;
4049
4050         if (frame_len == hw->max_frame_size) {
4051                 /* In this case, the hardware has overcounted the number of
4052                  * oversize frames.
4053                  */
4054                 if (stats->roc > 0)
4055                         stats->roc--;
4056         }
4057
4058         /* Adjust the bin counters when the extra byte put the frame in the
4059          * wrong bin. Remember that the frame_len was adjusted above.
4060          */
4061         if (frame_len == 64) {
4062                 stats->prc64++;
4063                 stats->prc127--;
4064         } else if (frame_len == 127) {
4065                 stats->prc127++;
4066                 stats->prc255--;
4067         } else if (frame_len == 255) {
4068                 stats->prc255++;
4069                 stats->prc511--;
4070         } else if (frame_len == 511) {
4071                 stats->prc511++;
4072                 stats->prc1023--;
4073         } else if (frame_len == 1023) {
4074                 stats->prc1023++;
4075                 stats->prc1522--;
4076         } else if (frame_len == 1522) {
4077                 stats->prc1522++;
4078         }
4079 }
4080
4081 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4082                                     u8 status, u8 errors,
4083                                     u32 length, const u8 *data)
4084 {
4085         struct e1000_hw *hw = &adapter->hw;
4086         u8 last_byte = *(data + length - 1);
4087
4088         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4089                 unsigned long irq_flags;
4090
4091                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4092                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4093                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4094
4095                 return true;
4096         }
4097
4098         return false;
4099 }
4100
4101 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4102                                           unsigned int bufsz)
4103 {
4104         struct sk_buff *skb = netdev_alloc_skb_ip_align(adapter->netdev, bufsz);
4105
4106         if (unlikely(!skb))
4107                 adapter->alloc_rx_buff_failed++;
4108         return skb;
4109 }
4110
4111 /**
4112  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4113  * @adapter: board private structure
4114  * @rx_ring: ring to clean
4115  * @work_done: amount of napi work completed this call
4116  * @work_to_do: max amount of work allowed for this call to do
4117  *
4118  * the return value indicates whether actual cleaning was done, there
4119  * is no guarantee that everything was cleaned
4120  */
4121 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4122                                      struct e1000_rx_ring *rx_ring,
4123                                      int *work_done, int work_to_do)
4124 {
4125         struct net_device *netdev = adapter->netdev;
4126         struct pci_dev *pdev = adapter->pdev;
4127         struct e1000_rx_desc *rx_desc, *next_rxd;
4128         struct e1000_rx_buffer *buffer_info, *next_buffer;
4129         u32 length;
4130         unsigned int i;
4131         int cleaned_count = 0;
4132         bool cleaned = false;
4133         unsigned int total_rx_bytes=0, total_rx_packets=0;
4134
4135         i = rx_ring->next_to_clean;
4136         rx_desc = E1000_RX_DESC(*rx_ring, i);
4137         buffer_info = &rx_ring->buffer_info[i];
4138
4139         while (rx_desc->status & E1000_RXD_STAT_DD) {
4140                 struct sk_buff *skb;
4141                 u8 status;
4142
4143                 if (*work_done >= work_to_do)
4144                         break;
4145                 (*work_done)++;
4146                 rmb(); /* read descriptor and rx_buffer_info after status DD */
4147
4148                 status = rx_desc->status;
4149
4150                 if (++i == rx_ring->count) i = 0;
4151                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4152                 prefetch(next_rxd);
4153
4154                 next_buffer = &rx_ring->buffer_info[i];
4155
4156                 cleaned = true;
4157                 cleaned_count++;
4158                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4159                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4160                 buffer_info->dma = 0;
4161
4162                 length = le16_to_cpu(rx_desc->length);
4163
4164                 /* errors is only valid for DD + EOP descriptors */
4165                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4166                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4167                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4168
4169                         if (e1000_tbi_should_accept(adapter, status,
4170                                                     rx_desc->errors,
4171                                                     length, mapped)) {
4172                                 length--;
4173                         } else if (netdev->features & NETIF_F_RXALL) {
4174                                 goto process_skb;
4175                         } else {
4176                                 /* an error means any chain goes out the window
4177                                  * too
4178                                  */
4179                                 if (rx_ring->rx_skb_top)
4180                                         dev_kfree_skb(rx_ring->rx_skb_top);
4181                                 rx_ring->rx_skb_top = NULL;
4182                                 goto next_desc;
4183                         }
4184                 }
4185
4186 #define rxtop rx_ring->rx_skb_top
4187 process_skb:
4188                 if (!(status & E1000_RXD_STAT_EOP)) {
4189                         /* this descriptor is only the beginning (or middle) */
4190                         if (!rxtop) {
4191                                 /* this is the beginning of a chain */
4192                                 rxtop = napi_get_frags(&adapter->napi);
4193                                 if (!rxtop)
4194                                         break;
4195
4196                                 skb_fill_page_desc(rxtop, 0,
4197                                                    buffer_info->rxbuf.page,
4198                                                    0, length);
4199                         } else {
4200                                 /* this is the middle of a chain */
4201                                 skb_fill_page_desc(rxtop,
4202                                     skb_shinfo(rxtop)->nr_frags,
4203                                     buffer_info->rxbuf.page, 0, length);
4204                         }
4205                         e1000_consume_page(buffer_info, rxtop, length);
4206                         goto next_desc;
4207                 } else {
4208                         if (rxtop) {
4209                                 /* end of the chain */
4210                                 skb_fill_page_desc(rxtop,
4211                                     skb_shinfo(rxtop)->nr_frags,
4212                                     buffer_info->rxbuf.page, 0, length);
4213                                 skb = rxtop;
4214                                 rxtop = NULL;
4215                                 e1000_consume_page(buffer_info, skb, length);
4216                         } else {
4217                                 struct page *p;
4218                                 /* no chain, got EOP, this buf is the packet
4219                                  * copybreak to save the put_page/alloc_page
4220                                  */
4221                                 p = buffer_info->rxbuf.page;
4222                                 if (length <= copybreak) {
4223                                         u8 *vaddr;
4224
4225                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4226                                                 length -= 4;
4227                                         skb = e1000_alloc_rx_skb(adapter,
4228                                                                  length);
4229                                         if (!skb)
4230                                                 break;
4231
4232                                         vaddr = kmap_atomic(p);
4233                                         memcpy(skb_tail_pointer(skb), vaddr,
4234                                                length);
4235                                         kunmap_atomic(vaddr);
4236                                         /* re-use the page, so don't erase
4237                                          * buffer_info->rxbuf.page
4238                                          */
4239                                         skb_put(skb, length);
4240                                         e1000_rx_checksum(adapter,
4241                                                           status | rx_desc->errors << 24,
4242                                                           le16_to_cpu(rx_desc->csum), skb);
4243
4244                                         total_rx_bytes += skb->len;
4245                                         total_rx_packets++;
4246
4247                                         e1000_receive_skb(adapter, status,
4248                                                           rx_desc->special, skb);
4249                                         goto next_desc;
4250                                 } else {
4251                                         skb = napi_get_frags(&adapter->napi);
4252                                         if (!skb) {
4253                                                 adapter->alloc_rx_buff_failed++;
4254                                                 break;
4255                                         }
4256                                         skb_fill_page_desc(skb, 0, p, 0,
4257                                                            length);
4258                                         e1000_consume_page(buffer_info, skb,
4259                                                            length);
4260                                 }
4261                         }
4262                 }
4263
4264                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4265                 e1000_rx_checksum(adapter,
4266                                   (u32)(status) |
4267                                   ((u32)(rx_desc->errors) << 24),
4268                                   le16_to_cpu(rx_desc->csum), skb);
4269
4270                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4271                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4272                         pskb_trim(skb, skb->len - 4);
4273                 total_rx_packets++;
4274
4275                 if (status & E1000_RXD_STAT_VP) {
4276                         __le16 vlan = rx_desc->special;
4277                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4278
4279                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4280                 }
4281
4282                 napi_gro_frags(&adapter->napi);
4283
4284 next_desc:
4285                 rx_desc->status = 0;
4286
4287                 /* return some buffers to hardware, one at a time is too slow */
4288                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4289                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4290                         cleaned_count = 0;
4291                 }
4292
4293                 /* use prefetched values */
4294                 rx_desc = next_rxd;
4295                 buffer_info = next_buffer;
4296         }
4297         rx_ring->next_to_clean = i;
4298
4299         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4300         if (cleaned_count)
4301                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4302
4303         adapter->total_rx_packets += total_rx_packets;
4304         adapter->total_rx_bytes += total_rx_bytes;
4305         netdev->stats.rx_bytes += total_rx_bytes;
4306         netdev->stats.rx_packets += total_rx_packets;
4307         return cleaned;
4308 }
4309
4310 /* this should improve performance for small packets with large amounts
4311  * of reassembly being done in the stack
4312  */
4313 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4314                                        struct e1000_rx_buffer *buffer_info,
4315                                        u32 length, const void *data)
4316 {
4317         struct sk_buff *skb;
4318
4319         if (length > copybreak)
4320                 return NULL;
4321
4322         skb = e1000_alloc_rx_skb(adapter, length);
4323         if (!skb)
4324                 return NULL;
4325
4326         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4327                                 length, DMA_FROM_DEVICE);
4328
4329         memcpy(skb_put(skb, length), data, length);
4330
4331         return skb;
4332 }
4333
4334 /**
4335  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4336  * @adapter: board private structure
4337  * @rx_ring: ring to clean
4338  * @work_done: amount of napi work completed this call
4339  * @work_to_do: max amount of work allowed for this call to do
4340  */
4341 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4342                                struct e1000_rx_ring *rx_ring,
4343                                int *work_done, int work_to_do)
4344 {
4345         struct net_device *netdev = adapter->netdev;
4346         struct pci_dev *pdev = adapter->pdev;
4347         struct e1000_rx_desc *rx_desc, *next_rxd;
4348         struct e1000_rx_buffer *buffer_info, *next_buffer;
4349         u32 length;
4350         unsigned int i;
4351         int cleaned_count = 0;
4352         bool cleaned = false;
4353         unsigned int total_rx_bytes=0, total_rx_packets=0;
4354
4355         i = rx_ring->next_to_clean;
4356         rx_desc = E1000_RX_DESC(*rx_ring, i);
4357         buffer_info = &rx_ring->buffer_info[i];
4358
4359         while (rx_desc->status & E1000_RXD_STAT_DD) {
4360                 struct sk_buff *skb;
4361                 u8 *data;
4362                 u8 status;
4363
4364                 if (*work_done >= work_to_do)
4365                         break;
4366                 (*work_done)++;
4367                 rmb(); /* read descriptor and rx_buffer_info after status DD */
4368
4369                 status = rx_desc->status;
4370                 length = le16_to_cpu(rx_desc->length);
4371
4372                 data = buffer_info->rxbuf.data;
4373                 prefetch(data);
4374                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4375                 if (!skb) {
4376                         unsigned int frag_len = e1000_frag_len(adapter);
4377
4378                         skb = build_skb(data - E1000_HEADROOM, frag_len);
4379                         if (!skb) {
4380                                 adapter->alloc_rx_buff_failed++;
4381                                 break;
4382                         }
4383
4384                         skb_reserve(skb, E1000_HEADROOM);
4385                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4386                                          adapter->rx_buffer_len,
4387                                          DMA_FROM_DEVICE);
4388                         buffer_info->dma = 0;
4389                         buffer_info->rxbuf.data = NULL;
4390                 }
4391
4392                 if (++i == rx_ring->count) i = 0;
4393                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4394                 prefetch(next_rxd);
4395
4396                 next_buffer = &rx_ring->buffer_info[i];
4397
4398                 cleaned = true;
4399                 cleaned_count++;
4400
4401                 /* !EOP means multiple descriptors were used to store a single
4402                  * packet, if thats the case we need to toss it.  In fact, we
4403                  * to toss every packet with the EOP bit clear and the next
4404                  * frame that _does_ have the EOP bit set, as it is by
4405                  * definition only a frame fragment
4406                  */
4407                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4408                         adapter->discarding = true;
4409
4410                 if (adapter->discarding) {
4411                         /* All receives must fit into a single buffer */
4412                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4413                         dev_kfree_skb(skb);
4414                         if (status & E1000_RXD_STAT_EOP)
4415                                 adapter->discarding = false;
4416                         goto next_desc;
4417                 }
4418
4419                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4420                         if (e1000_tbi_should_accept(adapter, status,
4421                                                     rx_desc->errors,
4422                                                     length, data)) {
4423                                 length--;
4424                         } else if (netdev->features & NETIF_F_RXALL) {
4425                                 goto process_skb;
4426                         } else {
4427                                 dev_kfree_skb(skb);
4428                                 goto next_desc;
4429                         }
4430                 }
4431
4432 process_skb:
4433                 total_rx_bytes += (length - 4); /* don't count FCS */
4434                 total_rx_packets++;
4435
4436                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4437                         /* adjust length to remove Ethernet CRC, this must be
4438                          * done after the TBI_ACCEPT workaround above
4439                          */
4440                         length -= 4;
4441
4442                 if (buffer_info->rxbuf.data == NULL)
4443                         skb_put(skb, length);
4444                 else /* copybreak skb */
4445                         skb_trim(skb, length);
4446
4447                 /* Receive Checksum Offload */
4448                 e1000_rx_checksum(adapter,
4449                                   (u32)(status) |
4450                                   ((u32)(rx_desc->errors) << 24),
4451                                   le16_to_cpu(rx_desc->csum), skb);
4452
4453                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4454
4455 next_desc:
4456                 rx_desc->status = 0;
4457
4458                 /* return some buffers to hardware, one at a time is too slow */
4459                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4460                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4461                         cleaned_count = 0;
4462                 }
4463
4464                 /* use prefetched values */
4465                 rx_desc = next_rxd;
4466                 buffer_info = next_buffer;
4467         }
4468         rx_ring->next_to_clean = i;
4469
4470         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4471         if (cleaned_count)
4472                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4473
4474         adapter->total_rx_packets += total_rx_packets;
4475         adapter->total_rx_bytes += total_rx_bytes;
4476         netdev->stats.rx_bytes += total_rx_bytes;
4477         netdev->stats.rx_packets += total_rx_packets;
4478         return cleaned;
4479 }
4480
4481 /**
4482  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4483  * @adapter: address of board private structure
4484  * @rx_ring: pointer to receive ring structure
4485  * @cleaned_count: number of buffers to allocate this pass
4486  **/
4487 static void
4488 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4489                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4490 {
4491         struct pci_dev *pdev = adapter->pdev;
4492         struct e1000_rx_desc *rx_desc;
4493         struct e1000_rx_buffer *buffer_info;
4494         unsigned int i;
4495
4496         i = rx_ring->next_to_use;
4497         buffer_info = &rx_ring->buffer_info[i];
4498
4499         while (cleaned_count--) {
4500                 /* allocate a new page if necessary */
4501                 if (!buffer_info->rxbuf.page) {
4502                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4503                         if (unlikely(!buffer_info->rxbuf.page)) {
4504                                 adapter->alloc_rx_buff_failed++;
4505                                 break;
4506                         }
4507                 }
4508
4509                 if (!buffer_info->dma) {
4510                         buffer_info->dma = dma_map_page(&pdev->dev,
4511                                                         buffer_info->rxbuf.page, 0,
4512                                                         adapter->rx_buffer_len,
4513                                                         DMA_FROM_DEVICE);
4514                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4515                                 put_page(buffer_info->rxbuf.page);
4516                                 buffer_info->rxbuf.page = NULL;
4517                                 buffer_info->dma = 0;
4518                                 adapter->alloc_rx_buff_failed++;
4519                                 break;
4520                         }
4521                 }
4522
4523                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4524                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4525
4526                 if (unlikely(++i == rx_ring->count))
4527                         i = 0;
4528                 buffer_info = &rx_ring->buffer_info[i];
4529         }
4530
4531         if (likely(rx_ring->next_to_use != i)) {
4532                 rx_ring->next_to_use = i;
4533                 if (unlikely(i-- == 0))
4534                         i = (rx_ring->count - 1);
4535
4536                 /* Force memory writes to complete before letting h/w
4537                  * know there are new descriptors to fetch.  (Only
4538                  * applicable for weak-ordered memory model archs,
4539                  * such as IA-64).
4540                  */
4541                 wmb();
4542                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4543         }
4544 }
4545
4546 /**
4547  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4548  * @adapter: address of board private structure
4549  **/
4550 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4551                                    struct e1000_rx_ring *rx_ring,
4552                                    int cleaned_count)
4553 {
4554         struct e1000_hw *hw = &adapter->hw;
4555         struct pci_dev *pdev = adapter->pdev;
4556         struct e1000_rx_desc *rx_desc;
4557         struct e1000_rx_buffer *buffer_info;
4558         unsigned int i;
4559         unsigned int bufsz = adapter->rx_buffer_len;
4560
4561         i = rx_ring->next_to_use;
4562         buffer_info = &rx_ring->buffer_info[i];
4563
4564         while (cleaned_count--) {
4565                 void *data;
4566
4567                 if (buffer_info->rxbuf.data)
4568                         goto skip;
4569
4570                 data = e1000_alloc_frag(adapter);
4571                 if (!data) {
4572                         /* Better luck next round */
4573                         adapter->alloc_rx_buff_failed++;
4574                         break;
4575                 }
4576
4577                 /* Fix for errata 23, can't cross 64kB boundary */
4578                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4579                         void *olddata = data;
4580                         e_err(rx_err, "skb align check failed: %u bytes at "
4581                               "%p\n", bufsz, data);
4582                         /* Try again, without freeing the previous */
4583                         data = e1000_alloc_frag(adapter);
4584                         /* Failed allocation, critical failure */
4585                         if (!data) {
4586                                 e1000_free_frag(olddata);
4587                                 adapter->alloc_rx_buff_failed++;
4588                                 break;
4589                         }
4590
4591                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4592                                 /* give up */
4593                                 e1000_free_frag(data);
4594                                 e1000_free_frag(olddata);
4595                                 adapter->alloc_rx_buff_failed++;
4596                                 break;
4597                         }
4598
4599                         /* Use new allocation */
4600                         e1000_free_frag(olddata);
4601                 }
4602                 buffer_info->dma = dma_map_single(&pdev->dev,
4603                                                   data,
4604                                                   adapter->rx_buffer_len,
4605                                                   DMA_FROM_DEVICE);
4606                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4607                         e1000_free_frag(data);
4608                         buffer_info->dma = 0;
4609                         adapter->alloc_rx_buff_failed++;
4610                         break;
4611                 }
4612
4613                 /* XXX if it was allocated cleanly it will never map to a
4614                  * boundary crossing
4615                  */
4616
4617                 /* Fix for errata 23, can't cross 64kB boundary */
4618                 if (!e1000_check_64k_bound(adapter,
4619                                         (void *)(unsigned long)buffer_info->dma,
4620                                         adapter->rx_buffer_len)) {
4621                         e_err(rx_err, "dma align check failed: %u bytes at "
4622                               "%p\n", adapter->rx_buffer_len,
4623                               (void *)(unsigned long)buffer_info->dma);
4624
4625                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4626                                          adapter->rx_buffer_len,
4627                                          DMA_FROM_DEVICE);
4628
4629                         e1000_free_frag(data);
4630                         buffer_info->rxbuf.data = NULL;
4631                         buffer_info->dma = 0;
4632
4633                         adapter->alloc_rx_buff_failed++;
4634                         break;
4635                 }
4636                 buffer_info->rxbuf.data = data;
4637  skip:
4638                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4639                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4640
4641                 if (unlikely(++i == rx_ring->count))
4642                         i = 0;
4643                 buffer_info = &rx_ring->buffer_info[i];
4644         }
4645
4646         if (likely(rx_ring->next_to_use != i)) {
4647                 rx_ring->next_to_use = i;
4648                 if (unlikely(i-- == 0))
4649                         i = (rx_ring->count - 1);
4650
4651                 /* Force memory writes to complete before letting h/w
4652                  * know there are new descriptors to fetch.  (Only
4653                  * applicable for weak-ordered memory model archs,
4654                  * such as IA-64).
4655                  */
4656                 wmb();
4657                 writel(i, hw->hw_addr + rx_ring->rdt);
4658         }
4659 }
4660
4661 /**
4662  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4663  * @adapter:
4664  **/
4665 static void e1000_smartspeed(struct e1000_adapter *adapter)
4666 {
4667         struct e1000_hw *hw = &adapter->hw;
4668         u16 phy_status;
4669         u16 phy_ctrl;
4670
4671         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4672            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4673                 return;
4674
4675         if (adapter->smartspeed == 0) {
4676                 /* If Master/Slave config fault is asserted twice,
4677                  * we assume back-to-back
4678                  */
4679                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4680                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
4681                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4682                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
4683                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4684                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4685                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4686                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4687                                             phy_ctrl);
4688                         adapter->smartspeed++;
4689                         if (!e1000_phy_setup_autoneg(hw) &&
4690                            !e1000_read_phy_reg(hw, PHY_CTRL,
4691                                                &phy_ctrl)) {
4692                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4693                                              MII_CR_RESTART_AUTO_NEG);
4694                                 e1000_write_phy_reg(hw, PHY_CTRL,
4695                                                     phy_ctrl);
4696                         }
4697                 }
4698                 return;
4699         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4700                 /* If still no link, perhaps using 2/3 pair cable */
4701                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4702                 phy_ctrl |= CR_1000T_MS_ENABLE;
4703                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4704                 if (!e1000_phy_setup_autoneg(hw) &&
4705                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4706                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4707                                      MII_CR_RESTART_AUTO_NEG);
4708                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4709                 }
4710         }
4711         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4712         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4713                 adapter->smartspeed = 0;
4714 }
4715
4716 /**
4717  * e1000_ioctl -
4718  * @netdev:
4719  * @ifreq:
4720  * @cmd:
4721  **/
4722 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4723 {
4724         switch (cmd) {
4725         case SIOCGMIIPHY:
4726         case SIOCGMIIREG:
4727         case SIOCSMIIREG:
4728                 return e1000_mii_ioctl(netdev, ifr, cmd);
4729         default:
4730                 return -EOPNOTSUPP;
4731         }
4732 }
4733
4734 /**
4735  * e1000_mii_ioctl -
4736  * @netdev:
4737  * @ifreq:
4738  * @cmd:
4739  **/
4740 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4741                            int cmd)
4742 {
4743         struct e1000_adapter *adapter = netdev_priv(netdev);
4744         struct e1000_hw *hw = &adapter->hw;
4745         struct mii_ioctl_data *data = if_mii(ifr);
4746         int retval;
4747         u16 mii_reg;
4748         unsigned long flags;
4749
4750         if (hw->media_type != e1000_media_type_copper)
4751                 return -EOPNOTSUPP;
4752
4753         switch (cmd) {
4754         case SIOCGMIIPHY:
4755                 data->phy_id = hw->phy_addr;
4756                 break;
4757         case SIOCGMIIREG:
4758                 spin_lock_irqsave(&adapter->stats_lock, flags);
4759                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4760                                    &data->val_out)) {
4761                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4762                         return -EIO;
4763                 }
4764                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4765                 break;
4766         case SIOCSMIIREG:
4767                 if (data->reg_num & ~(0x1F))
4768                         return -EFAULT;
4769                 mii_reg = data->val_in;
4770                 spin_lock_irqsave(&adapter->stats_lock, flags);
4771                 if (e1000_write_phy_reg(hw, data->reg_num,
4772                                         mii_reg)) {
4773                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4774                         return -EIO;
4775                 }
4776                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4777                 if (hw->media_type == e1000_media_type_copper) {
4778                         switch (data->reg_num) {
4779                         case PHY_CTRL:
4780                                 if (mii_reg & MII_CR_POWER_DOWN)
4781                                         break;
4782                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4783                                         hw->autoneg = 1;
4784                                         hw->autoneg_advertised = 0x2F;
4785                                 } else {
4786                                         u32 speed;
4787                                         if (mii_reg & 0x40)
4788                                                 speed = SPEED_1000;
4789                                         else if (mii_reg & 0x2000)
4790                                                 speed = SPEED_100;
4791                                         else
4792                                                 speed = SPEED_10;
4793                                         retval = e1000_set_spd_dplx(
4794                                                 adapter, speed,
4795                                                 ((mii_reg & 0x100)
4796                                                  ? DUPLEX_FULL :
4797                                                  DUPLEX_HALF));
4798                                         if (retval)
4799                                                 return retval;
4800                                 }
4801                                 if (netif_running(adapter->netdev))
4802                                         e1000_reinit_locked(adapter);
4803                                 else
4804                                         e1000_reset(adapter);
4805                                 break;
4806                         case M88E1000_PHY_SPEC_CTRL:
4807                         case M88E1000_EXT_PHY_SPEC_CTRL:
4808                                 if (e1000_phy_reset(hw))
4809                                         return -EIO;
4810                                 break;
4811                         }
4812                 } else {
4813                         switch (data->reg_num) {
4814                         case PHY_CTRL:
4815                                 if (mii_reg & MII_CR_POWER_DOWN)
4816                                         break;
4817                                 if (netif_running(adapter->netdev))
4818                                         e1000_reinit_locked(adapter);
4819                                 else
4820                                         e1000_reset(adapter);
4821                                 break;
4822                         }
4823                 }
4824                 break;
4825         default:
4826                 return -EOPNOTSUPP;
4827         }
4828         return E1000_SUCCESS;
4829 }
4830
4831 void e1000_pci_set_mwi(struct e1000_hw *hw)
4832 {
4833         struct e1000_adapter *adapter = hw->back;
4834         int ret_val = pci_set_mwi(adapter->pdev);
4835
4836         if (ret_val)
4837                 e_err(probe, "Error in setting MWI\n");
4838 }
4839
4840 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4841 {
4842         struct e1000_adapter *adapter = hw->back;
4843
4844         pci_clear_mwi(adapter->pdev);
4845 }
4846
4847 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4848 {
4849         struct e1000_adapter *adapter = hw->back;
4850         return pcix_get_mmrbc(adapter->pdev);
4851 }
4852
4853 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4854 {
4855         struct e1000_adapter *adapter = hw->back;
4856         pcix_set_mmrbc(adapter->pdev, mmrbc);
4857 }
4858
4859 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4860 {
4861         outl(value, port);
4862 }
4863
4864 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4865 {
4866         u16 vid;
4867
4868         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4869                 return true;
4870         return false;
4871 }
4872
4873 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4874                               netdev_features_t features)
4875 {
4876         struct e1000_hw *hw = &adapter->hw;
4877         u32 ctrl;
4878
4879         ctrl = er32(CTRL);
4880         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4881                 /* enable VLAN tag insert/strip */
4882                 ctrl |= E1000_CTRL_VME;
4883         } else {
4884                 /* disable VLAN tag insert/strip */
4885                 ctrl &= ~E1000_CTRL_VME;
4886         }
4887         ew32(CTRL, ctrl);
4888 }
4889 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4890                                      bool filter_on)
4891 {
4892         struct e1000_hw *hw = &adapter->hw;
4893         u32 rctl;
4894
4895         if (!test_bit(__E1000_DOWN, &adapter->flags))
4896                 e1000_irq_disable(adapter);
4897
4898         __e1000_vlan_mode(adapter, adapter->netdev->features);
4899         if (filter_on) {
4900                 /* enable VLAN receive filtering */
4901                 rctl = er32(RCTL);
4902                 rctl &= ~E1000_RCTL_CFIEN;
4903                 if (!(adapter->netdev->flags & IFF_PROMISC))
4904                         rctl |= E1000_RCTL_VFE;
4905                 ew32(RCTL, rctl);
4906                 e1000_update_mng_vlan(adapter);
4907         } else {
4908                 /* disable VLAN receive filtering */
4909                 rctl = er32(RCTL);
4910                 rctl &= ~E1000_RCTL_VFE;
4911                 ew32(RCTL, rctl);
4912         }
4913
4914         if (!test_bit(__E1000_DOWN, &adapter->flags))
4915                 e1000_irq_enable(adapter);
4916 }
4917
4918 static void e1000_vlan_mode(struct net_device *netdev,
4919                             netdev_features_t features)
4920 {
4921         struct e1000_adapter *adapter = netdev_priv(netdev);
4922
4923         if (!test_bit(__E1000_DOWN, &adapter->flags))
4924                 e1000_irq_disable(adapter);
4925
4926         __e1000_vlan_mode(adapter, features);
4927
4928         if (!test_bit(__E1000_DOWN, &adapter->flags))
4929                 e1000_irq_enable(adapter);
4930 }
4931
4932 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4933                                  __be16 proto, u16 vid)
4934 {
4935         struct e1000_adapter *adapter = netdev_priv(netdev);
4936         struct e1000_hw *hw = &adapter->hw;
4937         u32 vfta, index;
4938
4939         if ((hw->mng_cookie.status &
4940              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4941             (vid == adapter->mng_vlan_id))
4942                 return 0;
4943
4944         if (!e1000_vlan_used(adapter))
4945                 e1000_vlan_filter_on_off(adapter, true);
4946
4947         /* add VID to filter table */
4948         index = (vid >> 5) & 0x7F;
4949         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4950         vfta |= (1 << (vid & 0x1F));
4951         e1000_write_vfta(hw, index, vfta);
4952
4953         set_bit(vid, adapter->active_vlans);
4954
4955         return 0;
4956 }
4957
4958 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4959                                   __be16 proto, u16 vid)
4960 {
4961         struct e1000_adapter *adapter = netdev_priv(netdev);
4962         struct e1000_hw *hw = &adapter->hw;
4963         u32 vfta, index;
4964
4965         if (!test_bit(__E1000_DOWN, &adapter->flags))
4966                 e1000_irq_disable(adapter);
4967         if (!test_bit(__E1000_DOWN, &adapter->flags))
4968                 e1000_irq_enable(adapter);
4969
4970         /* remove VID from filter table */
4971         index = (vid >> 5) & 0x7F;
4972         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4973         vfta &= ~(1 << (vid & 0x1F));
4974         e1000_write_vfta(hw, index, vfta);
4975
4976         clear_bit(vid, adapter->active_vlans);
4977
4978         if (!e1000_vlan_used(adapter))
4979                 e1000_vlan_filter_on_off(adapter, false);
4980
4981         return 0;
4982 }
4983
4984 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4985 {
4986         u16 vid;
4987
4988         if (!e1000_vlan_used(adapter))
4989                 return;
4990
4991         e1000_vlan_filter_on_off(adapter, true);
4992         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4993                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
4994 }
4995
4996 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
4997 {
4998         struct e1000_hw *hw = &adapter->hw;
4999
5000         hw->autoneg = 0;
5001
5002         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5003          * for the switch() below to work
5004          */
5005         if ((spd & 1) || (dplx & ~1))
5006                 goto err_inval;
5007
5008         /* Fiber NICs only allow 1000 gbps Full duplex */
5009         if ((hw->media_type == e1000_media_type_fiber) &&
5010             spd != SPEED_1000 &&
5011             dplx != DUPLEX_FULL)
5012                 goto err_inval;
5013
5014         switch (spd + dplx) {
5015         case SPEED_10 + DUPLEX_HALF:
5016                 hw->forced_speed_duplex = e1000_10_half;
5017                 break;
5018         case SPEED_10 + DUPLEX_FULL:
5019                 hw->forced_speed_duplex = e1000_10_full;
5020                 break;
5021         case SPEED_100 + DUPLEX_HALF:
5022                 hw->forced_speed_duplex = e1000_100_half;
5023                 break;
5024         case SPEED_100 + DUPLEX_FULL:
5025                 hw->forced_speed_duplex = e1000_100_full;
5026                 break;
5027         case SPEED_1000 + DUPLEX_FULL:
5028                 hw->autoneg = 1;
5029                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5030                 break;
5031         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5032         default:
5033                 goto err_inval;
5034         }
5035
5036         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5037         hw->mdix = AUTO_ALL_MODES;
5038
5039         return 0;
5040
5041 err_inval:
5042         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5043         return -EINVAL;
5044 }
5045
5046 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5047 {
5048         struct net_device *netdev = pci_get_drvdata(pdev);
5049         struct e1000_adapter *adapter = netdev_priv(netdev);
5050         struct e1000_hw *hw = &adapter->hw;
5051         u32 ctrl, ctrl_ext, rctl, status;
5052         u32 wufc = adapter->wol;
5053 #ifdef CONFIG_PM
5054         int retval = 0;
5055 #endif
5056
5057         netif_device_detach(netdev);
5058
5059         if (netif_running(netdev)) {
5060                 int count = E1000_CHECK_RESET_COUNT;
5061
5062                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5063                         usleep_range(10000, 20000);
5064
5065                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5066                 e1000_down(adapter);
5067         }
5068
5069 #ifdef CONFIG_PM
5070         retval = pci_save_state(pdev);
5071         if (retval)
5072                 return retval;
5073 #endif
5074
5075         status = er32(STATUS);
5076         if (status & E1000_STATUS_LU)
5077                 wufc &= ~E1000_WUFC_LNKC;
5078
5079         if (wufc) {
5080                 e1000_setup_rctl(adapter);
5081                 e1000_set_rx_mode(netdev);
5082
5083                 rctl = er32(RCTL);
5084
5085                 /* turn on all-multi mode if wake on multicast is enabled */
5086                 if (wufc & E1000_WUFC_MC)
5087                         rctl |= E1000_RCTL_MPE;
5088
5089                 /* enable receives in the hardware */
5090                 ew32(RCTL, rctl | E1000_RCTL_EN);
5091
5092                 if (hw->mac_type >= e1000_82540) {
5093                         ctrl = er32(CTRL);
5094                         /* advertise wake from D3Cold */
5095                         #define E1000_CTRL_ADVD3WUC 0x00100000
5096                         /* phy power management enable */
5097                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5098                         ctrl |= E1000_CTRL_ADVD3WUC |
5099                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5100                         ew32(CTRL, ctrl);
5101                 }
5102
5103                 if (hw->media_type == e1000_media_type_fiber ||
5104                     hw->media_type == e1000_media_type_internal_serdes) {
5105                         /* keep the laser running in D3 */
5106                         ctrl_ext = er32(CTRL_EXT);
5107                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5108                         ew32(CTRL_EXT, ctrl_ext);
5109                 }
5110
5111                 ew32(WUC, E1000_WUC_PME_EN);
5112                 ew32(WUFC, wufc);
5113         } else {
5114                 ew32(WUC, 0);
5115                 ew32(WUFC, 0);
5116         }
5117
5118         e1000_release_manageability(adapter);
5119
5120         *enable_wake = !!wufc;
5121
5122         /* make sure adapter isn't asleep if manageability is enabled */
5123         if (adapter->en_mng_pt)
5124                 *enable_wake = true;
5125
5126         if (netif_running(netdev))
5127                 e1000_free_irq(adapter);
5128
5129         pci_disable_device(pdev);
5130
5131         return 0;
5132 }
5133
5134 #ifdef CONFIG_PM
5135 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
5136 {
5137         int retval;
5138         bool wake;
5139
5140         retval = __e1000_shutdown(pdev, &wake);
5141         if (retval)
5142                 return retval;
5143
5144         if (wake) {
5145                 pci_prepare_to_sleep(pdev);
5146         } else {
5147                 pci_wake_from_d3(pdev, false);
5148                 pci_set_power_state(pdev, PCI_D3hot);
5149         }
5150
5151         return 0;
5152 }
5153
5154 static int e1000_resume(struct pci_dev *pdev)
5155 {
5156         struct net_device *netdev = pci_get_drvdata(pdev);
5157         struct e1000_adapter *adapter = netdev_priv(netdev);
5158         struct e1000_hw *hw = &adapter->hw;
5159         u32 err;
5160
5161         pci_set_power_state(pdev, PCI_D0);
5162         pci_restore_state(pdev);
5163         pci_save_state(pdev);
5164
5165         if (adapter->need_ioport)
5166                 err = pci_enable_device(pdev);
5167         else
5168                 err = pci_enable_device_mem(pdev);
5169         if (err) {
5170                 pr_err("Cannot enable PCI device from suspend\n");
5171                 return err;
5172         }
5173         pci_set_master(pdev);
5174
5175         pci_enable_wake(pdev, PCI_D3hot, 0);
5176         pci_enable_wake(pdev, PCI_D3cold, 0);
5177
5178         if (netif_running(netdev)) {
5179                 err = e1000_request_irq(adapter);
5180                 if (err)
5181                         return err;
5182         }
5183
5184         e1000_power_up_phy(adapter);
5185         e1000_reset(adapter);
5186         ew32(WUS, ~0);
5187
5188         e1000_init_manageability(adapter);
5189
5190         if (netif_running(netdev))
5191                 e1000_up(adapter);
5192
5193         netif_device_attach(netdev);
5194
5195         return 0;
5196 }
5197 #endif
5198
5199 static void e1000_shutdown(struct pci_dev *pdev)
5200 {
5201         bool wake;
5202
5203         __e1000_shutdown(pdev, &wake);
5204
5205         if (system_state == SYSTEM_POWER_OFF) {
5206                 pci_wake_from_d3(pdev, wake);
5207                 pci_set_power_state(pdev, PCI_D3hot);
5208         }
5209 }
5210
5211 #ifdef CONFIG_NET_POLL_CONTROLLER
5212 /* Polling 'interrupt' - used by things like netconsole to send skbs
5213  * without having to re-enable interrupts. It's not called while
5214  * the interrupt routine is executing.
5215  */
5216 static void e1000_netpoll(struct net_device *netdev)
5217 {
5218         struct e1000_adapter *adapter = netdev_priv(netdev);
5219
5220         disable_irq(adapter->pdev->irq);
5221         e1000_intr(adapter->pdev->irq, netdev);
5222         enable_irq(adapter->pdev->irq);
5223 }
5224 #endif
5225
5226 /**
5227  * e1000_io_error_detected - called when PCI error is detected
5228  * @pdev: Pointer to PCI device
5229  * @state: The current pci connection state
5230  *
5231  * This function is called after a PCI bus error affecting
5232  * this device has been detected.
5233  */
5234 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5235                                                 pci_channel_state_t state)
5236 {
5237         struct net_device *netdev = pci_get_drvdata(pdev);
5238         struct e1000_adapter *adapter = netdev_priv(netdev);
5239
5240         netif_device_detach(netdev);
5241
5242         if (state == pci_channel_io_perm_failure)
5243                 return PCI_ERS_RESULT_DISCONNECT;
5244
5245         if (netif_running(netdev))
5246                 e1000_down(adapter);
5247         pci_disable_device(pdev);
5248
5249         /* Request a slot slot reset. */
5250         return PCI_ERS_RESULT_NEED_RESET;
5251 }
5252
5253 /**
5254  * e1000_io_slot_reset - called after the pci bus has been reset.
5255  * @pdev: Pointer to PCI device
5256  *
5257  * Restart the card from scratch, as if from a cold-boot. Implementation
5258  * resembles the first-half of the e1000_resume routine.
5259  */
5260 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5261 {
5262         struct net_device *netdev = pci_get_drvdata(pdev);
5263         struct e1000_adapter *adapter = netdev_priv(netdev);
5264         struct e1000_hw *hw = &adapter->hw;
5265         int err;
5266
5267         if (adapter->need_ioport)
5268                 err = pci_enable_device(pdev);
5269         else
5270                 err = pci_enable_device_mem(pdev);
5271         if (err) {
5272                 pr_err("Cannot re-enable PCI device after reset.\n");
5273                 return PCI_ERS_RESULT_DISCONNECT;
5274         }
5275         pci_set_master(pdev);
5276
5277         pci_enable_wake(pdev, PCI_D3hot, 0);
5278         pci_enable_wake(pdev, PCI_D3cold, 0);
5279
5280         e1000_reset(adapter);
5281         ew32(WUS, ~0);
5282
5283         return PCI_ERS_RESULT_RECOVERED;
5284 }
5285
5286 /**
5287  * e1000_io_resume - called when traffic can start flowing again.
5288  * @pdev: Pointer to PCI device
5289  *
5290  * This callback is called when the error recovery driver tells us that
5291  * its OK to resume normal operation. Implementation resembles the
5292  * second-half of the e1000_resume routine.
5293  */
5294 static void e1000_io_resume(struct pci_dev *pdev)
5295 {
5296         struct net_device *netdev = pci_get_drvdata(pdev);
5297         struct e1000_adapter *adapter = netdev_priv(netdev);
5298
5299         e1000_init_manageability(adapter);
5300
5301         if (netif_running(netdev)) {
5302                 if (e1000_up(adapter)) {
5303                         pr_info("can't bring device back up after reset\n");
5304                         return;
5305                 }
5306         }
5307
5308         netif_device_attach(netdev);
5309 }
5310
5311 /* e1000_main.c */