drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43
  44 #include <asm/system.h>
  45 #include <asm/io.h>
  46 #include <asm/byteorder.h>
  47 #include <asm/uaccess.h>
  48
  49 #ifdef CONFIG_SPARC64
  50 #include <asm/idprom.h>
  51 #include <asm/oplib.h>
  52 #include <asm/pbm.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #ifdef NETIF_F_TSO
  62 #define TG3_TSO_SUPPORT 1
  63 #else
  64 #define TG3_TSO_SUPPORT 0
  65 #endif
  66
  67 #include "tg3.h"
  68
  69 #define DRV_MODULE_NAME         "tg3"
  70 #define PFX DRV_MODULE_NAME     ": "
  71 #define DRV_MODULE_VERSION      "3.60"
  72 #define DRV_MODULE_RELDATE      "June 17, 2006"
  73
  74 #define TG3_DEF_MAC_MODE        0
  75 #define TG3_DEF_RX_MODE         0
  76 #define TG3_DEF_TX_MODE         0
  77 #define TG3_DEF_MSG_ENABLE        \
  78         (NETIF_MSG_DRV          | \
  79          NETIF_MSG_PROBE        | \
  80          NETIF_MSG_LINK         | \
  81          NETIF_MSG_TIMER        | \
  82          NETIF_MSG_IFDOWN       | \
  83          NETIF_MSG_IFUP         | \
  84          NETIF_MSG_RX_ERR       | \
  85          NETIF_MSG_TX_ERR)
  86
  87 /* length of time before we decide the hardware is borked,
  88  * and dev->tx_timeout() should be called to fix the problem
  89  */
  90 #define TG3_TX_TIMEOUT                  (5 * HZ)
  91
  92 /* hardware minimum and maximum for a single frame's data payload */
  93 #define TG3_MIN_MTU                     60
  94 #define TG3_MAX_MTU(tp) \
  95         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  96
  97 /* These numbers seem to be hard coded in the NIC firmware somehow.
  98  * You can't change the ring sizes, but you can change where you place
  99  * them in the NIC onboard memory.
 100  */
 101 #define TG3_RX_RING_SIZE                512
 102 #define TG3_DEF_RX_RING_PENDING         200
 103 #define TG3_RX_JUMBO_RING_SIZE          256
 104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 105
 106 /* Do not place this n-ring entries value into the tp struct itself,
 107  * we really want to expose these constants to GCC so that modulo et
 108  * al.  operations are done with shifts and masks instead of with
 109  * hw multiply/modulo instructions.  Another solution would be to
 110  * replace things like '% foo' with '& (foo - 1)'.
 111  */
 112 #define TG3_RX_RCB_RING_SIZE(tp)        \
 113         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 114
 115 #define TG3_TX_RING_SIZE                512
 116 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 117
 118 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 119                                  TG3_RX_RING_SIZE)
 120 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 121                                  TG3_RX_JUMBO_RING_SIZE)
 122 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 123                                    TG3_RX_RCB_RING_SIZE(tp))
 124 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 125                                  TG3_TX_RING_SIZE)
 126 #define TX_BUFFS_AVAIL(TP)                                              \
 127         ((TP)->tx_pending -                                             \
 128          (((TP)->tx_prod - (TP)->tx_cons) & (TG3_TX_RING_SIZE - 1)))
 129 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 130
 131 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 132 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 133
 134 /* minimum number of free TX descriptors required to wake up TX process */
 135 #define TG3_TX_WAKEUP_THRESH            (TG3_TX_RING_SIZE / 4)
 136
 137 /* number of ETHTOOL_GSTATS u64's */
 138 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 139
 140 #define TG3_NUM_TEST            6
 141
 142 static char version[] __devinitdata =
 143         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 144
 145 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 146 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 147 MODULE_LICENSE("GPL");
 148 MODULE_VERSION(DRV_MODULE_VERSION);
 149
 150 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 151 module_param(tg3_debug, int, 0);
 152 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 153
 154 static struct pci_device_id tg3_pci_tbl[] = {
 155         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700,
 156           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 157         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701,
 158           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 159         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702,
 160           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 161         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703,
 162           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 163         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704,
 164           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 165         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE,
 166           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 167         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705,
 168           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 169         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2,
 170           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 171         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M,
 172           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 173         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2,
 174           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 175         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X,
 176           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 177         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X,
 178           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 179         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S,
 180           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 181         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3,
 182           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 183         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3,
 184           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 185         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782,
 186           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 187         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788,
 188           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 189         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789,
 190           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 191         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901,
 192           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 193         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2,
 194           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 195         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2,
 196           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 197         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F,
 198           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 199         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720,
 200           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 201         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721,
 202           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 203         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750,
 204           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 205         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751,
 206           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 207         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M,
 208           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 209         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M,
 210           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 211         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F,
 212           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 213         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752,
 214           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 215         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M,
 216           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 217         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753,
 218           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 219         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M,
 220           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 221         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F,
 222           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 223         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754,
 224           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 225         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M,
 226           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 227         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755,
 228           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 229         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M,
 230           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 231         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786,
 232           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 233         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787,
 234           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 235         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M,
 236           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 237         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714,
 238           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 239         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S,
 240           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 241         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715,
 242           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 243         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S,
 244           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 245         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780,
 246           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 247         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S,
 248           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 249         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781,
 250           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 251         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX,
 252           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 253         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX,
 254           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 255         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000,
 256           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 257         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001,
 258           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 259         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003,
 260           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 261         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100,
 262           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 263         { PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3,
 264           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 265         { 0, }
 266 };
 267
 268 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 269
 270 static struct {
 271         const char string[ETH_GSTRING_LEN];
 272 } ethtool_stats_keys[TG3_NUM_STATS] = {
 273         { "rx_octets" },
 274         { "rx_fragments" },
 275         { "rx_ucast_packets" },
 276         { "rx_mcast_packets" },
 277         { "rx_bcast_packets" },
 278         { "rx_fcs_errors" },
 279         { "rx_align_errors" },
 280         { "rx_xon_pause_rcvd" },
 281         { "rx_xoff_pause_rcvd" },
 282         { "rx_mac_ctrl_rcvd" },
 283         { "rx_xoff_entered" },
 284         { "rx_frame_too_long_errors" },
 285         { "rx_jabbers" },
 286         { "rx_undersize_packets" },
 287         { "rx_in_length_errors" },
 288         { "rx_out_length_errors" },
 289         { "rx_64_or_less_octet_packets" },
 290         { "rx_65_to_127_octet_packets" },
 291         { "rx_128_to_255_octet_packets" },
 292         { "rx_256_to_511_octet_packets" },
 293         { "rx_512_to_1023_octet_packets" },
 294         { "rx_1024_to_1522_octet_packets" },
 295         { "rx_1523_to_2047_octet_packets" },
 296         { "rx_2048_to_4095_octet_packets" },
 297         { "rx_4096_to_8191_octet_packets" },
 298         { "rx_8192_to_9022_octet_packets" },
 299
 300         { "tx_octets" },
 301         { "tx_collisions" },
 302
 303         { "tx_xon_sent" },
 304         { "tx_xoff_sent" },
 305         { "tx_flow_control" },
 306         { "tx_mac_errors" },
 307         { "tx_single_collisions" },
 308         { "tx_mult_collisions" },
 309         { "tx_deferred" },
 310         { "tx_excessive_collisions" },
 311         { "tx_late_collisions" },
 312         { "tx_collide_2times" },
 313         { "tx_collide_3times" },
 314         { "tx_collide_4times" },
 315         { "tx_collide_5times" },
 316         { "tx_collide_6times" },
 317         { "tx_collide_7times" },
 318         { "tx_collide_8times" },
 319         { "tx_collide_9times" },
 320         { "tx_collide_10times" },
 321         { "tx_collide_11times" },
 322         { "tx_collide_12times" },
 323         { "tx_collide_13times" },
 324         { "tx_collide_14times" },
 325         { "tx_collide_15times" },
 326         { "tx_ucast_packets" },
 327         { "tx_mcast_packets" },
 328         { "tx_bcast_packets" },
 329         { "tx_carrier_sense_errors" },
 330         { "tx_discards" },
 331         { "tx_errors" },
 332
 333         { "dma_writeq_full" },
 334         { "dma_write_prioq_full" },
 335         { "rxbds_empty" },
 336         { "rx_discards" },
 337         { "rx_errors" },
 338         { "rx_threshold_hit" },
 339
 340         { "dma_readq_full" },
 341         { "dma_read_prioq_full" },
 342         { "tx_comp_queue_full" },
 343
 344         { "ring_set_send_prod_index" },
 345         { "ring_status_update" },
 346         { "nic_irqs" },
 347         { "nic_avoided_irqs" },
 348         { "nic_tx_threshold_hit" }
 349 };
 350
 351 static struct {
 352         const char string[ETH_GSTRING_LEN];
 353 } ethtool_test_keys[TG3_NUM_TEST] = {
 354         { "nvram test     (online) " },
 355         { "link test      (online) " },
 356         { "register test  (offline)" },
 357         { "memory test    (offline)" },
 358         { "loopback test  (offline)" },
 359         { "interrupt test (offline)" },
 360 };
 361
 362 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 363 {
 364         writel(val, tp->regs + off);
 365 }
 366
 367 static u32 tg3_read32(struct tg3 *tp, u32 off)
 368 {
 369         return (readl(tp->regs + off));
 370 }
 371
 372 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 373 {
 374         unsigned long flags;
 375
 376         spin_lock_irqsave(&tp->indirect_lock, flags);
 377         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 378         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 379         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 380 }
 381
 382 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 383 {
 384         writel(val, tp->regs + off);
 385         readl(tp->regs + off);
 386 }
 387
 388 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 389 {
 390         unsigned long flags;
 391         u32 val;
 392
 393         spin_lock_irqsave(&tp->indirect_lock, flags);
 394         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 395         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 396         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 397         return val;
 398 }
 399
 400 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 401 {
 402         unsigned long flags;
 403
 404         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 405                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 406                                        TG3_64BIT_REG_LOW, val);
 407                 return;
 408         }
 409         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 410                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 411                                        TG3_64BIT_REG_LOW, val);
 412                 return;
 413         }
 414
 415         spin_lock_irqsave(&tp->indirect_lock, flags);
 416         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 417         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 418         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 419
 420         /* In indirect mode when disabling interrupts, we also need
 421          * to clear the interrupt bit in the GRC local ctrl register.
 422          */
 423         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 424             (val == 0x1)) {
 425                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 426                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 427         }
 428 }
 429
 430 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 431 {
 432         unsigned long flags;
 433         u32 val;
 434
 435         spin_lock_irqsave(&tp->indirect_lock, flags);
 436         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 437         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 438         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 439         return val;
 440 }
 441
 442 /* usec_wait specifies the wait time in usec when writing to certain registers
 443  * where it is unsafe to read back the register without some delay.
 444  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 445  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 446  */
 447 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 448 {
 449         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 450             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 451                 /* Non-posted methods */
 452                 tp->write32(tp, off, val);
 453         else {
 454                 /* Posted method */
 455                 tg3_write32(tp, off, val);
 456                 if (usec_wait)
 457                         udelay(usec_wait);
 458                 tp->read32(tp, off);
 459         }
 460         /* Wait again after the read for the posted method to guarantee that
 461          * the wait time is met.
 462          */
 463         if (usec_wait)
 464                 udelay(usec_wait);
 465 }
 466
 467 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 468 {
 469         tp->write32_mbox(tp, off, val);
 470         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 471             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 472                 tp->read32_mbox(tp, off);
 473 }
 474
 475 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 476 {
 477         void __iomem *mbox = tp->regs + off;
 478         writel(val, mbox);
 479         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 480                 writel(val, mbox);
 481         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 482                 readl(mbox);
 483 }
 484
 485 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 486 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 487 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 488 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 489 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 490
 491 #define tw32(reg,val)           tp->write32(tp, reg, val)
 492 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 493 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 494 #define tr32(reg)               tp->read32(tp, reg)
 495
 496 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 497 {
 498         unsigned long flags;
 499
 500         spin_lock_irqsave(&tp->indirect_lock, flags);
 501         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 502                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 503                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 504
 505                 /* Always leave this as zero. */
 506                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 507         } else {
 508                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 509                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 510
 511                 /* Always leave this as zero. */
 512                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 513         }
 514         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 515 }
 516
 517 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 518 {
 519         unsigned long flags;
 520
 521         spin_lock_irqsave(&tp->indirect_lock, flags);
 522         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 523                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 524                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 525
 526                 /* Always leave this as zero. */
 527                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 528         } else {
 529                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 530                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 531
 532                 /* Always leave this as zero. */
 533                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 534         }
 535         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 536 }
 537
 538 static void tg3_disable_ints(struct tg3 *tp)
 539 {
 540         tw32(TG3PCI_MISC_HOST_CTRL,
 541              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 542         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 543 }
 544
 545 static inline void tg3_cond_int(struct tg3 *tp)
 546 {
 547         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 548             (tp->hw_status->status & SD_STATUS_UPDATED))
 549                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 550 }
 551
 552 static void tg3_enable_ints(struct tg3 *tp)
 553 {
 554         tp->irq_sync = 0;
 555         wmb();
 556
 557         tw32(TG3PCI_MISC_HOST_CTRL,
 558              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 559         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 560                        (tp->last_tag << 24));
 561         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 562                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 563                                (tp->last_tag << 24));
 564         tg3_cond_int(tp);
 565 }
 566
 567 static inline unsigned int tg3_has_work(struct tg3 *tp)
 568 {
 569         struct tg3_hw_status *sblk = tp->hw_status;
 570         unsigned int work_exists = 0;
 571
 572         /* check for phy events */
 573         if (!(tp->tg3_flags &
 574               (TG3_FLAG_USE_LINKCHG_REG |
 575                TG3_FLAG_POLL_SERDES))) {
 576                 if (sblk->status & SD_STATUS_LINK_CHG)
 577                         work_exists = 1;
 578         }
 579         /* check for RX/TX work to do */
 580         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 581             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 582                 work_exists = 1;
 583
 584         return work_exists;
 585 }
 586
 587 /* tg3_restart_ints
 588  *  similar to tg3_enable_ints, but it accurately determines whether there
 589  *  is new work pending and can return without flushing the PIO write
 590  *  which reenables interrupts
 591  */
 592 static void tg3_restart_ints(struct tg3 *tp)
 593 {
 594         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 595                      tp->last_tag << 24);
 596         mmiowb();
 597
 598         /* When doing tagged status, this work check is unnecessary.
 599          * The last_tag we write above tells the chip which piece of
 600          * work we've completed.
 601          */
 602         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 603             tg3_has_work(tp))
 604                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 605                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 606 }
 607
 608 static inline void tg3_netif_stop(struct tg3 *tp)
 609 {
 610         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 611         netif_poll_disable(tp->dev);
 612         netif_tx_disable(tp->dev);
 613 }
 614
 615 static inline void tg3_netif_start(struct tg3 *tp)
 616 {
 617         netif_wake_queue(tp->dev);
 618         /* NOTE: unconditional netif_wake_queue is only appropriate
 619          * so long as all callers are assured to have free tx slots
 620          * (such as after tg3_init_hw)
 621          */
 622         netif_poll_enable(tp->dev);
 623         tp->hw_status->status |= SD_STATUS_UPDATED;
 624         tg3_enable_ints(tp);
 625 }
 626
 627 static void tg3_switch_clocks(struct tg3 *tp)
 628 {
 629         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 630         u32 orig_clock_ctrl;
 631
 632         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
 633                 return;
 634
 635         orig_clock_ctrl = clock_ctrl;
 636         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 637                        CLOCK_CTRL_CLKRUN_OENABLE |
 638                        0x1f);
 639         tp->pci_clock_ctrl = clock_ctrl;
 640
 641         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 642                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 643                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 644                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 645                 }
 646         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 647                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 648                             clock_ctrl |
 649                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 650                             40);
 651                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 652                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 653                             40);
 654         }
 655         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 656 }
 657
 658 #define PHY_BUSY_LOOPS  5000
 659
 660 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 661 {
 662         u32 frame_val;
 663         unsigned int loops;
 664         int ret;
 665
 666         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 667                 tw32_f(MAC_MI_MODE,
 668                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 669                 udelay(80);
 670         }
 671
 672         *val = 0x0;
 673
 674         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 675                       MI_COM_PHY_ADDR_MASK);
 676         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 677                       MI_COM_REG_ADDR_MASK);
 678         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 679
 680         tw32_f(MAC_MI_COM, frame_val);
 681
 682         loops = PHY_BUSY_LOOPS;
 683         while (loops != 0) {
 684                 udelay(10);
 685                 frame_val = tr32(MAC_MI_COM);
 686
 687                 if ((frame_val & MI_COM_BUSY) == 0) {
 688                         udelay(5);
 689                         frame_val = tr32(MAC_MI_COM);
 690                         break;
 691                 }
 692                 loops -= 1;
 693         }
 694
 695         ret = -EBUSY;
 696         if (loops != 0) {
 697                 *val = frame_val & MI_COM_DATA_MASK;
 698                 ret = 0;
 699         }
 700
 701         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 702                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 703                 udelay(80);
 704         }
 705
 706         return ret;
 707 }
 708
 709 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 710 {
 711         u32 frame_val;
 712         unsigned int loops;
 713         int ret;
 714
 715         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 716                 tw32_f(MAC_MI_MODE,
 717                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 718                 udelay(80);
 719         }
 720
 721         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 722                       MI_COM_PHY_ADDR_MASK);
 723         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 724                       MI_COM_REG_ADDR_MASK);
 725         frame_val |= (val & MI_COM_DATA_MASK);
 726         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 727
 728         tw32_f(MAC_MI_COM, frame_val);
 729
 730         loops = PHY_BUSY_LOOPS;
 731         while (loops != 0) {
 732                 udelay(10);
 733                 frame_val = tr32(MAC_MI_COM);
 734                 if ((frame_val & MI_COM_BUSY) == 0) {
 735                         udelay(5);
 736                         frame_val = tr32(MAC_MI_COM);
 737                         break;
 738                 }
 739                 loops -= 1;
 740         }
 741
 742         ret = -EBUSY;
 743         if (loops != 0)
 744                 ret = 0;
 745
 746         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 747                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 748                 udelay(80);
 749         }
 750
 751         return ret;
 752 }
 753
 754 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 755 {
 756         u32 val;
 757
 758         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 759                 return;
 760
 761         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 762             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 763                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 764                              (val | (1 << 15) | (1 << 4)));
 765 }
 766
 767 static int tg3_bmcr_reset(struct tg3 *tp)
 768 {
 769         u32 phy_control;
 770         int limit, err;
 771
 772         /* OK, reset it, and poll the BMCR_RESET bit until it
 773          * clears or we time out.
 774          */
 775         phy_control = BMCR_RESET;
 776         err = tg3_writephy(tp, MII_BMCR, phy_control);
 777         if (err != 0)
 778                 return -EBUSY;
 779
 780         limit = 5000;
 781         while (limit--) {
 782                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 783                 if (err != 0)
 784                         return -EBUSY;
 785
 786                 if ((phy_control & BMCR_RESET) == 0) {
 787                         udelay(40);
 788                         break;
 789                 }
 790                 udelay(10);
 791         }
 792         if (limit <= 0)
 793                 return -EBUSY;
 794
 795         return 0;
 796 }
 797
 798 static int tg3_wait_macro_done(struct tg3 *tp)
 799 {
 800         int limit = 100;
 801
 802         while (limit--) {
 803                 u32 tmp32;
 804
 805                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 806                         if ((tmp32 & 0x1000) == 0)
 807                                 break;
 808                 }
 809         }
 810         if (limit <= 0)
 811                 return -EBUSY;
 812
 813         return 0;
 814 }
 815
 816 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 817 {
 818         static const u32 test_pat[4][6] = {
 819         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 820         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 821         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 822         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 823         };
 824         int chan;
 825
 826         for (chan = 0; chan < 4; chan++) {
 827                 int i;
 828
 829                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 830                              (chan * 0x2000) | 0x0200);
 831                 tg3_writephy(tp, 0x16, 0x0002);
 832
 833                 for (i = 0; i < 6; i++)
 834                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 835                                      test_pat[chan][i]);
 836
 837                 tg3_writephy(tp, 0x16, 0x0202);
 838                 if (tg3_wait_macro_done(tp)) {
 839                         *resetp = 1;
 840                         return -EBUSY;
 841                 }
 842
 843                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 844                              (chan * 0x2000) | 0x0200);
 845                 tg3_writephy(tp, 0x16, 0x0082);
 846                 if (tg3_wait_macro_done(tp)) {
 847                         *resetp = 1;
 848                         return -EBUSY;
 849                 }
 850
 851                 tg3_writephy(tp, 0x16, 0x0802);
 852                 if (tg3_wait_macro_done(tp)) {
 853                         *resetp = 1;
 854                         return -EBUSY;
 855                 }
 856
 857                 for (i = 0; i < 6; i += 2) {
 858                         u32 low, high;
 859
 860                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
 861                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
 862                             tg3_wait_macro_done(tp)) {
 863                                 *resetp = 1;
 864                                 return -EBUSY;
 865                         }
 866                         low &= 0x7fff;
 867                         high &= 0x000f;
 868                         if (low != test_pat[chan][i] ||
 869                             high != test_pat[chan][i+1]) {
 870                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
 871                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
 872                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
 873
 874                                 return -EBUSY;
 875                         }
 876                 }
 877         }
 878
 879         return 0;
 880 }
 881
 882 static int tg3_phy_reset_chanpat(struct tg3 *tp)
 883 {
 884         int chan;
 885
 886         for (chan = 0; chan < 4; chan++) {
 887                 int i;
 888
 889                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 890                              (chan * 0x2000) | 0x0200);
 891                 tg3_writephy(tp, 0x16, 0x0002);
 892                 for (i = 0; i < 6; i++)
 893                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
 894                 tg3_writephy(tp, 0x16, 0x0202);
 895                 if (tg3_wait_macro_done(tp))
 896                         return -EBUSY;
 897         }
 898
 899         return 0;
 900 }
 901
 902 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
 903 {
 904         u32 reg32, phy9_orig;
 905         int retries, do_phy_reset, err;
 906
 907         retries = 10;
 908         do_phy_reset = 1;
 909         do {
 910                 if (do_phy_reset) {
 911                         err = tg3_bmcr_reset(tp);
 912                         if (err)
 913                                 return err;
 914                         do_phy_reset = 0;
 915                 }
 916
 917                 /* Disable transmitter and interrupt.  */
 918                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
 919                         continue;
 920
 921                 reg32 |= 0x3000;
 922                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 923
 924                 /* Set full-duplex, 1000 mbps.  */
 925                 tg3_writephy(tp, MII_BMCR,
 926                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
 927
 928                 /* Set to master mode.  */
 929                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
 930                         continue;
 931
 932                 tg3_writephy(tp, MII_TG3_CTRL,
 933                              (MII_TG3_CTRL_AS_MASTER |
 934                               MII_TG3_CTRL_ENABLE_AS_MASTER));
 935
 936                 /* Enable SM_DSP_CLOCK and 6dB.  */
 937                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
 938
 939                 /* Block the PHY control access.  */
 940                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 941                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
 942
 943                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
 944                 if (!err)
 945                         break;
 946         } while (--retries);
 947
 948         err = tg3_phy_reset_chanpat(tp);
 949         if (err)
 950                 return err;
 951
 952         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 953         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
 954
 955         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
 956         tg3_writephy(tp, 0x16, 0x0000);
 957
 958         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 959             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
 960                 /* Set Extended packet length bit for jumbo frames */
 961                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
 962         }
 963         else {
 964                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
 965         }
 966
 967         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
 968
 969         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
 970                 reg32 &= ~0x3000;
 971                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 972         } else if (!err)
 973                 err = -EBUSY;
 974
 975         return err;
 976 }
 977
 978 static void tg3_link_report(struct tg3 *);
 979
 980 /* This will reset the tigon3 PHY if there is no valid
 981  * link unless the FORCE argument is non-zero.
 982  */
 983 static int tg3_phy_reset(struct tg3 *tp)
 984 {
 985         u32 phy_status;
 986         int err;
 987
 988         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
 989         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
 990         if (err != 0)
 991                 return -EBUSY;
 992
 993         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
 994                 netif_carrier_off(tp->dev);
 995                 tg3_link_report(tp);
 996         }
 997
 998         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 999             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1000             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1001                 err = tg3_phy_reset_5703_4_5(tp);
1002                 if (err)
1003                         return err;
1004                 goto out;
1005         }
1006
1007         err = tg3_bmcr_reset(tp);
1008         if (err)
1009                 return err;
1010
1011 out:
1012         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1013                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1014                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1015                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1016                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1017                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1018                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1019         }
1020         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1021                 tg3_writephy(tp, 0x1c, 0x8d68);
1022                 tg3_writephy(tp, 0x1c, 0x8d68);
1023         }
1024         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1025                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1026                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1027                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1028                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1029                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1030                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1031                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1032                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1033         }
1034         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1035                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1036                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1037                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1038                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1039         }
1040         /* Set Extended packet length bit (bit 14) on all chips that */
1041         /* support jumbo frames */
1042         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1043                 /* Cannot do read-modify-write on 5401 */
1044                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1045         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1046                 u32 phy_reg;
1047
1048                 /* Set bit 14 with read-modify-write to preserve other bits */
1049                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1050                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1051                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1052         }
1053
1054         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1055          * jumbo frames transmission.
1056          */
1057         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1058                 u32 phy_reg;
1059
1060                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1061                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1062                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1063         }
1064
1065         tg3_phy_set_wirespeed(tp);
1066         return 0;
1067 }
1068
1069 static void tg3_frob_aux_power(struct tg3 *tp)
1070 {
1071         struct tg3 *tp_peer = tp;
1072
1073         if ((tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) != 0)
1074                 return;
1075
1076         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1077             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1078                 struct net_device *dev_peer;
1079
1080                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1081                 /* remove_one() may have been run on the peer. */
1082                 if (!dev_peer)
1083                         tp_peer = tp;
1084                 else
1085                         tp_peer = netdev_priv(dev_peer);
1086         }
1087
1088         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1089             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1090             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1091             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1092                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1093                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1094                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1095                                     (GRC_LCLCTRL_GPIO_OE0 |
1096                                      GRC_LCLCTRL_GPIO_OE1 |
1097                                      GRC_LCLCTRL_GPIO_OE2 |
1098                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1099                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1100                                     100);
1101                 } else {
1102                         u32 no_gpio2;
1103                         u32 grc_local_ctrl = 0;
1104
1105                         if (tp_peer != tp &&
1106                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1107                                 return;
1108
1109                         /* Workaround to prevent overdrawing Amps. */
1110                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1111                             ASIC_REV_5714) {
1112                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1113                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1114                                             grc_local_ctrl, 100);
1115                         }
1116
1117                         /* On 5753 and variants, GPIO2 cannot be used. */
1118                         no_gpio2 = tp->nic_sram_data_cfg &
1119                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1120
1121                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1122                                          GRC_LCLCTRL_GPIO_OE1 |
1123                                          GRC_LCLCTRL_GPIO_OE2 |
1124                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1125                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1126                         if (no_gpio2) {
1127                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1128                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1129                         }
1130                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1131                                                     grc_local_ctrl, 100);
1132
1133                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1134
1135                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1136                                                     grc_local_ctrl, 100);
1137
1138                         if (!no_gpio2) {
1139                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1140                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1141                                             grc_local_ctrl, 100);
1142                         }
1143                 }
1144         } else {
1145                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1146                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1147                         if (tp_peer != tp &&
1148                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1149                                 return;
1150
1151                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1152                                     (GRC_LCLCTRL_GPIO_OE1 |
1153                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1154
1155                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1156                                     GRC_LCLCTRL_GPIO_OE1, 100);
1157
1158                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1159                                     (GRC_LCLCTRL_GPIO_OE1 |
1160                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1161                 }
1162         }
1163 }
1164
1165 static int tg3_setup_phy(struct tg3 *, int);
1166
1167 #define RESET_KIND_SHUTDOWN     0
1168 #define RESET_KIND_INIT         1
1169 #define RESET_KIND_SUSPEND      2
1170
1171 static void tg3_write_sig_post_reset(struct tg3 *, int);
1172 static int tg3_halt_cpu(struct tg3 *, u32);
1173 static int tg3_nvram_lock(struct tg3 *);
1174 static void tg3_nvram_unlock(struct tg3 *);
1175
1176 static void tg3_power_down_phy(struct tg3 *tp)
1177 {
1178         /* The PHY should not be powered down on some chips because
1179          * of bugs.
1180          */
1181         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1182             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1183             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1184              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1185                 return;
1186         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1187 }
1188
1189 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1190 {
1191         u32 misc_host_ctrl;
1192         u16 power_control, power_caps;
1193         int pm = tp->pm_cap;
1194
1195         /* Make sure register accesses (indirect or otherwise)
1196          * will function correctly.
1197          */
1198         pci_write_config_dword(tp->pdev,
1199                                TG3PCI_MISC_HOST_CTRL,
1200                                tp->misc_host_ctrl);
1201
1202         pci_read_config_word(tp->pdev,
1203                              pm + PCI_PM_CTRL,
1204                              &power_control);
1205         power_control |= PCI_PM_CTRL_PME_STATUS;
1206         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1207         switch (state) {
1208         case PCI_D0:
1209                 power_control |= 0;
1210                 pci_write_config_word(tp->pdev,
1211                                       pm + PCI_PM_CTRL,
1212                                       power_control);
1213                 udelay(100);    /* Delay after power state change */
1214
1215                 /* Switch out of Vaux if it is not a LOM */
1216                 if (!(tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
1217                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1218
1219                 return 0;
1220
1221         case PCI_D1:
1222                 power_control |= 1;
1223                 break;
1224
1225         case PCI_D2:
1226                 power_control |= 2;
1227                 break;
1228
1229         case PCI_D3hot:
1230                 power_control |= 3;
1231                 break;
1232
1233         default:
1234                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1235                        "requested.\n",
1236                        tp->dev->name, state);
1237                 return -EINVAL;
1238         };
1239
1240         power_control |= PCI_PM_CTRL_PME_ENABLE;
1241
1242         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1243         tw32(TG3PCI_MISC_HOST_CTRL,
1244              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1245
1246         if (tp->link_config.phy_is_low_power == 0) {
1247                 tp->link_config.phy_is_low_power = 1;
1248                 tp->link_config.orig_speed = tp->link_config.speed;
1249                 tp->link_config.orig_duplex = tp->link_config.duplex;
1250                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1251         }
1252
1253         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1254                 tp->link_config.speed = SPEED_10;
1255                 tp->link_config.duplex = DUPLEX_HALF;
1256                 tp->link_config.autoneg = AUTONEG_ENABLE;
1257                 tg3_setup_phy(tp, 0);
1258         }
1259
1260         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1261                 int i;
1262                 u32 val;
1263
1264                 for (i = 0; i < 200; i++) {
1265                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1266                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1267                                 break;
1268                         msleep(1);
1269                 }
1270         }
1271         tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1272                                              WOL_DRV_STATE_SHUTDOWN |
1273                                              WOL_DRV_WOL | WOL_SET_MAGIC_PKT);
1274
1275         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1276
1277         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1278                 u32 mac_mode;
1279
1280                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1281                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1282                         udelay(40);
1283
1284                         mac_mode = MAC_MODE_PORT_MODE_MII;
1285
1286                         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 ||
1287                             !(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB))
1288                                 mac_mode |= MAC_MODE_LINK_POLARITY;
1289                 } else {
1290                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1291                 }
1292
1293                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1294                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1295
1296                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1297                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1298                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1299
1300                 tw32_f(MAC_MODE, mac_mode);
1301                 udelay(100);
1302
1303                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1304                 udelay(10);
1305         }
1306
1307         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1308             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1309              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1310                 u32 base_val;
1311
1312                 base_val = tp->pci_clock_ctrl;
1313                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1314                              CLOCK_CTRL_TXCLK_DISABLE);
1315
1316                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1317                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1318         } else if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
1319                 /* do nothing */
1320         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1321                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1322                 u32 newbits1, newbits2;
1323
1324                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1325                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1326                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1327                                     CLOCK_CTRL_TXCLK_DISABLE |
1328                                     CLOCK_CTRL_ALTCLK);
1329                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1330                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1331                         newbits1 = CLOCK_CTRL_625_CORE;
1332                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1333                 } else {
1334                         newbits1 = CLOCK_CTRL_ALTCLK;
1335                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1336                 }
1337
1338                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1339                             40);
1340
1341                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1342                             40);
1343
1344                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1345                         u32 newbits3;
1346
1347                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1348                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1349                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1350                                             CLOCK_CTRL_TXCLK_DISABLE |
1351                                             CLOCK_CTRL_44MHZ_CORE);
1352                         } else {
1353                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1354                         }
1355
1356                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1357                                     tp->pci_clock_ctrl | newbits3, 40);
1358                 }
1359         }
1360
1361         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1362             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1363                 /* Turn off the PHY */
1364                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1365                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1366                                      MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1367                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1368                         tg3_power_down_phy(tp);
1369                 }
1370         }
1371
1372         tg3_frob_aux_power(tp);
1373
1374         /* Workaround for unstable PLL clock */
1375         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1376             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1377                 u32 val = tr32(0x7d00);
1378
1379                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1380                 tw32(0x7d00, val);
1381                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1382                         int err;
1383
1384                         err = tg3_nvram_lock(tp);
1385                         tg3_halt_cpu(tp, RX_CPU_BASE);
1386                         if (!err)
1387                                 tg3_nvram_unlock(tp);
1388                 }
1389         }
1390
1391         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1392
1393         /* Finally, set the new power state. */
1394         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1395         udelay(100);    /* Delay after power state change */
1396
1397         return 0;
1398 }
1399
1400 static void tg3_link_report(struct tg3 *tp)
1401 {
1402         if (!netif_carrier_ok(tp->dev)) {
1403                 printk(KERN_INFO PFX "%s: Link is down.\n", tp->dev->name);
1404         } else {
1405                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1406                        tp->dev->name,
1407                        (tp->link_config.active_speed == SPEED_1000 ?
1408                         1000 :
1409                         (tp->link_config.active_speed == SPEED_100 ?
1410                          100 : 10)),
1411                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1412                         "full" : "half"));
1413
1414                 printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
1415                        "%s for RX.\n",
1416                        tp->dev->name,
1417                        (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
1418                        (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
1419         }
1420 }
1421
1422 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1423 {
1424         u32 new_tg3_flags = 0;
1425         u32 old_rx_mode = tp->rx_mode;
1426         u32 old_tx_mode = tp->tx_mode;
1427
1428         if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) {
1429
1430                 /* Convert 1000BaseX flow control bits to 1000BaseT
1431                  * bits before resolving flow control.
1432                  */
1433                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
1434                         local_adv &= ~(ADVERTISE_PAUSE_CAP |
1435                                        ADVERTISE_PAUSE_ASYM);
1436                         remote_adv &= ~(LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1437
1438                         if (local_adv & ADVERTISE_1000XPAUSE)
1439                                 local_adv |= ADVERTISE_PAUSE_CAP;
1440                         if (local_adv & ADVERTISE_1000XPSE_ASYM)
1441                                 local_adv |= ADVERTISE_PAUSE_ASYM;
1442                         if (remote_adv & LPA_1000XPAUSE)
1443                                 remote_adv |= LPA_PAUSE_CAP;
1444                         if (remote_adv & LPA_1000XPAUSE_ASYM)
1445                                 remote_adv |= LPA_PAUSE_ASYM;
1446                 }
1447
1448                 if (local_adv & ADVERTISE_PAUSE_CAP) {
1449                         if (local_adv & ADVERTISE_PAUSE_ASYM) {
1450                                 if (remote_adv & LPA_PAUSE_CAP)
1451                                         new_tg3_flags |=
1452                                                 (TG3_FLAG_RX_PAUSE |
1453                                                 TG3_FLAG_TX_PAUSE);
1454                                 else if (remote_adv & LPA_PAUSE_ASYM)
1455                                         new_tg3_flags |=
1456                                                 (TG3_FLAG_RX_PAUSE);
1457                         } else {
1458                                 if (remote_adv & LPA_PAUSE_CAP)
1459                                         new_tg3_flags |=
1460                                                 (TG3_FLAG_RX_PAUSE |
1461                                                 TG3_FLAG_TX_PAUSE);
1462                         }
1463                 } else if (local_adv & ADVERTISE_PAUSE_ASYM) {
1464                         if ((remote_adv & LPA_PAUSE_CAP) &&
1465                         (remote_adv & LPA_PAUSE_ASYM))
1466                                 new_tg3_flags |= TG3_FLAG_TX_PAUSE;
1467                 }
1468
1469                 tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
1470                 tp->tg3_flags |= new_tg3_flags;
1471         } else {
1472                 new_tg3_flags = tp->tg3_flags;
1473         }
1474
1475         if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
1476                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1477         else
1478                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1479
1480         if (old_rx_mode != tp->rx_mode) {
1481                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1482         }
1483
1484         if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
1485                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1486         else
1487                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1488
1489         if (old_tx_mode != tp->tx_mode) {
1490                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1491         }
1492 }
1493
1494 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1495 {
1496         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1497         case MII_TG3_AUX_STAT_10HALF:
1498                 *speed = SPEED_10;
1499                 *duplex = DUPLEX_HALF;
1500                 break;
1501
1502         case MII_TG3_AUX_STAT_10FULL:
1503                 *speed = SPEED_10;
1504                 *duplex = DUPLEX_FULL;
1505                 break;
1506
1507         case MII_TG3_AUX_STAT_100HALF:
1508                 *speed = SPEED_100;
1509                 *duplex = DUPLEX_HALF;
1510                 break;
1511
1512         case MII_TG3_AUX_STAT_100FULL:
1513                 *speed = SPEED_100;
1514                 *duplex = DUPLEX_FULL;
1515                 break;
1516
1517         case MII_TG3_AUX_STAT_1000HALF:
1518                 *speed = SPEED_1000;
1519                 *duplex = DUPLEX_HALF;
1520                 break;
1521
1522         case MII_TG3_AUX_STAT_1000FULL:
1523                 *speed = SPEED_1000;
1524                 *duplex = DUPLEX_FULL;
1525                 break;
1526
1527         default:
1528                 *speed = SPEED_INVALID;
1529                 *duplex = DUPLEX_INVALID;
1530                 break;
1531         };
1532 }
1533
1534 static void tg3_phy_copper_begin(struct tg3 *tp)
1535 {
1536         u32 new_adv;
1537         int i;
1538
1539         if (tp->link_config.phy_is_low_power) {
1540                 /* Entering low power mode.  Disable gigabit and
1541                  * 100baseT advertisements.
1542                  */
1543                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1544
1545                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1546                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1547                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1548                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1549
1550                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1551         } else if (tp->link_config.speed == SPEED_INVALID) {
1552                 tp->link_config.advertising =
1553                         (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
1554                          ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
1555                          ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
1556                          ADVERTISED_Autoneg | ADVERTISED_MII);
1557
1558                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1559                         tp->link_config.advertising &=
1560                                 ~(ADVERTISED_1000baseT_Half |
1561                                   ADVERTISED_1000baseT_Full);
1562
1563                 new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1564                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1565                         new_adv |= ADVERTISE_10HALF;
1566                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1567                         new_adv |= ADVERTISE_10FULL;
1568                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1569                         new_adv |= ADVERTISE_100HALF;
1570                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1571                         new_adv |= ADVERTISE_100FULL;
1572                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1573
1574                 if (tp->link_config.advertising &
1575                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1576                         new_adv = 0;
1577                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1578                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1579                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1580                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1581                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1582                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1583                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1584                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1585                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1586                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1587                 } else {
1588                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1589                 }
1590         } else {
1591                 /* Asking for a specific link mode. */
1592                 if (tp->link_config.speed == SPEED_1000) {
1593                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1594                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1595
1596                         if (tp->link_config.duplex == DUPLEX_FULL)
1597                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1598                         else
1599                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1600                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1601                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1602                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1603                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1604                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1605                 } else {
1606                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1607
1608                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1609                         if (tp->link_config.speed == SPEED_100) {
1610                                 if (tp->link_config.duplex == DUPLEX_FULL)
1611                                         new_adv |= ADVERTISE_100FULL;
1612                                 else
1613                                         new_adv |= ADVERTISE_100HALF;
1614                         } else {
1615                                 if (tp->link_config.duplex == DUPLEX_FULL)
1616                                         new_adv |= ADVERTISE_10FULL;
1617                                 else
1618                                         new_adv |= ADVERTISE_10HALF;
1619                         }
1620                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1621                 }
1622         }
1623
1624         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1625             tp->link_config.speed != SPEED_INVALID) {
1626                 u32 bmcr, orig_bmcr;
1627
1628                 tp->link_config.active_speed = tp->link_config.speed;
1629                 tp->link_config.active_duplex = tp->link_config.duplex;
1630
1631                 bmcr = 0;
1632                 switch (tp->link_config.speed) {
1633                 default:
1634                 case SPEED_10:
1635                         break;
1636
1637                 case SPEED_100:
1638                         bmcr |= BMCR_SPEED100;
1639                         break;
1640
1641                 case SPEED_1000:
1642                         bmcr |= TG3_BMCR_SPEED1000;
1643                         break;
1644                 };
1645
1646                 if (tp->link_config.duplex == DUPLEX_FULL)
1647                         bmcr |= BMCR_FULLDPLX;
1648
1649                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1650                     (bmcr != orig_bmcr)) {
1651                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1652                         for (i = 0; i < 1500; i++) {
1653                                 u32 tmp;
1654
1655                                 udelay(10);
1656                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1657                                     tg3_readphy(tp, MII_BMSR, &tmp))
1658                                         continue;
1659                                 if (!(tmp & BMSR_LSTATUS)) {
1660                                         udelay(40);
1661                                         break;
1662                                 }
1663                         }
1664                         tg3_writephy(tp, MII_BMCR, bmcr);
1665                         udelay(40);
1666                 }
1667         } else {
1668                 tg3_writephy(tp, MII_BMCR,
1669                              BMCR_ANENABLE | BMCR_ANRESTART);
1670         }
1671 }
1672
1673 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1674 {
1675         int err;
1676
1677         /* Turn off tap power management. */
1678         /* Set Extended packet length bit */
1679         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1680
1681         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1682         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1683
1684         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1685         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1686
1687         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1688         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1689
1690         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1691         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1692
1693         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1694         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1695
1696         udelay(40);
1697
1698         return err;
1699 }
1700
1701 static int tg3_copper_is_advertising_all(struct tg3 *tp)
1702 {
1703         u32 adv_reg, all_mask;
1704
1705         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1706                 return 0;
1707
1708         all_mask = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1709                     ADVERTISE_100HALF | ADVERTISE_100FULL);
1710         if ((adv_reg & all_mask) != all_mask)
1711                 return 0;
1712         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1713                 u32 tg3_ctrl;
1714
1715                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1716                         return 0;
1717
1718                 all_mask = (MII_TG3_CTRL_ADV_1000_HALF |
1719                             MII_TG3_CTRL_ADV_1000_FULL);
1720                 if ((tg3_ctrl & all_mask) != all_mask)
1721                         return 0;
1722         }
1723         return 1;
1724 }
1725
1726 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
1727 {
1728         int current_link_up;
1729         u32 bmsr, dummy;
1730         u16 current_speed;
1731         u8 current_duplex;
1732         int i, err;
1733
1734         tw32(MAC_EVENT, 0);
1735
1736         tw32_f(MAC_STATUS,
1737              (MAC_STATUS_SYNC_CHANGED |
1738               MAC_STATUS_CFG_CHANGED |
1739               MAC_STATUS_MI_COMPLETION |
1740               MAC_STATUS_LNKSTATE_CHANGED));
1741         udelay(40);
1742
1743         tp->mi_mode = MAC_MI_MODE_BASE;
1744         tw32_f(MAC_MI_MODE, tp->mi_mode);
1745         udelay(80);
1746
1747         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
1748
1749         /* Some third-party PHYs need to be reset on link going
1750          * down.
1751          */
1752         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1753              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1754              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
1755             netif_carrier_ok(tp->dev)) {
1756                 tg3_readphy(tp, MII_BMSR, &bmsr);
1757                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1758                     !(bmsr & BMSR_LSTATUS))
1759                         force_reset = 1;
1760         }
1761         if (force_reset)
1762                 tg3_phy_reset(tp);
1763
1764         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1765                 tg3_readphy(tp, MII_BMSR, &bmsr);
1766                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
1767                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
1768                         bmsr = 0;
1769
1770                 if (!(bmsr & BMSR_LSTATUS)) {
1771                         err = tg3_init_5401phy_dsp(tp);
1772                         if (err)
1773                                 return err;
1774
1775                         tg3_readphy(tp, MII_BMSR, &bmsr);
1776                         for (i = 0; i < 1000; i++) {
1777                                 udelay(10);
1778                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1779                                     (bmsr & BMSR_LSTATUS)) {
1780                                         udelay(40);
1781                                         break;
1782                                 }
1783                         }
1784
1785                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
1786                             !(bmsr & BMSR_LSTATUS) &&
1787                             tp->link_config.active_speed == SPEED_1000) {
1788                                 err = tg3_phy_reset(tp);
1789                                 if (!err)
1790                                         err = tg3_init_5401phy_dsp(tp);
1791                                 if (err)
1792                                         return err;
1793                         }
1794                 }
1795         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1796                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
1797                 /* 5701 {A0,B0} CRC bug workaround */
1798                 tg3_writephy(tp, 0x15, 0x0a75);
1799                 tg3_writephy(tp, 0x1c, 0x8c68);
1800                 tg3_writephy(tp, 0x1c, 0x8d68);
1801                 tg3_writephy(tp, 0x1c, 0x8c68);
1802         }
1803
1804         /* Clear pending interrupts... */
1805         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1806         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1807
1808         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
1809                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
1810         else
1811                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
1812
1813         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1814             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1815                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
1816                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1817                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
1818                 else
1819                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
1820         }
1821
1822         current_link_up = 0;
1823         current_speed = SPEED_INVALID;
1824         current_duplex = DUPLEX_INVALID;
1825
1826         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
1827                 u32 val;
1828
1829                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
1830                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
1831                 if (!(val & (1 << 10))) {
1832                         val |= (1 << 10);
1833                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
1834                         goto relink;
1835                 }
1836         }
1837
1838         bmsr = 0;
1839         for (i = 0; i < 100; i++) {
1840                 tg3_readphy(tp, MII_BMSR, &bmsr);
1841                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1842                     (bmsr & BMSR_LSTATUS))
1843                         break;
1844                 udelay(40);
1845         }
1846
1847         if (bmsr & BMSR_LSTATUS) {
1848                 u32 aux_stat, bmcr;
1849
1850                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
1851                 for (i = 0; i < 2000; i++) {
1852                         udelay(10);
1853                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
1854                             aux_stat)
1855                                 break;
1856                 }
1857
1858                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
1859                                              &current_speed,
1860                                              &current_duplex);
1861
1862                 bmcr = 0;
1863                 for (i = 0; i < 200; i++) {
1864                         tg3_readphy(tp, MII_BMCR, &bmcr);
1865                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
1866                                 continue;
1867                         if (bmcr && bmcr != 0x7fff)
1868                                 break;
1869                         udelay(10);
1870                 }
1871
1872                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
1873                         if (bmcr & BMCR_ANENABLE) {
1874                                 current_link_up = 1;
1875
1876                                 /* Force autoneg restart if we are exiting
1877                                  * low power mode.
1878                                  */
1879                                 if (!tg3_copper_is_advertising_all(tp))
1880                                         current_link_up = 0;
1881                         } else {
1882                                 current_link_up = 0;
1883                         }
1884                 } else {
1885                         if (!(bmcr & BMCR_ANENABLE) &&
1886                             tp->link_config.speed == current_speed &&
1887                             tp->link_config.duplex == current_duplex) {
1888                                 current_link_up = 1;
1889                         } else {
1890                                 current_link_up = 0;
1891                         }
1892                 }
1893
1894                 tp->link_config.active_speed = current_speed;
1895                 tp->link_config.active_duplex = current_duplex;
1896         }
1897
1898         if (current_link_up == 1 &&
1899             (tp->link_config.active_duplex == DUPLEX_FULL) &&
1900             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
1901                 u32 local_adv, remote_adv;
1902
1903                 if (tg3_readphy(tp, MII_ADVERTISE, &local_adv))
1904                         local_adv = 0;
1905                 local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
1906
1907                 if (tg3_readphy(tp, MII_LPA, &remote_adv))
1908                         remote_adv = 0;
1909
1910                 remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1911
1912                 /* If we are not advertising full pause capability,
1913                  * something is wrong.  Bring the link down and reconfigure.
1914                  */
1915                 if (local_adv != ADVERTISE_PAUSE_CAP) {
1916                         current_link_up = 0;
1917                 } else {
1918                         tg3_setup_flow_control(tp, local_adv, remote_adv);
1919                 }
1920         }
1921 relink:
1922         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
1923                 u32 tmp;
1924
1925                 tg3_phy_copper_begin(tp);
1926
1927                 tg3_readphy(tp, MII_BMSR, &tmp);
1928                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
1929                     (tmp & BMSR_LSTATUS))
1930                         current_link_up = 1;
1931         }
1932
1933         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
1934         if (current_link_up == 1) {
1935                 if (tp->link_config.active_speed == SPEED_100 ||
1936                     tp->link_config.active_speed == SPEED_10)
1937                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
1938                 else
1939                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1940         } else
1941                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1942
1943         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
1944         if (tp->link_config.active_duplex == DUPLEX_HALF)
1945                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
1946
1947         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
1948         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
1949                 if ((tp->led_ctrl == LED_CTRL_MODE_PHY_2) ||
1950                     (current_link_up == 1 &&
1951                      tp->link_config.active_speed == SPEED_10))
1952                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1953         } else {
1954                 if (current_link_up == 1)
1955                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1956         }
1957
1958         /* ??? Without this setting Netgear GA302T PHY does not
1959          * ??? send/receive packets...
1960          */
1961         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
1962             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
1963                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
1964                 tw32_f(MAC_MI_MODE, tp->mi_mode);
1965                 udelay(80);
1966         }
1967
1968         tw32_f(MAC_MODE, tp->mac_mode);
1969         udelay(40);
1970
1971         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
1972                 /* Polled via timer. */
1973                 tw32_f(MAC_EVENT, 0);
1974         } else {
1975                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
1976         }
1977         udelay(40);
1978
1979         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
1980             current_link_up == 1 &&
1981             tp->link_config.active_speed == SPEED_1000 &&
1982             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
1983              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
1984                 udelay(120);
1985                 tw32_f(MAC_STATUS,
1986                      (MAC_STATUS_SYNC_CHANGED |
1987                       MAC_STATUS_CFG_CHANGED));
1988                 udelay(40);
1989                 tg3_write_mem(tp,
1990                               NIC_SRAM_FIRMWARE_MBOX,
1991                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
1992         }
1993
1994         if (current_link_up != netif_carrier_ok(tp->dev)) {
1995                 if (current_link_up)
1996                         netif_carrier_on(tp->dev);
1997                 else
1998                         netif_carrier_off(tp->dev);
1999                 tg3_link_report(tp);
2000         }
2001
2002         return 0;
2003 }
2004
2005 struct tg3_fiber_aneginfo {
2006         int state;
2007 #define ANEG_STATE_UNKNOWN              0
2008 #define ANEG_STATE_AN_ENABLE            1
2009 #define ANEG_STATE_RESTART_INIT         2
2010 #define ANEG_STATE_RESTART              3
2011 #define ANEG_STATE_DISABLE_LINK_OK      4
2012 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2013 #define ANEG_STATE_ABILITY_DETECT       6
2014 #define ANEG_STATE_ACK_DETECT_INIT      7
2015 #define ANEG_STATE_ACK_DETECT           8
2016 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2017 #define ANEG_STATE_COMPLETE_ACK         10
2018 #define ANEG_STATE_IDLE_DETECT_INIT     11
2019 #define ANEG_STATE_IDLE_DETECT          12
2020 #define ANEG_STATE_LINK_OK              13
2021 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2022 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2023
2024         u32 flags;
2025 #define MR_AN_ENABLE            0x00000001
2026 #define MR_RESTART_AN           0x00000002
2027 #define MR_AN_COMPLETE          0x00000004
2028 #define MR_PAGE_RX              0x00000008
2029 #define MR_NP_LOADED            0x00000010
2030 #define MR_TOGGLE_TX            0x00000020
2031 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2032 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2033 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2034 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2035 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2036 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2037 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2038 #define MR_TOGGLE_RX            0x00002000
2039 #define MR_NP_RX                0x00004000
2040
2041 #define MR_LINK_OK              0x80000000
2042
2043         unsigned long link_time, cur_time;
2044
2045         u32 ability_match_cfg;
2046         int ability_match_count;
2047
2048         char ability_match, idle_match, ack_match;
2049
2050         u32 txconfig, rxconfig;
2051 #define ANEG_CFG_NP             0x00000080
2052 #define ANEG_CFG_ACK            0x00000040
2053 #define ANEG_CFG_RF2            0x00000020
2054 #define ANEG_CFG_RF1            0x00000010
2055 #define ANEG_CFG_PS2            0x00000001
2056 #define ANEG_CFG_PS1            0x00008000
2057 #define ANEG_CFG_HD             0x00004000
2058 #define ANEG_CFG_FD             0x00002000
2059 #define ANEG_CFG_INVAL          0x00001f06
2060
2061 };
2062 #define ANEG_OK         0
2063 #define ANEG_DONE       1
2064 #define ANEG_TIMER_ENAB 2
2065 #define ANEG_FAILED     -1
2066
2067 #define ANEG_STATE_SETTLE_TIME  10000
2068
2069 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2070                                    struct tg3_fiber_aneginfo *ap)
2071 {
2072         unsigned long delta;
2073         u32 rx_cfg_reg;
2074         int ret;
2075
2076         if (ap->state == ANEG_STATE_UNKNOWN) {
2077                 ap->rxconfig = 0;
2078                 ap->link_time = 0;
2079                 ap->cur_time = 0;
2080                 ap->ability_match_cfg = 0;
2081                 ap->ability_match_count = 0;
2082                 ap->ability_match = 0;
2083                 ap->idle_match = 0;
2084                 ap->ack_match = 0;
2085         }
2086         ap->cur_time++;
2087
2088         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2089                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2090
2091                 if (rx_cfg_reg != ap->ability_match_cfg) {
2092                         ap->ability_match_cfg = rx_cfg_reg;
2093                         ap->ability_match = 0;
2094                         ap->ability_match_count = 0;
2095                 } else {
2096                         if (++ap->ability_match_count > 1) {
2097                                 ap->ability_match = 1;
2098                                 ap->ability_match_cfg = rx_cfg_reg;
2099                         }
2100                 }
2101                 if (rx_cfg_reg & ANEG_CFG_ACK)
2102                         ap->ack_match = 1;
2103                 else
2104                         ap->ack_match = 0;
2105
2106                 ap->idle_match = 0;
2107         } else {
2108                 ap->idle_match = 1;
2109                 ap->ability_match_cfg = 0;
2110                 ap->ability_match_count = 0;
2111                 ap->ability_match = 0;
2112                 ap->ack_match = 0;
2113
2114                 rx_cfg_reg = 0;
2115         }
2116
2117         ap->rxconfig = rx_cfg_reg;
2118         ret = ANEG_OK;
2119
2120         switch(ap->state) {
2121         case ANEG_STATE_UNKNOWN:
2122                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2123                         ap->state = ANEG_STATE_AN_ENABLE;
2124
2125                 /* fallthru */
2126         case ANEG_STATE_AN_ENABLE:
2127                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2128                 if (ap->flags & MR_AN_ENABLE) {
2129                         ap->link_time = 0;
2130                         ap->cur_time = 0;
2131                         ap->ability_match_cfg = 0;
2132                         ap->ability_match_count = 0;
2133                         ap->ability_match = 0;
2134                         ap->idle_match = 0;
2135                         ap->ack_match = 0;
2136
2137                         ap->state = ANEG_STATE_RESTART_INIT;
2138                 } else {
2139                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2140                 }
2141                 break;
2142
2143         case ANEG_STATE_RESTART_INIT:
2144                 ap->link_time = ap->cur_time;
2145                 ap->flags &= ~(MR_NP_LOADED);
2146                 ap->txconfig = 0;
2147                 tw32(MAC_TX_AUTO_NEG, 0);
2148                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2149                 tw32_f(MAC_MODE, tp->mac_mode);
2150                 udelay(40);
2151
2152                 ret = ANEG_TIMER_ENAB;
2153                 ap->state = ANEG_STATE_RESTART;
2154
2155                 /* fallthru */
2156         case ANEG_STATE_RESTART:
2157                 delta = ap->cur_time - ap->link_time;
2158                 if (delta > ANEG_STATE_SETTLE_TIME) {
2159                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2160                 } else {
2161                         ret = ANEG_TIMER_ENAB;
2162                 }
2163                 break;
2164
2165         case ANEG_STATE_DISABLE_LINK_OK:
2166                 ret = ANEG_DONE;
2167                 break;
2168
2169         case ANEG_STATE_ABILITY_DETECT_INIT:
2170                 ap->flags &= ~(MR_TOGGLE_TX);
2171                 ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
2172                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2173                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2174                 tw32_f(MAC_MODE, tp->mac_mode);
2175                 udelay(40);
2176
2177                 ap->state = ANEG_STATE_ABILITY_DETECT;
2178                 break;
2179
2180         case ANEG_STATE_ABILITY_DETECT:
2181                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2182                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2183                 }
2184                 break;
2185
2186         case ANEG_STATE_ACK_DETECT_INIT:
2187                 ap->txconfig |= ANEG_CFG_ACK;
2188                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2189                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2190                 tw32_f(MAC_MODE, tp->mac_mode);
2191                 udelay(40);
2192
2193                 ap->state = ANEG_STATE_ACK_DETECT;
2194
2195                 /* fallthru */
2196         case ANEG_STATE_ACK_DETECT:
2197                 if (ap->ack_match != 0) {
2198                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2199                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2200                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2201                         } else {
2202                                 ap->state = ANEG_STATE_AN_ENABLE;
2203                         }
2204                 } else if (ap->ability_match != 0 &&
2205                            ap->rxconfig == 0) {
2206                         ap->state = ANEG_STATE_AN_ENABLE;
2207                 }
2208                 break;
2209
2210         case ANEG_STATE_COMPLETE_ACK_INIT:
2211                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2212                         ret = ANEG_FAILED;
2213                         break;
2214                 }
2215                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2216                                MR_LP_ADV_HALF_DUPLEX |
2217                                MR_LP_ADV_SYM_PAUSE |
2218                                MR_LP_ADV_ASYM_PAUSE |
2219                                MR_LP_ADV_REMOTE_FAULT1 |
2220                                MR_LP_ADV_REMOTE_FAULT2 |
2221                                MR_LP_ADV_NEXT_PAGE |
2222                                MR_TOGGLE_RX |
2223                                MR_NP_RX);
2224                 if (ap->rxconfig & ANEG_CFG_FD)
2225                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2226                 if (ap->rxconfig & ANEG_CFG_HD)
2227                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2228                 if (ap->rxconfig & ANEG_CFG_PS1)
2229                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2230                 if (ap->rxconfig & ANEG_CFG_PS2)
2231                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2232                 if (ap->rxconfig & ANEG_CFG_RF1)
2233                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2234                 if (ap->rxconfig & ANEG_CFG_RF2)
2235                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2236                 if (ap->rxconfig & ANEG_CFG_NP)
2237                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2238
2239                 ap->link_time = ap->cur_time;
2240
2241                 ap->flags ^= (MR_TOGGLE_TX);
2242                 if (ap->rxconfig & 0x0008)
2243                         ap->flags |= MR_TOGGLE_RX;
2244                 if (ap->rxconfig & ANEG_CFG_NP)
2245                         ap->flags |= MR_NP_RX;
2246                 ap->flags |= MR_PAGE_RX;
2247
2248                 ap->state = ANEG_STATE_COMPLETE_ACK;
2249                 ret = ANEG_TIMER_ENAB;
2250                 break;
2251
2252         case ANEG_STATE_COMPLETE_ACK:
2253                 if (ap->ability_match != 0 &&
2254                     ap->rxconfig == 0) {
2255                         ap->state = ANEG_STATE_AN_ENABLE;
2256                         break;
2257                 }
2258                 delta = ap->cur_time - ap->link_time;
2259                 if (delta > ANEG_STATE_SETTLE_TIME) {
2260                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2261                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2262                         } else {
2263                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2264                                     !(ap->flags & MR_NP_RX)) {
2265                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2266                                 } else {
2267                                         ret = ANEG_FAILED;
2268                                 }
2269                         }
2270                 }
2271                 break;
2272
2273         case ANEG_STATE_IDLE_DETECT_INIT:
2274                 ap->link_time = ap->cur_time;
2275                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2276                 tw32_f(MAC_MODE, tp->mac_mode);
2277                 udelay(40);
2278
2279                 ap->state = ANEG_STATE_IDLE_DETECT;
2280                 ret = ANEG_TIMER_ENAB;
2281                 break;
2282
2283         case ANEG_STATE_IDLE_DETECT:
2284                 if (ap->ability_match != 0 &&
2285                     ap->rxconfig == 0) {
2286                         ap->state = ANEG_STATE_AN_ENABLE;
2287                         break;
2288                 }
2289                 delta = ap->cur_time - ap->link_time;
2290                 if (delta > ANEG_STATE_SETTLE_TIME) {
2291                         /* XXX another gem from the Broadcom driver :( */
2292                         ap->state = ANEG_STATE_LINK_OK;
2293                 }
2294                 break;
2295
2296         case ANEG_STATE_LINK_OK:
2297                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2298                 ret = ANEG_DONE;
2299                 break;
2300
2301         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2302                 /* ??? unimplemented */
2303                 break;
2304
2305         case ANEG_STATE_NEXT_PAGE_WAIT:
2306                 /* ??? unimplemented */
2307                 break;
2308
2309         default:
2310                 ret = ANEG_FAILED;
2311                 break;
2312         };
2313
2314         return ret;
2315 }
2316
2317 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
2318 {
2319         int res = 0;
2320         struct tg3_fiber_aneginfo aninfo;
2321         int status = ANEG_FAILED;
2322         unsigned int tick;
2323         u32 tmp;
2324
2325         tw32_f(MAC_TX_AUTO_NEG, 0);
2326
2327         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2328         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2329         udelay(40);
2330
2331         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2332         udelay(40);
2333
2334         memset(&aninfo, 0, sizeof(aninfo));
2335         aninfo.flags |= MR_AN_ENABLE;
2336         aninfo.state = ANEG_STATE_UNKNOWN;
2337         aninfo.cur_time = 0;
2338         tick = 0;
2339         while (++tick < 195000) {
2340                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2341                 if (status == ANEG_DONE || status == ANEG_FAILED)
2342                         break;
2343
2344                 udelay(1);
2345         }
2346
2347         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2348         tw32_f(MAC_MODE, tp->mac_mode);
2349         udelay(40);
2350
2351         *flags = aninfo.flags;
2352
2353         if (status == ANEG_DONE &&
2354             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2355                              MR_LP_ADV_FULL_DUPLEX)))
2356                 res = 1;
2357
2358         return res;
2359 }
2360
2361 static void tg3_init_bcm8002(struct tg3 *tp)
2362 {
2363         u32 mac_status = tr32(MAC_STATUS);
2364         int i;
2365
2366         /* Reset when initting first time or we have a link. */
2367         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2368             !(mac_status & MAC_STATUS_PCS_SYNCED))
2369                 return;
2370
2371         /* Set PLL lock range. */
2372         tg3_writephy(tp, 0x16, 0x8007);
2373
2374         /* SW reset */
2375         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2376
2377         /* Wait for reset to complete. */
2378         /* XXX schedule_timeout() ... */
2379         for (i = 0; i < 500; i++)
2380                 udelay(10);
2381
2382         /* Config mode; select PMA/Ch 1 regs. */
2383         tg3_writephy(tp, 0x10, 0x8411);
2384
2385         /* Enable auto-lock and comdet, select txclk for tx. */
2386         tg3_writephy(tp, 0x11, 0x0a10);
2387
2388         tg3_writephy(tp, 0x18, 0x00a0);
2389         tg3_writephy(tp, 0x16, 0x41ff);
2390
2391         /* Assert and deassert POR. */
2392         tg3_writephy(tp, 0x13, 0x0400);
2393         udelay(40);
2394         tg3_writephy(tp, 0x13, 0x0000);
2395
2396         tg3_writephy(tp, 0x11, 0x0a50);
2397         udelay(40);
2398         tg3_writephy(tp, 0x11, 0x0a10);
2399
2400         /* Wait for signal to stabilize */
2401         /* XXX schedule_timeout() ... */
2402         for (i = 0; i < 15000; i++)
2403                 udelay(10);
2404
2405         /* Deselect the channel register so we can read the PHYID
2406          * later.
2407          */
2408         tg3_writephy(tp, 0x10, 0x8011);
2409 }
2410
2411 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2412 {
2413         u32 sg_dig_ctrl, sg_dig_status;
2414         u32 serdes_cfg, expected_sg_dig_ctrl;
2415         int workaround, port_a;
2416         int current_link_up;
2417
2418         serdes_cfg = 0;
2419         expected_sg_dig_ctrl = 0;
2420         workaround = 0;
2421         port_a = 1;
2422         current_link_up = 0;
2423
2424         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2425             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2426                 workaround = 1;
2427                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2428                         port_a = 0;
2429
2430                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2431                 /* preserve bits 20-23 for voltage regulator */
2432                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2433         }
2434
2435         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2436
2437         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2438                 if (sg_dig_ctrl & (1 << 31)) {
2439                         if (workaround) {
2440                                 u32 val = serdes_cfg;
2441
2442                                 if (port_a)
2443                                         val |= 0xc010000;
2444                                 else
2445                                         val |= 0x4010000;
2446                                 tw32_f(MAC_SERDES_CFG, val);
2447                         }
2448                         tw32_f(SG_DIG_CTRL, 0x01388400);
2449                 }
2450                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2451                         tg3_setup_flow_control(tp, 0, 0);
2452                         current_link_up = 1;
2453                 }
2454                 goto out;
2455         }
2456
2457         /* Want auto-negotiation.  */
2458         expected_sg_dig_ctrl = 0x81388400;
2459
2460         /* Pause capability */
2461         expected_sg_dig_ctrl |= (1 << 11);
2462
2463         /* Asymettric pause */
2464         expected_sg_dig_ctrl |= (1 << 12);
2465
2466         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2467                 if (workaround)
2468                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2469                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
2470                 udelay(5);
2471                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2472
2473                 tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2474         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2475                                  MAC_STATUS_SIGNAL_DET)) {
2476                 int i;
2477
2478                 /* Giver time to negotiate (~200ms) */
2479                 for (i = 0; i < 40000; i++) {
2480                         sg_dig_status = tr32(SG_DIG_STATUS);
2481                         if (sg_dig_status & (0x3))
2482                                 break;
2483                         udelay(5);
2484                 }
2485                 mac_status = tr32(MAC_STATUS);
2486
2487                 if ((sg_dig_status & (1 << 1)) &&
2488                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2489                         u32 local_adv, remote_adv;
2490
2491                         local_adv = ADVERTISE_PAUSE_CAP;
2492                         remote_adv = 0;
2493                         if (sg_dig_status & (1 << 19))
2494                                 remote_adv |= LPA_PAUSE_CAP;
2495                         if (sg_dig_status & (1 << 20))
2496                                 remote_adv |= LPA_PAUSE_ASYM;
2497
2498                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2499                         current_link_up = 1;
2500                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2501                 } else if (!(sg_dig_status & (1 << 1))) {
2502                         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED)
2503                                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2504                         else {
2505                                 if (workaround) {
2506                                         u32 val = serdes_cfg;
2507
2508                                         if (port_a)
2509                                                 val |= 0xc010000;
2510                                         else
2511                                                 val |= 0x4010000;
2512
2513                                         tw32_f(MAC_SERDES_CFG, val);
2514                                 }
2515
2516                                 tw32_f(SG_DIG_CTRL, 0x01388400);
2517                                 udelay(40);
2518
2519                                 /* Link parallel detection - link is up */
2520                                 /* only if we have PCS_SYNC and not */
2521                                 /* receiving config code words */
2522                                 mac_status = tr32(MAC_STATUS);
2523                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2524                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2525                                         tg3_setup_flow_control(tp, 0, 0);
2526                                         current_link_up = 1;
2527                                 }
2528                         }
2529                 }
2530         }
2531
2532 out:
2533         return current_link_up;
2534 }
2535
2536 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2537 {
2538         int current_link_up = 0;
2539
2540         if (!(mac_status & MAC_STATUS_PCS_SYNCED)) {
2541                 tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
2542                 goto out;
2543         }
2544
2545         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2546                 u32 flags;
2547                 int i;
2548
2549                 if (fiber_autoneg(tp, &flags)) {
2550                         u32 local_adv, remote_adv;
2551
2552                         local_adv = ADVERTISE_PAUSE_CAP;
2553                         remote_adv = 0;
2554                         if (flags & MR_LP_ADV_SYM_PAUSE)
2555                                 remote_adv |= LPA_PAUSE_CAP;
2556                         if (flags & MR_LP_ADV_ASYM_PAUSE)
2557                                 remote_adv |= LPA_PAUSE_ASYM;
2558
2559                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2560
2561                         tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2562                         current_link_up = 1;
2563                 }
2564                 for (i = 0; i < 30; i++) {
2565                         udelay(20);
2566                         tw32_f(MAC_STATUS,
2567                                (MAC_STATUS_SYNC_CHANGED |
2568                                 MAC_STATUS_CFG_CHANGED));
2569                         udelay(40);
2570                         if ((tr32(MAC_STATUS) &
2571                              (MAC_STATUS_SYNC_CHANGED |
2572                               MAC_STATUS_CFG_CHANGED)) == 0)
2573                                 break;
2574                 }
2575
2576                 mac_status = tr32(MAC_STATUS);
2577                 if (current_link_up == 0 &&
2578                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2579                     !(mac_status & MAC_STATUS_RCVD_CFG))
2580                         current_link_up = 1;
2581         } else {
2582                 /* Forcing 1000FD link up. */
2583                 current_link_up = 1;
2584                 tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2585
2586                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2587                 udelay(40);
2588         }
2589
2590 out:
2591         return current_link_up;
2592 }
2593
2594 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2595 {
2596         u32 orig_pause_cfg;
2597         u16 orig_active_speed;
2598         u8 orig_active_duplex;
2599         u32 mac_status;
2600         int current_link_up;
2601         int i;
2602
2603         orig_pause_cfg =
2604                 (tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2605                                   TG3_FLAG_TX_PAUSE));
2606         orig_active_speed = tp->link_config.active_speed;
2607         orig_active_duplex = tp->link_config.active_duplex;
2608
2609         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2610             netif_carrier_ok(tp->dev) &&
2611             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2612                 mac_status = tr32(MAC_STATUS);
2613                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2614                                MAC_STATUS_SIGNAL_DET |
2615                                MAC_STATUS_CFG_CHANGED |
2616                                MAC_STATUS_RCVD_CFG);
2617                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2618                                    MAC_STATUS_SIGNAL_DET)) {
2619                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2620                                             MAC_STATUS_CFG_CHANGED));
2621                         return 0;
2622                 }
2623         }
2624
2625         tw32_f(MAC_TX_AUTO_NEG, 0);
2626
2627         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2628         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2629         tw32_f(MAC_MODE, tp->mac_mode);
2630         udelay(40);
2631
2632         if (tp->phy_id == PHY_ID_BCM8002)
2633                 tg3_init_bcm8002(tp);
2634
2635         /* Enable link change event even when serdes polling.  */
2636         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2637         udelay(40);
2638
2639         current_link_up = 0;
2640         mac_status = tr32(MAC_STATUS);
2641
2642         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2643                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2644         else
2645                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2646
2647         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2648         tw32_f(MAC_MODE, tp->mac_mode);
2649         udelay(40);
2650
2651         tp->hw_status->status =
2652                 (SD_STATUS_UPDATED |
2653                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2654
2655         for (i = 0; i < 100; i++) {
2656                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2657                                     MAC_STATUS_CFG_CHANGED));
2658                 udelay(5);
2659                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2660                                          MAC_STATUS_CFG_CHANGED)) == 0)
2661                         break;
2662         }
2663
2664         mac_status = tr32(MAC_STATUS);
2665         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2666                 current_link_up = 0;
2667                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2668                         tw32_f(MAC_MODE, (tp->mac_mode |
2669                                           MAC_MODE_SEND_CONFIGS));
2670                         udelay(1);
2671                         tw32_f(MAC_MODE, tp->mac_mode);
2672                 }
2673         }
2674
2675         if (current_link_up == 1) {
2676                 tp->link_config.active_speed = SPEED_1000;
2677                 tp->link_config.active_duplex = DUPLEX_FULL;
2678                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2679                                     LED_CTRL_LNKLED_OVERRIDE |
2680                                     LED_CTRL_1000MBPS_ON));
2681         } else {
2682                 tp->link_config.active_speed = SPEED_INVALID;
2683                 tp->link_config.active_duplex = DUPLEX_INVALID;
2684                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2685                                     LED_CTRL_LNKLED_OVERRIDE |
2686                                     LED_CTRL_TRAFFIC_OVERRIDE));
2687         }
2688
2689         if (current_link_up != netif_carrier_ok(tp->dev)) {
2690                 if (current_link_up)
2691                         netif_carrier_on(tp->dev);
2692                 else
2693                         netif_carrier_off(tp->dev);
2694                 tg3_link_report(tp);
2695         } else {
2696                 u32 now_pause_cfg =
2697                         tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2698                                          TG3_FLAG_TX_PAUSE);
2699                 if (orig_pause_cfg != now_pause_cfg ||
2700                     orig_active_speed != tp->link_config.active_speed ||
2701                     orig_active_duplex != tp->link_config.active_duplex)
2702                         tg3_link_report(tp);
2703         }
2704
2705         return 0;
2706 }
2707
2708 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2709 {
2710         int current_link_up, err = 0;
2711         u32 bmsr, bmcr;
2712         u16 current_speed;
2713         u8 current_duplex;
2714
2715         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2716         tw32_f(MAC_MODE, tp->mac_mode);
2717         udelay(40);
2718
2719         tw32(MAC_EVENT, 0);
2720
2721         tw32_f(MAC_STATUS,
2722              (MAC_STATUS_SYNC_CHANGED |
2723               MAC_STATUS_CFG_CHANGED |
2724               MAC_STATUS_MI_COMPLETION |
2725               MAC_STATUS_LNKSTATE_CHANGED));
2726         udelay(40);
2727
2728         if (force_reset)
2729                 tg3_phy_reset(tp);
2730
2731         current_link_up = 0;
2732         current_speed = SPEED_INVALID;
2733         current_duplex = DUPLEX_INVALID;
2734
2735         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2736         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2737         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2738                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2739                         bmsr |= BMSR_LSTATUS;
2740                 else
2741                         bmsr &= ~BMSR_LSTATUS;
2742         }
2743
2744         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
2745
2746         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
2747             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2748                 /* do nothing, just check for link up at the end */
2749         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2750                 u32 adv, new_adv;
2751
2752                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2753                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
2754                                   ADVERTISE_1000XPAUSE |
2755                                   ADVERTISE_1000XPSE_ASYM |
2756                                   ADVERTISE_SLCT);
2757
2758                 /* Always advertise symmetric PAUSE just like copper */
2759                 new_adv |= ADVERTISE_1000XPAUSE;
2760
2761                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2762                         new_adv |= ADVERTISE_1000XHALF;
2763                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2764                         new_adv |= ADVERTISE_1000XFULL;
2765
2766                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
2767                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2768                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
2769                         tg3_writephy(tp, MII_BMCR, bmcr);
2770
2771                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2772                         tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2773                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2774
2775                         return err;
2776                 }
2777         } else {
2778                 u32 new_bmcr;
2779
2780                 bmcr &= ~BMCR_SPEED1000;
2781                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
2782
2783                 if (tp->link_config.duplex == DUPLEX_FULL)
2784                         new_bmcr |= BMCR_FULLDPLX;
2785
2786                 if (new_bmcr != bmcr) {
2787                         /* BMCR_SPEED1000 is a reserved bit that needs
2788                          * to be set on write.
2789                          */
2790                         new_bmcr |= BMCR_SPEED1000;
2791
2792                         /* Force a linkdown */
2793                         if (netif_carrier_ok(tp->dev)) {
2794                                 u32 adv;
2795
2796                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2797                                 adv &= ~(ADVERTISE_1000XFULL |
2798                                          ADVERTISE_1000XHALF |
2799                                          ADVERTISE_SLCT);
2800                                 tg3_writephy(tp, MII_ADVERTISE, adv);
2801                                 tg3_writephy(tp, MII_BMCR, bmcr |
2802                                                            BMCR_ANRESTART |
2803                                                            BMCR_ANENABLE);
2804                                 udelay(10);
2805                                 netif_carrier_off(tp->dev);
2806                         }
2807                         tg3_writephy(tp, MII_BMCR, new_bmcr);
2808                         bmcr = new_bmcr;
2809                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2810                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2811                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2812                             ASIC_REV_5714) {
2813                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2814                                         bmsr |= BMSR_LSTATUS;
2815                                 else
2816                                         bmsr &= ~BMSR_LSTATUS;
2817                         }
2818                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2819                 }
2820         }
2821
2822         if (bmsr & BMSR_LSTATUS) {
2823                 current_speed = SPEED_1000;
2824                 current_link_up = 1;
2825                 if (bmcr & BMCR_FULLDPLX)
2826                         current_duplex = DUPLEX_FULL;
2827                 else
2828                         current_duplex = DUPLEX_HALF;
2829
2830                 if (bmcr & BMCR_ANENABLE) {
2831                         u32 local_adv, remote_adv, common;
2832
2833                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
2834                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
2835                         common = local_adv & remote_adv;
2836                         if (common & (ADVERTISE_1000XHALF |
2837                                       ADVERTISE_1000XFULL)) {
2838                                 if (common & ADVERTISE_1000XFULL)
2839                                         current_duplex = DUPLEX_FULL;
2840                                 else
2841                                         current_duplex = DUPLEX_HALF;
2842
2843                                 tg3_setup_flow_control(tp, local_adv,
2844                                                        remote_adv);
2845                         }
2846                         else
2847                                 current_link_up = 0;
2848                 }
2849         }
2850
2851         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2852         if (tp->link_config.active_duplex == DUPLEX_HALF)
2853                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2854
2855         tw32_f(MAC_MODE, tp->mac_mode);
2856         udelay(40);
2857
2858         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2859
2860         tp->link_config.active_speed = current_speed;
2861         tp->link_config.active_duplex = current_duplex;
2862
2863         if (current_link_up != netif_carrier_ok(tp->dev)) {
2864                 if (current_link_up)
2865                         netif_carrier_on(tp->dev);
2866                 else {
2867                         netif_carrier_off(tp->dev);
2868                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2869                 }
2870                 tg3_link_report(tp);
2871         }
2872         return err;
2873 }
2874
2875 static void tg3_serdes_parallel_detect(struct tg3 *tp)
2876 {
2877         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED) {
2878                 /* Give autoneg time to complete. */
2879                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2880                 return;
2881         }
2882         if (!netif_carrier_ok(tp->dev) &&
2883             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
2884                 u32 bmcr;
2885
2886                 tg3_readphy(tp, MII_BMCR, &bmcr);
2887                 if (bmcr & BMCR_ANENABLE) {
2888                         u32 phy1, phy2;
2889
2890                         /* Select shadow register 0x1f */
2891                         tg3_writephy(tp, 0x1c, 0x7c00);
2892                         tg3_readphy(tp, 0x1c, &phy1);
2893
2894                         /* Select expansion interrupt status register */
2895                         tg3_writephy(tp, 0x17, 0x0f01);
2896                         tg3_readphy(tp, 0x15, &phy2);
2897                         tg3_readphy(tp, 0x15, &phy2);
2898
2899                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
2900                                 /* We have signal detect and not receiving
2901                                  * config code words, link is up by parallel
2902                                  * detection.
2903                                  */
2904
2905                                 bmcr &= ~BMCR_ANENABLE;
2906                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
2907                                 tg3_writephy(tp, MII_BMCR, bmcr);
2908                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
2909                         }
2910                 }
2911         }
2912         else if (netif_carrier_ok(tp->dev) &&
2913                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
2914                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2915                 u32 phy2;
2916
2917                 /* Select expansion interrupt status register */
2918                 tg3_writephy(tp, 0x17, 0x0f01);
2919                 tg3_readphy(tp, 0x15, &phy2);
2920                 if (phy2 & 0x20) {
2921                         u32 bmcr;
2922
2923                         /* Config code words received, turn on autoneg. */
2924                         tg3_readphy(tp, MII_BMCR, &bmcr);
2925                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
2926
2927                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2928
2929                 }
2930         }
2931 }
2932
2933 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
2934 {
2935         int err;
2936
2937         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2938                 err = tg3_setup_fiber_phy(tp, force_reset);
2939         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
2940                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
2941         } else {
2942                 err = tg3_setup_copper_phy(tp, force_reset);
2943         }
2944
2945         if (tp->link_config.active_speed == SPEED_1000 &&
2946             tp->link_config.active_duplex == DUPLEX_HALF)
2947                 tw32(MAC_TX_LENGTHS,
2948                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2949                       (6 << TX_LENGTHS_IPG_SHIFT) |
2950                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2951         else
2952                 tw32(MAC_TX_LENGTHS,
2953                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2954                       (6 << TX_LENGTHS_IPG_SHIFT) |
2955                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2956
2957         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2958                 if (netif_carrier_ok(tp->dev)) {
2959                         tw32(HOSTCC_STAT_COAL_TICKS,
2960                              tp->coal.stats_block_coalesce_usecs);
2961                 } else {
2962                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
2963                 }
2964         }
2965
2966         return err;
2967 }
2968
2969 /* This is called whenever we suspect that the system chipset is re-
2970  * ordering the sequence of MMIO to the tx send mailbox. The symptom
2971  * is bogus tx completions. We try to recover by setting the
2972  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
2973  * in the workqueue.
2974  */
2975 static void tg3_tx_recover(struct tg3 *tp)
2976 {
2977         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
2978                tp->write32_tx_mbox == tg3_write_indirect_mbox);
2979
2980         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
2981                "mapped I/O cycles to the network device, attempting to "
2982                "recover. Please report the problem to the driver maintainer "
2983                "and include system chipset information.\n", tp->dev->name);
2984
2985         spin_lock(&tp->lock);
2986         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
2987         spin_unlock(&tp->lock);
2988 }
2989
2990 /* Tigon3 never reports partial packet sends.  So we do not
2991  * need special logic to handle SKBs that have not had all
2992  * of their frags sent yet, like SunGEM does.
2993  */
2994 static void tg3_tx(struct tg3 *tp)
2995 {
2996         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
2997         u32 sw_idx = tp->tx_cons;
2998
2999         while (sw_idx != hw_idx) {
3000                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3001                 struct sk_buff *skb = ri->skb;
3002                 int i, tx_bug = 0;
3003
3004                 if (unlikely(skb == NULL)) {
3005                         tg3_tx_recover(tp);
3006                         return;
3007                 }
3008
3009                 pci_unmap_single(tp->pdev,
3010                                  pci_unmap_addr(ri, mapping),
3011                                  skb_headlen(skb),
3012                                  PCI_DMA_TODEVICE);
3013
3014                 ri->skb = NULL;
3015
3016                 sw_idx = NEXT_TX(sw_idx);
3017
3018                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3019                         ri = &tp->tx_buffers[sw_idx];
3020                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3021                                 tx_bug = 1;
3022
3023                         pci_unmap_page(tp->pdev,
3024                                        pci_unmap_addr(ri, mapping),
3025                                        skb_shinfo(skb)->frags[i].size,
3026                                        PCI_DMA_TODEVICE);
3027
3028                         sw_idx = NEXT_TX(sw_idx);
3029                 }
3030
3031                 dev_kfree_skb(skb);
3032
3033                 if (unlikely(tx_bug)) {
3034                         tg3_tx_recover(tp);
3035                         return;
3036                 }
3037         }
3038
3039         tp->tx_cons = sw_idx;
3040
3041         if (unlikely(netif_queue_stopped(tp->dev))) {
3042                 spin_lock(&tp->tx_lock);
3043                 if (netif_queue_stopped(tp->dev) &&
3044                     (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH))
3045                         netif_wake_queue(tp->dev);
3046                 spin_unlock(&tp->tx_lock);
3047         }
3048 }
3049
3050 /* Returns size of skb allocated or < 0 on error.
3051  *
3052  * We only need to fill in the address because the other members
3053  * of the RX descriptor are invariant, see tg3_init_rings.
3054  *
3055  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3056  * posting buffers we only dirty the first cache line of the RX
3057  * descriptor (containing the address).  Whereas for the RX status
3058  * buffers the cpu only reads the last cacheline of the RX descriptor
3059  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3060  */
3061 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3062                             int src_idx, u32 dest_idx_unmasked)
3063 {
3064         struct tg3_rx_buffer_desc *desc;
3065         struct ring_info *map, *src_map;
3066         struct sk_buff *skb;
3067         dma_addr_t mapping;
3068         int skb_size, dest_idx;
3069
3070         src_map = NULL;
3071         switch (opaque_key) {
3072         case RXD_OPAQUE_RING_STD:
3073                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3074                 desc = &tp->rx_std[dest_idx];
3075                 map = &tp->rx_std_buffers[dest_idx];
3076                 if (src_idx >= 0)
3077                         src_map = &tp->rx_std_buffers[src_idx];
3078                 skb_size = tp->rx_pkt_buf_sz;
3079                 break;
3080
3081         case RXD_OPAQUE_RING_JUMBO:
3082                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3083                 desc = &tp->rx_jumbo[dest_idx];
3084                 map = &tp->rx_jumbo_buffers[dest_idx];
3085                 if (src_idx >= 0)
3086                         src_map = &tp->rx_jumbo_buffers[src_idx];
3087                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3088                 break;
3089
3090         default:
3091                 return -EINVAL;
3092         };
3093
3094         /* Do not overwrite any of the map or rp information
3095          * until we are sure we can commit to a new buffer.
3096          *
3097          * Callers depend upon this behavior and assume that
3098          * we leave everything unchanged if we fail.
3099          */
3100         skb = dev_alloc_skb(skb_size);
3101         if (skb == NULL)
3102                 return -ENOMEM;
3103
3104         skb->dev = tp->dev;
3105         skb_reserve(skb, tp->rx_offset);
3106
3107         mapping = pci_map_single(tp->pdev, skb->data,
3108                                  skb_size - tp->rx_offset,
3109                                  PCI_DMA_FROMDEVICE);
3110
3111         map->skb = skb;
3112         pci_unmap_addr_set(map, mapping, mapping);
3113
3114         if (src_map != NULL)
3115                 src_map->skb = NULL;
3116
3117         desc->addr_hi = ((u64)mapping >> 32);
3118         desc->addr_lo = ((u64)mapping & 0xffffffff);
3119
3120         return skb_size;
3121 }
3122
3123 /* We only need to move over in the address because the other
3124  * members of the RX descriptor are invariant.  See notes above
3125  * tg3_alloc_rx_skb for full details.
3126  */
3127 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3128                            int src_idx, u32 dest_idx_unmasked)
3129 {
3130         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3131         struct ring_info *src_map, *dest_map;
3132         int dest_idx;
3133
3134         switch (opaque_key) {
3135         case RXD_OPAQUE_RING_STD:
3136                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3137                 dest_desc = &tp->rx_std[dest_idx];
3138                 dest_map = &tp->rx_std_buffers[dest_idx];
3139                 src_desc = &tp->rx_std[src_idx];
3140                 src_map = &tp->rx_std_buffers[src_idx];
3141                 break;
3142
3143         case RXD_OPAQUE_RING_JUMBO:
3144                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3145                 dest_desc = &tp->rx_jumbo[dest_idx];
3146                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3147                 src_desc = &tp->rx_jumbo[src_idx];
3148                 src_map = &tp->rx_jumbo_buffers[src_idx];
3149                 break;
3150
3151         default:
3152                 return;
3153         };
3154
3155         dest_map->skb = src_map->skb;
3156         pci_unmap_addr_set(dest_map, mapping,
3157                            pci_unmap_addr(src_map, mapping));
3158         dest_desc->addr_hi = src_desc->addr_hi;
3159         dest_desc->addr_lo = src_desc->addr_lo;
3160
3161         src_map->skb = NULL;
3162 }
3163
3164 #if TG3_VLAN_TAG_USED
3165 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3166 {
3167         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3168 }
3169 #endif
3170
3171 /* The RX ring scheme is composed of multiple rings which post fresh
3172  * buffers to the chip, and one special ring the chip uses to report
3173  * status back to the host.
3174  *
3175  * The special ring reports the status of received packets to the
3176  * host.  The chip does not write into the original descriptor the
3177  * RX buffer was obtained from.  The chip simply takes the original
3178  * descriptor as provided by the host, updates the status and length
3179  * field, then writes this into the next status ring entry.
3180  *
3181  * Each ring the host uses to post buffers to the chip is described
3182  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3183  * it is first placed into the on-chip ram.  When the packet's length
3184  * is known, it walks down the TG3_BDINFO entries to select the ring.
3185  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3186  * which is within the range of the new packet's length is chosen.
3187  *
3188  * The "separate ring for rx status" scheme may sound queer, but it makes
3189  * sense from a cache coherency perspective.  If only the host writes
3190  * to the buffer post rings, and only the chip writes to the rx status
3191  * rings, then cache lines never move beyond shared-modified state.
3192  * If both the host and chip were to write into the same ring, cache line
3193  * eviction could occur since both entities want it in an exclusive state.
3194  */
3195 static int tg3_rx(struct tg3 *tp, int budget)
3196 {
3197         u32 work_mask;
3198         u32 sw_idx = tp->rx_rcb_ptr;
3199         u16 hw_idx;
3200         int received;
3201
3202         hw_idx = tp->hw_status->idx[0].rx_producer;
3203         /*
3204          * We need to order the read of hw_idx and the read of
3205          * the opaque cookie.
3206          */
3207         rmb();
3208         work_mask = 0;
3209         received = 0;
3210         while (sw_idx != hw_idx && budget > 0) {
3211                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3212                 unsigned int len;
3213                 struct sk_buff *skb;
3214                 dma_addr_t dma_addr;
3215                 u32 opaque_key, desc_idx, *post_ptr;
3216
3217                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3218                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3219                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3220                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3221                                                   mapping);
3222                         skb = tp->rx_std_buffers[desc_idx].skb;
3223                         post_ptr = &tp->rx_std_ptr;
3224                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3225                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3226                                                   mapping);
3227                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3228                         post_ptr = &tp->rx_jumbo_ptr;
3229                 }
3230                 else {
3231                         goto next_pkt_nopost;
3232                 }
3233
3234                 work_mask |= opaque_key;
3235
3236                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3237                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3238                 drop_it:
3239                         tg3_recycle_rx(tp, opaque_key,
3240                                        desc_idx, *post_ptr);
3241                 drop_it_no_recycle:
3242                         /* Other statistics kept track of by card. */
3243                         tp->net_stats.rx_dropped++;
3244                         goto next_pkt;
3245                 }
3246
3247                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3248
3249                 if (len > RX_COPY_THRESHOLD
3250                         && tp->rx_offset == 2
3251                         /* rx_offset != 2 iff this is a 5701 card running
3252                          * in PCI-X mode [see tg3_get_invariants()] */
3253                 ) {
3254                         int skb_size;
3255
3256                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3257                                                     desc_idx, *post_ptr);
3258                         if (skb_size < 0)
3259                                 goto drop_it;
3260
3261                         pci_unmap_single(tp->pdev, dma_addr,
3262                                          skb_size - tp->rx_offset,
3263                                          PCI_DMA_FROMDEVICE);
3264
3265                         skb_put(skb, len);
3266                 } else {
3267                         struct sk_buff *copy_skb;
3268
3269                         tg3_recycle_rx(tp, opaque_key,
3270                                        desc_idx, *post_ptr);
3271
3272                         copy_skb = dev_alloc_skb(len + 2);
3273                         if (copy_skb == NULL)
3274                                 goto drop_it_no_recycle;
3275
3276                         copy_skb->dev = tp->dev;
3277                         skb_reserve(copy_skb, 2);
3278                         skb_put(copy_skb, len);
3279                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3280                         memcpy(copy_skb->data, skb->data, len);
3281                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3282
3283                         /* We'll reuse the original ring buffer. */
3284                         skb = copy_skb;
3285                 }
3286
3287                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3288                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3289                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3290                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3291                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3292                 else
3293                         skb->ip_summed = CHECKSUM_NONE;
3294
3295                 skb->protocol = eth_type_trans(skb, tp->dev);
3296 #if TG3_VLAN_TAG_USED
3297                 if (tp->vlgrp != NULL &&
3298                     desc->type_flags & RXD_FLAG_VLAN) {
3299                         tg3_vlan_rx(tp, skb,
3300                                     desc->err_vlan & RXD_VLAN_MASK);
3301                 } else
3302 #endif
3303                         netif_receive_skb(skb);
3304
3305                 tp->dev->last_rx = jiffies;
3306                 received++;
3307                 budget--;
3308
3309 next_pkt:
3310                 (*post_ptr)++;
3311 next_pkt_nopost:
3312                 sw_idx++;
3313                 sw_idx %= TG3_RX_RCB_RING_SIZE(tp);
3314
3315                 /* Refresh hw_idx to see if there is new work */
3316                 if (sw_idx == hw_idx) {
3317                         hw_idx = tp->hw_status->idx[0].rx_producer;
3318                         rmb();
3319                 }
3320         }
3321
3322         /* ACK the status ring. */
3323         tp->rx_rcb_ptr = sw_idx;
3324         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3325
3326         /* Refill RX ring(s). */
3327         if (work_mask & RXD_OPAQUE_RING_STD) {
3328                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3329                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3330                              sw_idx);
3331         }
3332         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3333                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3334                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3335                              sw_idx);
3336         }
3337         mmiowb();
3338
3339         return received;
3340 }
3341
3342 static int tg3_poll(struct net_device *netdev, int *budget)
3343 {
3344         struct tg3 *tp = netdev_priv(netdev);
3345         struct tg3_hw_status *sblk = tp->hw_status;
3346         int done;
3347
3348         /* handle link change and other phy events */
3349         if (!(tp->tg3_flags &
3350               (TG3_FLAG_USE_LINKCHG_REG |
3351                TG3_FLAG_POLL_SERDES))) {
3352                 if (sblk->status & SD_STATUS_LINK_CHG) {
3353                         sblk->status = SD_STATUS_UPDATED |
3354                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3355                         spin_lock(&tp->lock);
3356                         tg3_setup_phy(tp, 0);
3357                         spin_unlock(&tp->lock);
3358                 }
3359         }
3360
3361         /* run TX completion thread */
3362         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3363                 tg3_tx(tp);
3364                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) {
3365                         netif_rx_complete(netdev);
3366                         schedule_work(&tp->reset_task);
3367                         return 0;
3368                 }
3369         }
3370
3371         /* run RX thread, within the bounds set by NAPI.
3372          * All RX "locking" is done by ensuring outside
3373          * code synchronizes with dev->poll()
3374          */
3375         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr) {
3376                 int orig_budget = *budget;
3377                 int work_done;
3378
3379                 if (orig_budget > netdev->quota)
3380                         orig_budget = netdev->quota;
3381
3382                 work_done = tg3_rx(tp, orig_budget);
3383
3384                 *budget -= work_done;
3385                 netdev->quota -= work_done;
3386         }
3387
3388         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3389                 tp->last_tag = sblk->status_tag;
3390                 rmb();
3391         } else
3392                 sblk->status &= ~SD_STATUS_UPDATED;
3393
3394         /* if no more work, tell net stack and NIC we're done */
3395         done = !tg3_has_work(tp);
3396         if (done) {
3397                 netif_rx_complete(netdev);
3398                 tg3_restart_ints(tp);
3399         }
3400
3401         return (done ? 0 : 1);
3402 }
3403
3404 static void tg3_irq_quiesce(struct tg3 *tp)
3405 {
3406         BUG_ON(tp->irq_sync);
3407
3408         tp->irq_sync = 1;
3409         smp_mb();
3410
3411         synchronize_irq(tp->pdev->irq);
3412 }
3413
3414 static inline int tg3_irq_sync(struct tg3 *tp)
3415 {
3416         return tp->irq_sync;
3417 }
3418
3419 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3420  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3421  * with as well.  Most of the time, this is not necessary except when
3422  * shutting down the device.
3423  */
3424 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3425 {
3426         if (irq_sync)
3427                 tg3_irq_quiesce(tp);
3428         spin_lock_bh(&tp->lock);
3429 }
3430
3431 static inline void tg3_full_unlock(struct tg3 *tp)
3432 {
3433         spin_unlock_bh(&tp->lock);
3434 }
3435
3436 /* One-shot MSI handler - Chip automatically disables interrupt
3437  * after sending MSI so driver doesn't have to do it.
3438  */
3439 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id, struct pt_regs *regs)
3440 {
3441         struct net_device *dev = dev_id;
3442         struct tg3 *tp = netdev_priv(dev);
3443
3444         prefetch(tp->hw_status);
3445         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3446
3447         if (likely(!tg3_irq_sync(tp)))
3448                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3449
3450         return IRQ_HANDLED;
3451 }
3452
3453 /* MSI ISR - No need to check for interrupt sharing and no need to
3454  * flush status block and interrupt mailbox. PCI ordering rules
3455  * guarantee that MSI will arrive after the status block.
3456  */
3457 static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs)
3458 {
3459         struct net_device *dev = dev_id;
3460         struct tg3 *tp = netdev_priv(dev);
3461
3462         prefetch(tp->hw_status);
3463         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3464         /*
3465          * Writing any value to intr-mbox-0 clears PCI INTA# and
3466          * chip-internal interrupt pending events.
3467          * Writing non-zero to intr-mbox-0 additional tells the
3468          * NIC to stop sending us irqs, engaging "in-intr-handler"
3469          * event coalescing.
3470          */
3471         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3472         if (likely(!tg3_irq_sync(tp)))
3473                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3474
3475         return IRQ_RETVAL(1);
3476 }
3477
3478 static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs)
3479 {
3480         struct net_device *dev = dev_id;
3481         struct tg3 *tp = netdev_priv(dev);
3482         struct tg3_hw_status *sblk = tp->hw_status;
3483         unsigned int handled = 1;
3484
3485         /* In INTx mode, it is possible for the interrupt to arrive at
3486          * the CPU before the status block posted prior to the interrupt.
3487          * Reading the PCI State register will confirm whether the
3488          * interrupt is ours and will flush the status block.
3489          */
3490         if ((sblk->status & SD_STATUS_UPDATED) ||
3491             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3492                 /*
3493                  * Writing any value to intr-mbox-0 clears PCI INTA# and
3494                  * chip-internal interrupt pending events.
3495                  * Writing non-zero to intr-mbox-0 additional tells the
3496                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3497                  * event coalescing.
3498                  */
3499                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3500                              0x00000001);
3501                 if (tg3_irq_sync(tp))
3502                         goto out;
3503                 sblk->status &= ~SD_STATUS_UPDATED;
3504                 if (likely(tg3_has_work(tp))) {
3505                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3506                         netif_rx_schedule(dev);         /* schedule NAPI poll */
3507                 } else {
3508                         /* No work, shared interrupt perhaps?  re-enable
3509                          * interrupts, and flush that PCI write
3510                          */
3511                         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3512                                 0x00000000);
3513                 }
3514         } else {        /* shared interrupt */
3515                 handled = 0;
3516         }
3517 out:
3518         return IRQ_RETVAL(handled);
3519 }
3520
3521 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *regs)
3522 {
3523         struct net_device *dev = dev_id;
3524         struct tg3 *tp = netdev_priv(dev);
3525         struct tg3_hw_status *sblk = tp->hw_status;
3526         unsigned int handled = 1;
3527
3528         /* In INTx mode, it is possible for the interrupt to arrive at
3529          * the CPU before the status block posted prior to the interrupt.
3530          * Reading the PCI State register will confirm whether the
3531          * interrupt is ours and will flush the status block.
3532          */
3533         if ((sblk->status_tag != tp->last_tag) ||
3534             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3535                 /*
3536                  * writing any value to intr-mbox-0 clears PCI INTA# and
3537                  * chip-internal interrupt pending events.
3538                  * writing non-zero to intr-mbox-0 additional tells the
3539                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3540                  * event coalescing.
3541                  */
3542                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3543                              0x00000001);
3544                 if (tg3_irq_sync(tp))
3545                         goto out;
3546                 if (netif_rx_schedule_prep(dev)) {
3547                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3548                         /* Update last_tag to mark that this status has been
3549                          * seen. Because interrupt may be shared, we may be
3550                          * racing with tg3_poll(), so only update last_tag
3551                          * if tg3_poll() is not scheduled.
3552                          */
3553                         tp->last_tag = sblk->status_tag;
3554                         __netif_rx_schedule(dev);
3555                 }
3556         } else {        /* shared interrupt */
3557                 handled = 0;
3558         }
3559 out:
3560         return IRQ_RETVAL(handled);
3561 }
3562
3563 /* ISR for interrupt test */
3564 static irqreturn_t tg3_test_isr(int irq, void *dev_id,
3565                 struct pt_regs *regs)
3566 {
3567         struct net_device *dev = dev_id;
3568         struct tg3 *tp = netdev_priv(dev);
3569         struct tg3_hw_status *sblk = tp->hw_status;
3570
3571         if ((sblk->status & SD_STATUS_UPDATED) ||
3572             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3573                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3574                              0x00000001);
3575                 return IRQ_RETVAL(1);
3576         }
3577         return IRQ_RETVAL(0);
3578 }
3579
3580 static int tg3_init_hw(struct tg3 *, int);
3581 static int tg3_halt(struct tg3 *, int, int);
3582
3583 #ifdef CONFIG_NET_POLL_CONTROLLER
3584 static void tg3_poll_controller(struct net_device *dev)
3585 {
3586         struct tg3 *tp = netdev_priv(dev);
3587
3588         tg3_interrupt(tp->pdev->irq, dev, NULL);
3589 }
3590 #endif
3591
3592 static void tg3_reset_task(void *_data)
3593 {
3594         struct tg3 *tp = _data;
3595         unsigned int restart_timer;
3596
3597         tg3_full_lock(tp, 0);
3598         tp->tg3_flags |= TG3_FLAG_IN_RESET_TASK;
3599
3600         if (!netif_running(tp->dev)) {
3601                 tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3602                 tg3_full_unlock(tp);
3603                 return;
3604         }
3605
3606         tg3_full_unlock(tp);
3607
3608         tg3_netif_stop(tp);
3609
3610         tg3_full_lock(tp, 1);
3611
3612         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3613         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3614
3615         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3616                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
3617                 tp->write32_rx_mbox = tg3_write_flush_reg32;
3618                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
3619                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
3620         }
3621
3622         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
3623         tg3_init_hw(tp, 1);
3624
3625         tg3_netif_start(tp);
3626
3627         if (restart_timer)
3628                 mod_timer(&tp->timer, jiffies + 1);
3629
3630         tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3631
3632         tg3_full_unlock(tp);
3633 }
3634
3635 static void tg3_tx_timeout(struct net_device *dev)
3636 {
3637         struct tg3 *tp = netdev_priv(dev);
3638
3639         printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
3640                dev->name);
3641
3642         schedule_work(&tp->reset_task);
3643 }
3644
3645 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
3646 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
3647 {
3648         u32 base = (u32) mapping & 0xffffffff;
3649
3650         return ((base > 0xffffdcc0) &&
3651                 (base + len + 8 < base));
3652 }
3653
3654 /* Test for DMA addresses > 40-bit */
3655 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
3656                                           int len)
3657 {
3658 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
3659         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
3660                 return (((u64) mapping + len) > DMA_40BIT_MASK);
3661         return 0;
3662 #else
3663         return 0;
3664 #endif
3665 }
3666
3667 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
3668
3669 /* Workaround 4GB and 40-bit hardware DMA bugs. */
3670 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
3671                                        u32 last_plus_one, u32 *start,
3672                                        u32 base_flags, u32 mss)
3673 {
3674         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
3675         dma_addr_t new_addr = 0;
3676         u32 entry = *start;
3677         int i, ret = 0;
3678
3679         if (!new_skb) {
3680                 ret = -1;
3681         } else {
3682                 /* New SKB is guaranteed to be linear. */
3683                 entry = *start;
3684                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
3685                                           PCI_DMA_TODEVICE);
3686                 /* Make sure new skb does not cross any 4G boundaries.
3687                  * Drop the packet if it does.
3688                  */
3689                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
3690                         ret = -1;
3691                         dev_kfree_skb(new_skb);
3692                         new_skb = NULL;
3693                 } else {
3694                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
3695                                     base_flags, 1 | (mss << 1));
3696                         *start = NEXT_TX(entry);
3697                 }
3698         }
3699
3700         /* Now clean up the sw ring entries. */
3701         i = 0;
3702         while (entry != last_plus_one) {
3703                 int len;
3704
3705                 if (i == 0)
3706                         len = skb_headlen(skb);
3707                 else
3708                         len = skb_shinfo(skb)->frags[i-1].size;
3709                 pci_unmap_single(tp->pdev,
3710                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
3711                                  len, PCI_DMA_TODEVICE);
3712                 if (i == 0) {
3713                         tp->tx_buffers[entry].skb = new_skb;
3714                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
3715                 } else {
3716                         tp->tx_buffers[entry].skb = NULL;
3717                 }
3718                 entry = NEXT_TX(entry);
3719                 i++;
3720         }
3721
3722         dev_kfree_skb(skb);
3723
3724         return ret;
3725 }
3726
3727 static void tg3_set_txd(struct tg3 *tp, int entry,
3728                         dma_addr_t mapping, int len, u32 flags,
3729                         u32 mss_and_is_end)
3730 {
3731         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
3732         int is_end = (mss_and_is_end & 0x1);
3733         u32 mss = (mss_and_is_end >> 1);
3734         u32 vlan_tag = 0;
3735
3736         if (is_end)
3737                 flags |= TXD_FLAG_END;
3738         if (flags & TXD_FLAG_VLAN) {
3739                 vlan_tag = flags >> 16;
3740                 flags &= 0xffff;
3741         }
3742         vlan_tag |= (mss << TXD_MSS_SHIFT);
3743
3744         txd->addr_hi = ((u64) mapping >> 32);
3745         txd->addr_lo = ((u64) mapping & 0xffffffff);
3746         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
3747         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
3748 }
3749
3750 /* hard_start_xmit for devices that don't have any bugs and
3751  * support TG3_FLG2_HW_TSO_2 only.
3752  */
3753 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
3754 {
3755         struct tg3 *tp = netdev_priv(dev);
3756         dma_addr_t mapping;
3757         u32 len, entry, base_flags, mss;
3758
3759         len = skb_headlen(skb);
3760
3761         /* We are running in BH disabled context with netif_tx_lock
3762          * and TX reclaim runs via tp->poll inside of a software
3763          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3764          * no IRQ context deadlocks to worry about either.  Rejoice!
3765          */
3766         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3767                 if (!netif_queue_stopped(dev)) {
3768                         netif_stop_queue(dev);
3769
3770                         /* This is a hard error, log it. */
3771                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3772                                "queue awake!\n", dev->name);
3773                 }
3774                 return NETDEV_TX_BUSY;
3775         }
3776
3777         entry = tp->tx_prod;
3778         base_flags = 0;
3779 #if TG3_TSO_SUPPORT != 0
3780         mss = 0;
3781         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3782             (mss = skb_shinfo(skb)->gso_size) != 0) {
3783                 int tcp_opt_len, ip_tcp_len;
3784
3785                 if (skb_header_cloned(skb) &&
3786                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3787                         dev_kfree_skb(skb);
3788                         goto out_unlock;
3789                 }
3790
3791                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3792                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
3793
3794                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3795                                TXD_FLAG_CPU_POST_DMA);
3796
3797                 skb->nh.iph->check = 0;
3798                 skb->nh.iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
3799
3800                 skb->h.th->check = 0;
3801
3802                 mss |= (ip_tcp_len + tcp_opt_len) << 9;
3803         }
3804         else if (skb->ip_summed == CHECKSUM_HW)
3805                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3806 #else
3807         mss = 0;
3808         if (skb->ip_summed == CHECKSUM_HW)
3809                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3810 #endif
3811 #if TG3_VLAN_TAG_USED
3812         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3813                 base_flags |= (TXD_FLAG_VLAN |
3814                                (vlan_tx_tag_get(skb) << 16));
3815 #endif
3816
3817         /* Queue skb data, a.k.a. the main skb fragment. */
3818         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3819
3820         tp->tx_buffers[entry].skb = skb;
3821         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3822
3823         tg3_set_txd(tp, entry, mapping, len, base_flags,
3824                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3825
3826         entry = NEXT_TX(entry);
3827
3828         /* Now loop through additional data fragments, and queue them. */
3829         if (skb_shinfo(skb)->nr_frags > 0) {
3830                 unsigned int i, last;
3831
3832                 last = skb_shinfo(skb)->nr_frags - 1;
3833                 for (i = 0; i <= last; i++) {
3834                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3835
3836                         len = frag->size;
3837                         mapping = pci_map_page(tp->pdev,
3838                                                frag->page,
3839                                                frag->page_offset,
3840                                                len, PCI_DMA_TODEVICE);
3841
3842                         tp->tx_buffers[entry].skb = NULL;
3843                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3844
3845                         tg3_set_txd(tp, entry, mapping, len,
3846                                     base_flags, (i == last) | (mss << 1));
3847
3848                         entry = NEXT_TX(entry);
3849                 }
3850         }
3851
3852         /* Packets are ready, update Tx producer idx local and on card. */
3853         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
3854
3855         tp->tx_prod = entry;
3856         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
3857                 spin_lock(&tp->tx_lock);
3858                 netif_stop_queue(dev);
3859                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
3860                         netif_wake_queue(tp->dev);
3861                 spin_unlock(&tp->tx_lock);
3862         }
3863
3864 out_unlock:
3865         mmiowb();
3866
3867         dev->trans_start = jiffies;
3868
3869         return NETDEV_TX_OK;
3870 }
3871
3872 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
3873  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
3874  */
3875 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
3876 {
3877         struct tg3 *tp = netdev_priv(dev);
3878         dma_addr_t mapping;
3879         u32 len, entry, base_flags, mss;
3880         int would_hit_hwbug;
3881
3882         len = skb_headlen(skb);
3883
3884         /* We are running in BH disabled context with netif_tx_lock
3885          * and TX reclaim runs via tp->poll inside of a software
3886          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3887          * no IRQ context deadlocks to worry about either.  Rejoice!
3888          */
3889         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3890                 if (!netif_queue_stopped(dev)) {
3891                         netif_stop_queue(dev);
3892
3893                         /* This is a hard error, log it. */
3894                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3895                                "queue awake!\n", dev->name);
3896                 }
3897                 return NETDEV_TX_BUSY;
3898         }
3899
3900         entry = tp->tx_prod;
3901         base_flags = 0;
3902         if (skb->ip_summed == CHECKSUM_HW)
3903                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3904 #if TG3_TSO_SUPPORT != 0
3905         mss = 0;
3906         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3907             (mss = skb_shinfo(skb)->gso_size) != 0) {
3908                 int tcp_opt_len, ip_tcp_len;
3909
3910                 if (skb_header_cloned(skb) &&
3911                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3912                         dev_kfree_skb(skb);
3913                         goto out_unlock;
3914                 }
3915
3916                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3917                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
3918
3919                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3920                                TXD_FLAG_CPU_POST_DMA);
3921
3922                 skb->nh.iph->check = 0;
3923                 skb->nh.iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
3924                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
3925                         skb->h.th->check = 0;
3926                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
3927                 }
3928                 else {
3929                         skb->h.th->check =
3930                                 ~csum_tcpudp_magic(skb->nh.iph->saddr,
3931                                                    skb->nh.iph->daddr,
3932                                                    0, IPPROTO_TCP, 0);
3933                 }
3934
3935                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
3936                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
3937                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
3938                                 int tsflags;
3939
3940                                 tsflags = ((skb->nh.iph->ihl - 5) +
3941                                            (tcp_opt_len >> 2));
3942                                 mss |= (tsflags << 11);
3943                         }
3944                 } else {
3945                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
3946                                 int tsflags;
3947
3948                                 tsflags = ((skb->nh.iph->ihl - 5) +
3949                                            (tcp_opt_len >> 2));
3950                                 base_flags |= tsflags << 12;
3951                         }
3952                 }
3953         }
3954 #else
3955         mss = 0;
3956 #endif
3957 #if TG3_VLAN_TAG_USED
3958         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3959                 base_flags |= (TXD_FLAG_VLAN |
3960                                (vlan_tx_tag_get(skb) << 16));
3961 #endif
3962
3963         /* Queue skb data, a.k.a. the main skb fragment. */
3964         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3965
3966         tp->tx_buffers[entry].skb = skb;
3967         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3968
3969         would_hit_hwbug = 0;
3970
3971         if (tg3_4g_overflow_test(mapping, len))
3972                 would_hit_hwbug = 1;
3973
3974         tg3_set_txd(tp, entry, mapping, len, base_flags,
3975                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3976
3977         entry = NEXT_TX(entry);
3978
3979         /* Now loop through additional data fragments, and queue them. */
3980         if (skb_shinfo(skb)->nr_frags > 0) {
3981                 unsigned int i, last;
3982
3983                 last = skb_shinfo(skb)->nr_frags - 1;
3984                 for (i = 0; i <= last; i++) {
3985                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3986
3987                         len = frag->size;
3988                         mapping = pci_map_page(tp->pdev,
3989                                                frag->page,
3990                                                frag->page_offset,
3991                                                len, PCI_DMA_TODEVICE);
3992
3993                         tp->tx_buffers[entry].skb = NULL;
3994                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3995
3996                         if (tg3_4g_overflow_test(mapping, len))
3997                                 would_hit_hwbug = 1;
3998
3999                         if (tg3_40bit_overflow_test(tp, mapping, len))
4000                                 would_hit_hwbug = 1;
4001
4002                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4003                                 tg3_set_txd(tp, entry, mapping, len,
4004                                             base_flags, (i == last)|(mss << 1));
4005                         else
4006                                 tg3_set_txd(tp, entry, mapping, len,
4007                                             base_flags, (i == last));
4008
4009                         entry = NEXT_TX(entry);
4010                 }
4011         }
4012
4013         if (would_hit_hwbug) {
4014                 u32 last_plus_one = entry;
4015                 u32 start;
4016
4017                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4018                 start &= (TG3_TX_RING_SIZE - 1);
4019
4020                 /* If the workaround fails due to memory/mapping
4021                  * failure, silently drop this packet.
4022                  */
4023                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4024                                                 &start, base_flags, mss))
4025                         goto out_unlock;
4026
4027                 entry = start;
4028         }
4029
4030         /* Packets are ready, update Tx producer idx local and on card. */
4031         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4032
4033         tp->tx_prod = entry;
4034         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
4035                 spin_lock(&tp->tx_lock);
4036                 netif_stop_queue(dev);
4037                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
4038                         netif_wake_queue(tp->dev);
4039                 spin_unlock(&tp->tx_lock);
4040         }
4041
4042 out_unlock:
4043         mmiowb();
4044
4045         dev->trans_start = jiffies;
4046
4047         return NETDEV_TX_OK;
4048 }
4049
4050 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4051                                int new_mtu)
4052 {
4053         dev->mtu = new_mtu;
4054
4055         if (new_mtu > ETH_DATA_LEN) {
4056                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4057                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4058                         ethtool_op_set_tso(dev, 0);
4059                 }
4060                 else
4061                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4062         } else {
4063                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4064                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4065                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4066         }
4067 }
4068
4069 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4070 {
4071         struct tg3 *tp = netdev_priv(dev);
4072
4073         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4074                 return -EINVAL;
4075
4076         if (!netif_running(dev)) {
4077                 /* We'll just catch it later when the
4078                  * device is up'd.
4079                  */
4080                 tg3_set_mtu(dev, tp, new_mtu);
4081                 return 0;
4082         }
4083
4084         tg3_netif_stop(tp);
4085
4086         tg3_full_lock(tp, 1);
4087
4088         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4089
4090         tg3_set_mtu(dev, tp, new_mtu);
4091
4092         tg3_init_hw(tp, 0);
4093
4094         tg3_netif_start(tp);
4095
4096         tg3_full_unlock(tp);
4097
4098         return 0;
4099 }
4100
4101 /* Free up pending packets in all rx/tx rings.
4102  *
4103  * The chip has been shut down and the driver detached from
4104  * the networking, so no interrupts or new tx packets will
4105  * end up in the driver.  tp->{tx,}lock is not held and we are not
4106  * in an interrupt context and thus may sleep.
4107  */
4108 static void tg3_free_rings(struct tg3 *tp)
4109 {
4110         struct ring_info *rxp;
4111         int i;
4112
4113         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4114                 rxp = &tp->rx_std_buffers[i];
4115
4116                 if (rxp->skb == NULL)
4117                         continue;
4118                 pci_unmap_single(tp->pdev,
4119                                  pci_unmap_addr(rxp, mapping),
4120                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4121                                  PCI_DMA_FROMDEVICE);
4122                 dev_kfree_skb_any(rxp->skb);
4123                 rxp->skb = NULL;
4124         }
4125
4126         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4127                 rxp = &tp->rx_jumbo_buffers[i];
4128
4129                 if (rxp->skb == NULL)
4130                         continue;
4131                 pci_unmap_single(tp->pdev,
4132                                  pci_unmap_addr(rxp, mapping),
4133                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4134                                  PCI_DMA_FROMDEVICE);
4135                 dev_kfree_skb_any(rxp->skb);
4136                 rxp->skb = NULL;
4137         }
4138
4139         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4140                 struct tx_ring_info *txp;
4141                 struct sk_buff *skb;
4142                 int j;
4143
4144                 txp = &tp->tx_buffers[i];
4145                 skb = txp->skb;
4146
4147                 if (skb == NULL) {
4148                         i++;
4149                         continue;
4150                 }
4151
4152                 pci_unmap_single(tp->pdev,
4153                                  pci_unmap_addr(txp, mapping),
4154                                  skb_headlen(skb),
4155                                  PCI_DMA_TODEVICE);
4156                 txp->skb = NULL;
4157
4158                 i++;
4159
4160                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4161                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4162                         pci_unmap_page(tp->pdev,
4163                                        pci_unmap_addr(txp, mapping),
4164                                        skb_shinfo(skb)->frags[j].size,
4165                                        PCI_DMA_TODEVICE);
4166                         i++;
4167                 }
4168
4169                 dev_kfree_skb_any(skb);
4170         }
4171 }
4172
4173 /* Initialize tx/rx rings for packet processing.
4174  *
4175  * The chip has been shut down and the driver detached from
4176  * the networking, so no interrupts or new tx packets will
4177  * end up in the driver.  tp->{tx,}lock are held and thus
4178  * we may not sleep.
4179  */
4180 static void tg3_init_rings(struct tg3 *tp)
4181 {
4182         u32 i;
4183
4184         /* Free up all the SKBs. */
4185         tg3_free_rings(tp);
4186
4187         /* Zero out all descriptors. */
4188         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4189         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4190         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4191         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4192
4193         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4194         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4195             (tp->dev->mtu > ETH_DATA_LEN))
4196                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4197
4198         /* Initialize invariants of the rings, we only set this
4199          * stuff once.  This works because the card does not
4200          * write into the rx buffer posting rings.
4201          */
4202         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4203                 struct tg3_rx_buffer_desc *rxd;
4204
4205                 rxd = &tp->rx_std[i];
4206                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4207                         << RXD_LEN_SHIFT;
4208                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4209                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4210                                (i << RXD_OPAQUE_INDEX_SHIFT));
4211         }
4212
4213         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4214                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4215                         struct tg3_rx_buffer_desc *rxd;
4216
4217                         rxd = &tp->rx_jumbo[i];
4218                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4219                                 << RXD_LEN_SHIFT;
4220                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4221                                 RXD_FLAG_JUMBO;
4222                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4223                                (i << RXD_OPAQUE_INDEX_SHIFT));
4224                 }
4225         }
4226
4227         /* Now allocate fresh SKBs for each rx ring. */
4228         for (i = 0; i < tp->rx_pending; i++) {
4229                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD,
4230                                      -1, i) < 0)
4231                         break;
4232         }
4233
4234         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4235                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4236                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4237                                              -1, i) < 0)
4238                                 break;
4239                 }
4240         }
4241 }
4242
4243 /*
4244  * Must not be invoked with interrupt sources disabled and
4245  * the hardware shutdown down.
4246  */
4247 static void tg3_free_consistent(struct tg3 *tp)
4248 {
4249         kfree(tp->rx_std_buffers);
4250         tp->rx_std_buffers = NULL;
4251         if (tp->rx_std) {
4252                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4253                                     tp->rx_std, tp->rx_std_mapping);
4254                 tp->rx_std = NULL;
4255         }
4256         if (tp->rx_jumbo) {
4257                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4258                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4259                 tp->rx_jumbo = NULL;
4260         }
4261         if (tp->rx_rcb) {
4262                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4263                                     tp->rx_rcb, tp->rx_rcb_mapping);
4264                 tp->rx_rcb = NULL;
4265         }
4266         if (tp->tx_ring) {
4267                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4268                         tp->tx_ring, tp->tx_desc_mapping);
4269                 tp->tx_ring = NULL;
4270         }
4271         if (tp->hw_status) {
4272                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4273                                     tp->hw_status, tp->status_mapping);
4274                 tp->hw_status = NULL;
4275         }
4276         if (tp->hw_stats) {
4277                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4278                                     tp->hw_stats, tp->stats_mapping);
4279                 tp->hw_stats = NULL;
4280         }
4281 }
4282
4283 /*
4284  * Must not be invoked with interrupt sources disabled and
4285  * the hardware shutdown down.  Can sleep.
4286  */
4287 static int tg3_alloc_consistent(struct tg3 *tp)
4288 {
4289         tp->rx_std_buffers = kmalloc((sizeof(struct ring_info) *
4290                                       (TG3_RX_RING_SIZE +
4291                                        TG3_RX_JUMBO_RING_SIZE)) +
4292                                      (sizeof(struct tx_ring_info) *
4293                                       TG3_TX_RING_SIZE),
4294                                      GFP_KERNEL);
4295         if (!tp->rx_std_buffers)
4296                 return -ENOMEM;
4297
4298         memset(tp->rx_std_buffers, 0,
4299                (sizeof(struct ring_info) *
4300                 (TG3_RX_RING_SIZE +
4301                  TG3_RX_JUMBO_RING_SIZE)) +
4302                (sizeof(struct tx_ring_info) *
4303                 TG3_TX_RING_SIZE));
4304
4305         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4306         tp->tx_buffers = (struct tx_ring_info *)
4307                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4308
4309         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4310                                           &tp->rx_std_mapping);
4311         if (!tp->rx_std)
4312                 goto err_out;
4313
4314         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4315                                             &tp->rx_jumbo_mapping);
4316
4317         if (!tp->rx_jumbo)
4318                 goto err_out;
4319
4320         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4321                                           &tp->rx_rcb_mapping);
4322         if (!tp->rx_rcb)
4323                 goto err_out;
4324
4325         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4326                                            &tp->tx_desc_mapping);
4327         if (!tp->tx_ring)
4328                 goto err_out;
4329
4330         tp->hw_status = pci_alloc_consistent(tp->pdev,
4331                                              TG3_HW_STATUS_SIZE,
4332                                              &tp->status_mapping);
4333         if (!tp->hw_status)
4334                 goto err_out;
4335
4336         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4337                                             sizeof(struct tg3_hw_stats),
4338                                             &tp->stats_mapping);
4339         if (!tp->hw_stats)
4340                 goto err_out;
4341
4342         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4343         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4344
4345         return 0;
4346
4347 err_out:
4348         tg3_free_consistent(tp);
4349         return -ENOMEM;
4350 }
4351
4352 #define MAX_WAIT_CNT 1000
4353
4354 /* To stop a block, clear the enable bit and poll till it
4355  * clears.  tp->lock is held.
4356  */
4357 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4358 {
4359         unsigned int i;
4360         u32 val;
4361
4362         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4363                 switch (ofs) {
4364                 case RCVLSC_MODE:
4365                 case DMAC_MODE:
4366                 case MBFREE_MODE:
4367                 case BUFMGR_MODE:
4368                 case MEMARB_MODE:
4369                         /* We can't enable/disable these bits of the
4370                          * 5705/5750, just say success.
4371                          */
4372                         return 0;
4373
4374                 default:
4375                         break;
4376                 };
4377         }
4378
4379         val = tr32(ofs);
4380         val &= ~enable_bit;
4381         tw32_f(ofs, val);
4382
4383         for (i = 0; i < MAX_WAIT_CNT; i++) {
4384                 udelay(100);
4385                 val = tr32(ofs);
4386                 if ((val & enable_bit) == 0)
4387                         break;
4388         }
4389
4390         if (i == MAX_WAIT_CNT && !silent) {
4391                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4392                        "ofs=%lx enable_bit=%x\n",
4393                        ofs, enable_bit);
4394                 return -ENODEV;
4395         }
4396
4397         return 0;
4398 }
4399
4400 /* tp->lock is held. */
4401 static int tg3_abort_hw(struct tg3 *tp, int silent)
4402 {
4403         int i, err;
4404
4405         tg3_disable_ints(tp);
4406
4407         tp->rx_mode &= ~RX_MODE_ENABLE;
4408         tw32_f(MAC_RX_MODE, tp->rx_mode);
4409         udelay(10);
4410
4411         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4412         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4413         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4414         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4415         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4416         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4417
4418         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4419         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4420         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4421         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4422         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4423         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4424         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4425
4426         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4427         tw32_f(MAC_MODE, tp->mac_mode);
4428         udelay(40);
4429
4430         tp->tx_mode &= ~TX_MODE_ENABLE;
4431         tw32_f(MAC_TX_MODE, tp->tx_mode);
4432
4433         for (i = 0; i < MAX_WAIT_CNT; i++) {
4434                 udelay(100);
4435                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4436                         break;
4437         }
4438         if (i >= MAX_WAIT_CNT) {
4439                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4440                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4441                        tp->dev->name, tr32(MAC_TX_MODE));
4442                 err |= -ENODEV;
4443         }
4444
4445         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4446         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4447         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4448
4449         tw32(FTQ_RESET, 0xffffffff);
4450         tw32(FTQ_RESET, 0x00000000);
4451
4452         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4453         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4454
4455         if (tp->hw_status)
4456                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4457         if (tp->hw_stats)
4458                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4459
4460         return err;
4461 }
4462
4463 /* tp->lock is held. */
4464 static int tg3_nvram_lock(struct tg3 *tp)
4465 {
4466         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4467                 int i;
4468
4469                 if (tp->nvram_lock_cnt == 0) {
4470                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4471                         for (i = 0; i < 8000; i++) {
4472                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4473                                         break;
4474                                 udelay(20);
4475                         }
4476                         if (i == 8000) {
4477                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4478                                 return -ENODEV;
4479                         }
4480                 }
4481                 tp->nvram_lock_cnt++;
4482         }
4483         return 0;
4484 }
4485
4486 /* tp->lock is held. */
4487 static void tg3_nvram_unlock(struct tg3 *tp)
4488 {
4489         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4490                 if (tp->nvram_lock_cnt > 0)
4491                         tp->nvram_lock_cnt--;
4492                 if (tp->nvram_lock_cnt == 0)
4493                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4494         }
4495 }
4496
4497 /* tp->lock is held. */
4498 static void tg3_enable_nvram_access(struct tg3 *tp)
4499 {
4500         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4501             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4502                 u32 nvaccess = tr32(NVRAM_ACCESS);
4503
4504                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4505         }
4506 }
4507
4508 /* tp->lock is held. */
4509 static void tg3_disable_nvram_access(struct tg3 *tp)
4510 {
4511         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4512             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4513                 u32 nvaccess = tr32(NVRAM_ACCESS);
4514
4515                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4516         }
4517 }
4518
4519 /* tp->lock is held. */
4520 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
4521 {
4522         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
4523                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
4524
4525         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4526                 switch (kind) {
4527                 case RESET_KIND_INIT:
4528                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4529                                       DRV_STATE_START);
4530                         break;
4531
4532                 case RESET_KIND_SHUTDOWN:
4533                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4534                                       DRV_STATE_UNLOAD);
4535                         break;
4536
4537                 case RESET_KIND_SUSPEND:
4538                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4539                                       DRV_STATE_SUSPEND);
4540                         break;
4541
4542                 default:
4543                         break;
4544                 };
4545         }
4546 }
4547
4548 /* tp->lock is held. */
4549 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
4550 {
4551         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4552                 switch (kind) {
4553                 case RESET_KIND_INIT:
4554                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4555                                       DRV_STATE_START_DONE);
4556                         break;
4557
4558                 case RESET_KIND_SHUTDOWN:
4559                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4560                                       DRV_STATE_UNLOAD_DONE);
4561                         break;
4562
4563                 default:
4564                         break;
4565                 };
4566         }
4567 }
4568
4569 /* tp->lock is held. */
4570 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
4571 {
4572         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4573                 switch (kind) {
4574                 case RESET_KIND_INIT:
4575                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4576                                       DRV_STATE_START);
4577                         break;
4578
4579                 case RESET_KIND_SHUTDOWN:
4580                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4581                                       DRV_STATE_UNLOAD);
4582                         break;
4583
4584                 case RESET_KIND_SUSPEND:
4585                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4586                                       DRV_STATE_SUSPEND);
4587                         break;
4588
4589                 default:
4590                         break;
4591                 };
4592         }
4593 }
4594
4595 static void tg3_stop_fw(struct tg3 *);
4596
4597 /* tp->lock is held. */
4598 static int tg3_chip_reset(struct tg3 *tp)
4599 {
4600         u32 val;
4601         void (*write_op)(struct tg3 *, u32, u32);
4602         int i;
4603
4604         tg3_nvram_lock(tp);
4605
4606         /* No matching tg3_nvram_unlock() after this because
4607          * chip reset below will undo the nvram lock.
4608          */
4609         tp->nvram_lock_cnt = 0;
4610
4611         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
4612             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
4613             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
4614                 tw32(GRC_FASTBOOT_PC, 0);
4615
4616         /*
4617          * We must avoid the readl() that normally takes place.
4618          * It locks machines, causes machine checks, and other
4619          * fun things.  So, temporarily disable the 5701
4620          * hardware workaround, while we do the reset.
4621          */
4622         write_op = tp->write32;
4623         if (write_op == tg3_write_flush_reg32)
4624                 tp->write32 = tg3_write32;
4625
4626         /* do the reset */
4627         val = GRC_MISC_CFG_CORECLK_RESET;
4628
4629         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4630                 if (tr32(0x7e2c) == 0x60) {
4631                         tw32(0x7e2c, 0x20);
4632                 }
4633                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4634                         tw32(GRC_MISC_CFG, (1 << 29));
4635                         val |= (1 << 29);
4636                 }
4637         }
4638
4639         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
4640                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
4641         tw32(GRC_MISC_CFG, val);
4642
4643         /* restore 5701 hardware bug workaround write method */
4644         tp->write32 = write_op;
4645
4646         /* Unfortunately, we have to delay before the PCI read back.
4647          * Some 575X chips even will not respond to a PCI cfg access
4648          * when the reset command is given to the chip.
4649          *
4650          * How do these hardware designers expect things to work
4651          * properly if the PCI write is posted for a long period
4652          * of time?  It is always necessary to have some method by
4653          * which a register read back can occur to push the write
4654          * out which does the reset.
4655          *
4656          * For most tg3 variants the trick below was working.
4657          * Ho hum...
4658          */
4659         udelay(120);
4660
4661         /* Flush PCI posted writes.  The normal MMIO registers
4662          * are inaccessible at this time so this is the only
4663          * way to make this reliably (actually, this is no longer
4664          * the case, see above).  I tried to use indirect
4665          * register read/write but this upset some 5701 variants.
4666          */
4667         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
4668
4669         udelay(120);
4670
4671         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4672                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
4673                         int i;
4674                         u32 cfg_val;
4675
4676                         /* Wait for link training to complete.  */
4677                         for (i = 0; i < 5000; i++)
4678                                 udelay(100);
4679
4680                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
4681                         pci_write_config_dword(tp->pdev, 0xc4,
4682                                                cfg_val | (1 << 15));
4683                 }
4684                 /* Set PCIE max payload size and clear error status.  */
4685                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
4686         }
4687
4688         /* Re-enable indirect register accesses. */
4689         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
4690                                tp->misc_host_ctrl);
4691
4692         /* Set MAX PCI retry to zero. */
4693         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
4694         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
4695             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
4696                 val |= PCISTATE_RETRY_SAME_DMA;
4697         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
4698
4699         pci_restore_state(tp->pdev);
4700
4701         /* Make sure PCI-X relaxed ordering bit is clear. */
4702         pci_read_config_dword(tp->pdev, TG3PCI_X_CAPS, &val);
4703         val &= ~PCIX_CAPS_RELAXED_ORDERING;
4704         pci_write_config_dword(tp->pdev, TG3PCI_X_CAPS, val);
4705
4706         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4707                 u32 val;
4708
4709                 /* Chip reset on 5780 will reset MSI enable bit,
4710                  * so need to restore it.
4711                  */
4712                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
4713                         u16 ctrl;
4714
4715                         pci_read_config_word(tp->pdev,
4716                                              tp->msi_cap + PCI_MSI_FLAGS,
4717                                              &ctrl);
4718                         pci_write_config_word(tp->pdev,
4719                                               tp->msi_cap + PCI_MSI_FLAGS,
4720                                               ctrl | PCI_MSI_FLAGS_ENABLE);
4721                         val = tr32(MSGINT_MODE);
4722                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
4723                 }
4724
4725                 val = tr32(MEMARB_MODE);
4726                 tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
4727
4728         } else
4729                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
4730
4731         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
4732                 tg3_stop_fw(tp);
4733                 tw32(0x5000, 0x400);
4734         }
4735
4736         tw32(GRC_MODE, tp->grc_mode);
4737
4738         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
4739                 u32 val = tr32(0xc4);
4740
4741                 tw32(0xc4, val | (1 << 15));
4742         }
4743
4744         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
4745             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
4746                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
4747                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
4748                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
4749                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
4750         }
4751
4752         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4753                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
4754                 tw32_f(MAC_MODE, tp->mac_mode);
4755         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4756                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
4757                 tw32_f(MAC_MODE, tp->mac_mode);
4758         } else
4759                 tw32_f(MAC_MODE, 0);
4760         udelay(40);
4761
4762         /* Wait for firmware initialization to complete. */
4763         for (i = 0; i < 100000; i++) {
4764                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
4765                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4766                         break;
4767                 udelay(10);
4768         }
4769
4770         /* Chip might not be fitted with firmare.  Some Sun onboard
4771          * parts are configured like that.  So don't signal the timeout
4772          * of the above loop as an error, but do report the lack of
4773          * running firmware once.
4774          */
4775         if (i >= 100000 &&
4776             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
4777                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
4778
4779                 printk(KERN_INFO PFX "%s: No firmware running.\n",
4780                        tp->dev->name);
4781         }
4782
4783         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
4784             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4785                 u32 val = tr32(0x7c00);
4786
4787                 tw32(0x7c00, val | (1 << 25));
4788         }
4789
4790         /* Reprobe ASF enable state.  */
4791         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
4792         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
4793         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
4794         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
4795                 u32 nic_cfg;
4796
4797                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
4798                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
4799                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
4800                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
4801                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
4802                 }
4803         }
4804
4805         return 0;
4806 }
4807
4808 /* tp->lock is held. */
4809 static void tg3_stop_fw(struct tg3 *tp)
4810 {
4811         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4812                 u32 val;
4813                 int i;
4814
4815                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
4816                 val = tr32(GRC_RX_CPU_EVENT);
4817                 val |= (1 << 14);
4818                 tw32(GRC_RX_CPU_EVENT, val);
4819
4820                 /* Wait for RX cpu to ACK the event.  */
4821                 for (i = 0; i < 100; i++) {
4822                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
4823                                 break;
4824                         udelay(1);
4825                 }
4826         }
4827 }
4828
4829 /* tp->lock is held. */
4830 static int tg3_halt(struct tg3 *tp, int kind, int silent)
4831 {
4832         int err;
4833
4834         tg3_stop_fw(tp);
4835
4836         tg3_write_sig_pre_reset(tp, kind);
4837
4838         tg3_abort_hw(tp, silent);
4839         err = tg3_chip_reset(tp);
4840
4841         tg3_write_sig_legacy(tp, kind);
4842         tg3_write_sig_post_reset(tp, kind);
4843
4844         if (err)
4845                 return err;
4846
4847         return 0;
4848 }
4849
4850 #define TG3_FW_RELEASE_MAJOR    0x0
4851 #define TG3_FW_RELASE_MINOR     0x0
4852 #define TG3_FW_RELEASE_FIX      0x0
4853 #define TG3_FW_START_ADDR       0x08000000
4854 #define TG3_FW_TEXT_ADDR        0x08000000
4855 #define TG3_FW_TEXT_LEN         0x9c0
4856 #define TG3_FW_RODATA_ADDR      0x080009c0
4857 #define TG3_FW_RODATA_LEN       0x60
4858 #define TG3_FW_DATA_ADDR        0x08000a40
4859 #define TG3_FW_DATA_LEN         0x20
4860 #define TG3_FW_SBSS_ADDR        0x08000a60
4861 #define TG3_FW_SBSS_LEN         0xc
4862 #define TG3_FW_BSS_ADDR         0x08000a70
4863 #define TG3_FW_BSS_LEN          0x10
4864
4865 static u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
4866         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
4867         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
4868         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
4869         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
4870         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
4871         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
4872         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
4873         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
4874         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
4875         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
4876         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
4877         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
4878         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
4879         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
4880         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
4881         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
4882         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
4883         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
4884         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
4885         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
4886         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
4887         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
4888         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
4889         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4890         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4891         0, 0, 0, 0, 0, 0,
4892         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
4893         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4894         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4895         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4896         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
4897         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
4898         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
4899         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
4900         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4901         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4902         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
4903         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4904         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4905         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4906         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
4907         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
4908         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
4909         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
4910         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
4911         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
4912         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
4913         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
4914         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
4915         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
4916         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
4917         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
4918         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
4919         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
4920         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
4921         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
4922         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
4923         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
4924         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
4925         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
4926         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
4927         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
4928         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
4929         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
4930         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
4931         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
4932         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
4933         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
4934         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
4935         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
4936         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
4937         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
4938         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
4939         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
4940         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
4941         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
4942         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
4943         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
4944         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
4945         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
4946         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
4947         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
4948         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
4949         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
4950         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
4951         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
4952         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
4953         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
4954         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
4955         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
4956         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
4957 };
4958
4959 static u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
4960         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
4961         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
4962         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
4963         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
4964         0x00000000
4965 };
4966
4967 #if 0 /* All zeros, don't eat up space with it. */
4968 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
4969         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
4970         0x00000000, 0x00000000, 0x00000000, 0x00000000
4971 };
4972 #endif
4973
4974 #define RX_CPU_SCRATCH_BASE     0x30000
4975 #define RX_CPU_SCRATCH_SIZE     0x04000
4976 #define TX_CPU_SCRATCH_BASE     0x34000
4977 #define TX_CPU_SCRATCH_SIZE     0x04000
4978
4979 /* tp->lock is held. */
4980 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
4981 {
4982         int i;
4983
4984         BUG_ON(offset == TX_CPU_BASE &&
4985             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
4986
4987         if (offset == RX_CPU_BASE) {
4988                 for (i = 0; i < 10000; i++) {
4989                         tw32(offset + CPU_STATE, 0xffffffff);
4990                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
4991                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
4992                                 break;
4993                 }
4994
4995                 tw32(offset + CPU_STATE, 0xffffffff);
4996                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
4997                 udelay(10);
4998         } else {
4999                 for (i = 0; i < 10000; i++) {
5000                         tw32(offset + CPU_STATE, 0xffffffff);
5001                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5002                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5003                                 break;
5004                 }
5005         }
5006
5007         if (i >= 10000) {
5008                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5009                        "and %s CPU\n",
5010                        tp->dev->name,
5011                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5012                 return -ENODEV;
5013         }
5014
5015         /* Clear firmware's nvram arbitration. */
5016         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5017                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5018         return 0;
5019 }
5020
5021 struct fw_info {
5022         unsigned int text_base;
5023         unsigned int text_len;
5024         u32 *text_data;
5025         unsigned int rodata_base;
5026         unsigned int rodata_len;
5027         u32 *rodata_data;
5028         unsigned int data_base;
5029         unsigned int data_len;
5030         u32 *data_data;
5031 };
5032
5033 /* tp->lock is held. */
5034 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5035                                  int cpu_scratch_size, struct fw_info *info)
5036 {
5037         int err, lock_err, i;
5038         void (*write_op)(struct tg3 *, u32, u32);
5039
5040         if (cpu_base == TX_CPU_BASE &&
5041             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5042                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5043                        "TX cpu firmware on %s which is 5705.\n",
5044                        tp->dev->name);
5045                 return -EINVAL;
5046         }
5047
5048         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5049                 write_op = tg3_write_mem;
5050         else
5051                 write_op = tg3_write_indirect_reg32;
5052
5053         /* It is possible that bootcode is still loading at this point.
5054          * Get the nvram lock first before halting the cpu.
5055          */
5056         lock_err = tg3_nvram_lock(tp);
5057         err = tg3_halt_cpu(tp, cpu_base);
5058         if (!lock_err)
5059                 tg3_nvram_unlock(tp);
5060         if (err)
5061                 goto out;
5062
5063         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5064                 write_op(tp, cpu_scratch_base + i, 0);
5065         tw32(cpu_base + CPU_STATE, 0xffffffff);
5066         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5067         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5068                 write_op(tp, (cpu_scratch_base +
5069                               (info->text_base & 0xffff) +
5070                               (i * sizeof(u32))),
5071                          (info->text_data ?
5072                           info->text_data[i] : 0));
5073         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5074                 write_op(tp, (cpu_scratch_base +
5075                               (info->rodata_base & 0xffff) +
5076                               (i * sizeof(u32))),
5077                          (info->rodata_data ?
5078                           info->rodata_data[i] : 0));
5079         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5080                 write_op(tp, (cpu_scratch_base +
5081                               (info->data_base & 0xffff) +
5082                               (i * sizeof(u32))),
5083                          (info->data_data ?
5084                           info->data_data[i] : 0));
5085
5086         err = 0;
5087
5088 out:
5089         return err;
5090 }
5091
5092 /* tp->lock is held. */
5093 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5094 {
5095         struct fw_info info;
5096         int err, i;
5097
5098         info.text_base = TG3_FW_TEXT_ADDR;
5099         info.text_len = TG3_FW_TEXT_LEN;
5100         info.text_data = &tg3FwText[0];
5101         info.rodata_base = TG3_FW_RODATA_ADDR;
5102         info.rodata_len = TG3_FW_RODATA_LEN;
5103         info.rodata_data = &tg3FwRodata[0];
5104         info.data_base = TG3_FW_DATA_ADDR;
5105         info.data_len = TG3_FW_DATA_LEN;
5106         info.data_data = NULL;
5107
5108         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5109                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5110                                     &info);
5111         if (err)
5112                 return err;
5113
5114         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5115                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5116                                     &info);
5117         if (err)
5118                 return err;
5119
5120         /* Now startup only the RX cpu. */
5121         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5122         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5123
5124         for (i = 0; i < 5; i++) {
5125                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5126                         break;
5127                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5128                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5129                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5130                 udelay(1000);
5131         }
5132         if (i >= 5) {
5133                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5134                        "to set RX CPU PC, is %08x should be %08x\n",
5135                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5136                        TG3_FW_TEXT_ADDR);
5137                 return -ENODEV;
5138         }
5139         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5140         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5141
5142         return 0;
5143 }
5144
5145 #if TG3_TSO_SUPPORT != 0
5146
5147 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5148 #define TG3_TSO_FW_RELASE_MINOR         0x6
5149 #define TG3_TSO_FW_RELEASE_FIX          0x0
5150 #define TG3_TSO_FW_START_ADDR           0x08000000
5151 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5152 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5153 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5154 #define TG3_TSO_FW_RODATA_LEN           0x60
5155 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5156 #define TG3_TSO_FW_DATA_LEN             0x30
5157 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5158 #define TG3_TSO_FW_SBSS_LEN             0x2c
5159 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5160 #define TG3_TSO_FW_BSS_LEN              0x894
5161
5162 static u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5163         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5164         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5165         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5166         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5167         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5168         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5169         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5170         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5171         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5172         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5173         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5174         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5175         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5176         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5177         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5178         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5179         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5180         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5181         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5182         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5183         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5184         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5185         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5186         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5187         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5188         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5189         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5190         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5191         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5192         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5193         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5194         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5195         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5196         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5197         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5198         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5199         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5200         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5201         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5202         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5203         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5204         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5205         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5206         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5207         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5208         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5209         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5210         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5211         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5212         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5213         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5214         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5215         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5216         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5217         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5218         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5219         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5220         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5221         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5222         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5223         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5224         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5225         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5226         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5227         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5228         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5229         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5230         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5231         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5232         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5233         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5234         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5235         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5236         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5237         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5238         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5239         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5240         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5241         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5242         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5243         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5244         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5245         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5246         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5247         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5248         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5249         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5250         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5251         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5252         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5253         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5254         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5255         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5256         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5257         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5258         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5259         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5260         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5261         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5262         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5263         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5264         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5265         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5266         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5267         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5268         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5269         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5270         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5271         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5272         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5273         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5274         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5275         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5276         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5277         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5278         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5279         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5280         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5281         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5282         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5283         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5284         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5285         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5286         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5287         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5288         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5289         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5290         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5291         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5292         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5293         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5294         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5295         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5296         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5297         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5298         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5299         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5300         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5301         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5302         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5303         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5304         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5305         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5306         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5307         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5308         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5309         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5310         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5311         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5312         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5313         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5314         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5315         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5316         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5317         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5318         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5319         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5320         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5321         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5322         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5323         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5324         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5325         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5326         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5327         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5328         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5329         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5330         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5331         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5332         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5333         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5334         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5335         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5336         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5337         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5338         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5339         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5340         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5341         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5342         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5343         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5344         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5345         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5346         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5347         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5348         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5349         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5350         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5351         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5352         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5353         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5354         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5355         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5356         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5357         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5358         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5359         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5360         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5361         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5362         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5363         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5364         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5365         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5366         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5367         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5368         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5369         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5370         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5371         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5372         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5373         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5374         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5375         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5376         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5377         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5378         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5379         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5380         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5381         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5382         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5383         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5384         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5385         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5386         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5387         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5388         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5389         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5390         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
5391         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
5392         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
5393         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
5394         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
5395         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
5396         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
5397         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
5398         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
5399         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
5400         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5401         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
5402         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
5403         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
5404         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
5405         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
5406         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
5407         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
5408         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
5409         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
5410         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
5411         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
5412         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
5413         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
5414         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
5415         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
5416         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
5417         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
5418         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
5419         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
5420         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
5421         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
5422         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
5423         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
5424         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
5425         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
5426         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
5427         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5428         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
5429         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
5430         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
5431         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
5432         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
5433         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
5434         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
5435         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
5436         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
5437         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
5438         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
5439         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
5440         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
5441         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
5442         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
5443         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
5444         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
5445         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
5446         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
5447 };
5448
5449 static u32 tg3TsoFwRodata[] = {
5450         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5451         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
5452         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
5453         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
5454         0x00000000,
5455 };
5456
5457 static u32 tg3TsoFwData[] = {
5458         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
5459         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5460         0x00000000,
5461 };
5462
5463 /* 5705 needs a special version of the TSO firmware.  */
5464 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
5465 #define TG3_TSO5_FW_RELASE_MINOR        0x2
5466 #define TG3_TSO5_FW_RELEASE_FIX         0x0
5467 #define TG3_TSO5_FW_START_ADDR          0x00010000
5468 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
5469 #define TG3_TSO5_FW_TEXT_LEN            0xe90
5470 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
5471 #define TG3_TSO5_FW_RODATA_LEN          0x50
5472 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
5473 #define TG3_TSO5_FW_DATA_LEN            0x20
5474 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
5475 #define TG3_TSO5_FW_SBSS_LEN            0x28
5476 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
5477 #define TG3_TSO5_FW_BSS_LEN             0x88
5478
5479 static u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
5480         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
5481         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
5482         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5483         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
5484         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
5485         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
5486         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5487         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
5488         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
5489         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
5490         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
5491         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
5492         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
5493         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
5494         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
5495         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
5496         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
5497         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
5498         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
5499         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
5500         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
5501         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
5502         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
5503         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
5504         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
5505         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
5506         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
5507         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
5508         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
5509         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
5510         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5511         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
5512         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
5513         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
5514         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
5515         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
5516         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
5517         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
5518         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
5519         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
5520         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
5521         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
5522         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
5523         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
5524         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
5525         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
5526         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
5527         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
5528         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
5529         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
5530         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
5531         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
5532         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
5533         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
5534         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
5535         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
5536         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
5537         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
5538         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
5539         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
5540         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
5541         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
5542         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
5543         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
5544         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
5545         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
5546         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5547         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
5548         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
5549         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
5550         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
5551         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
5552         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
5553         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
5554         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
5555         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
5556         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
5557         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
5558         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
5559         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
5560         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
5561         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
5562         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
5563         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
5564         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
5565         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
5566         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
5567         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
5568         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
5569         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
5570         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
5571         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
5572         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
5573         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
5574         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
5575         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
5576         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
5577         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
5578         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
5579         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
5580         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
5581         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
5582         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
5583         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
5584         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
5585         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
5586         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5587         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5588         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
5589         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
5590         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
5591         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
5592         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
5593         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
5594         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
5595         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
5596         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
5597         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5598         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5599         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
5600         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
5601         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
5602         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
5603         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5604         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
5605         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
5606         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
5607         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
5608         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
5609         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
5610         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
5611         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
5612         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
5613         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
5614         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
5615         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
5616         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
5617         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
5618         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
5619         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
5620         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
5621         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
5622         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
5623         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
5624         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
5625         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
5626         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
5627         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5628         0x00804821, 0x8faa0010, 0x3c020001, 0x8c420f3c, 0x3c040001, 0x8c840f44,
5629         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010001, 0xac230f3c, 0x14400003,
5630         0x00004021, 0x3c010001, 0xac200f3c, 0x3c020001, 0x8c420f3c, 0x3c030001,
5631         0x8c630f40, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5632         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020001, 0x8c420f3c,
5633         0x3c030001, 0x8c630f40, 0x8f64680c, 0x00021140, 0x00431021, 0xac440008,
5634         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5635         0x00000000, 0x00000000, 0x00000000,
5636 };
5637
5638 static u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
5639         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5640         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
5641         0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5642         0x00000000, 0x00000000, 0x00000000,
5643 };
5644
5645 static u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
5646         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
5647         0x00000000, 0x00000000, 0x00000000,
5648 };
5649
5650 /* tp->lock is held. */
5651 static int tg3_load_tso_firmware(struct tg3 *tp)
5652 {
5653         struct fw_info info;
5654         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
5655         int err, i;
5656
5657         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
5658                 return 0;
5659
5660         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5661                 info.text_base = TG3_TSO5_FW_TEXT_ADDR;
5662                 info.text_len = TG3_TSO5_FW_TEXT_LEN;
5663                 info.text_data = &tg3Tso5FwText[0];
5664                 info.rodata_base = TG3_TSO5_FW_RODATA_ADDR;
5665                 info.rodata_len = TG3_TSO5_FW_RODATA_LEN;
5666                 info.rodata_data = &tg3Tso5FwRodata[0];
5667                 info.data_base = TG3_TSO5_FW_DATA_ADDR;
5668                 info.data_len = TG3_TSO5_FW_DATA_LEN;
5669                 info.data_data = &tg3Tso5FwData[0];
5670                 cpu_base = RX_CPU_BASE;
5671                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
5672                 cpu_scratch_size = (info.text_len +
5673                                     info.rodata_len +
5674                                     info.data_len +
5675                                     TG3_TSO5_FW_SBSS_LEN +
5676                                     TG3_TSO5_FW_BSS_LEN);
5677         } else {
5678                 info.text_base = TG3_TSO_FW_TEXT_ADDR;
5679                 info.text_len = TG3_TSO_FW_TEXT_LEN;
5680                 info.text_data = &tg3TsoFwText[0];
5681                 info.rodata_base = TG3_TSO_FW_RODATA_ADDR;
5682                 info.rodata_len = TG3_TSO_FW_RODATA_LEN;
5683                 info.rodata_data = &tg3TsoFwRodata[0];
5684                 info.data_base = TG3_TSO_FW_DATA_ADDR;
5685                 info.data_len = TG3_TSO_FW_DATA_LEN;
5686                 info.data_data = &tg3TsoFwData[0];
5687                 cpu_base = TX_CPU_BASE;
5688                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
5689                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
5690         }
5691
5692         err = tg3_load_firmware_cpu(tp, cpu_base,
5693                                     cpu_scratch_base, cpu_scratch_size,
5694                                     &info);
5695         if (err)
5696                 return err;
5697
5698         /* Now startup the cpu. */
5699         tw32(cpu_base + CPU_STATE, 0xffffffff);
5700         tw32_f(cpu_base + CPU_PC,    info.text_base);
5701
5702         for (i = 0; i < 5; i++) {
5703                 if (tr32(cpu_base + CPU_PC) == info.text_base)
5704                         break;
5705                 tw32(cpu_base + CPU_STATE, 0xffffffff);
5706                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
5707                 tw32_f(cpu_base + CPU_PC,    info.text_base);
5708                 udelay(1000);
5709         }
5710         if (i >= 5) {
5711                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
5712                        "to set CPU PC, is %08x should be %08x\n",
5713                        tp->dev->name, tr32(cpu_base + CPU_PC),
5714                        info.text_base);
5715                 return -ENODEV;
5716         }
5717         tw32(cpu_base + CPU_STATE, 0xffffffff);
5718         tw32_f(cpu_base + CPU_MODE,  0x00000000);
5719         return 0;
5720 }
5721
5722 #endif /* TG3_TSO_SUPPORT != 0 */
5723
5724 /* tp->lock is held. */
5725 static void __tg3_set_mac_addr(struct tg3 *tp)
5726 {
5727         u32 addr_high, addr_low;
5728         int i;
5729
5730         addr_high = ((tp->dev->dev_addr[0] << 8) |
5731                      tp->dev->dev_addr[1]);
5732         addr_low = ((tp->dev->dev_addr[2] << 24) |
5733                     (tp->dev->dev_addr[3] << 16) |
5734                     (tp->dev->dev_addr[4] <<  8) |
5735                     (tp->dev->dev_addr[5] <<  0));
5736         for (i = 0; i < 4; i++) {
5737                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
5738                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
5739         }
5740
5741         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
5742             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
5743                 for (i = 0; i < 12; i++) {
5744                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
5745                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
5746                 }
5747         }
5748
5749         addr_high = (tp->dev->dev_addr[0] +
5750                      tp->dev->dev_addr[1] +
5751                      tp->dev->dev_addr[2] +
5752                      tp->dev->dev_addr[3] +
5753                      tp->dev->dev_addr[4] +
5754                      tp->dev->dev_addr[5]) &
5755                 TX_BACKOFF_SEED_MASK;
5756         tw32(MAC_TX_BACKOFF_SEED, addr_high);
5757 }
5758
5759 static int tg3_set_mac_addr(struct net_device *dev, void *p)
5760 {
5761         struct tg3 *tp = netdev_priv(dev);
5762         struct sockaddr *addr = p;
5763
5764         if (!is_valid_ether_addr(addr->sa_data))
5765                 return -EINVAL;
5766
5767         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5768
5769         if (!netif_running(dev))
5770                 return 0;
5771
5772         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
5773                 /* Reset chip so that ASF can re-init any MAC addresses it
5774                  * needs.
5775                  */
5776                 tg3_netif_stop(tp);
5777                 tg3_full_lock(tp, 1);
5778
5779                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5780                 tg3_init_hw(tp, 0);
5781
5782                 tg3_netif_start(tp);
5783                 tg3_full_unlock(tp);
5784         } else {
5785                 spin_lock_bh(&tp->lock);
5786                 __tg3_set_mac_addr(tp);
5787                 spin_unlock_bh(&tp->lock);
5788         }
5789
5790         return 0;
5791 }
5792
5793 /* tp->lock is held. */
5794 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
5795                            dma_addr_t mapping, u32 maxlen_flags,
5796                            u32 nic_addr)
5797 {
5798         tg3_write_mem(tp,
5799                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
5800                       ((u64) mapping >> 32));
5801         tg3_write_mem(tp,
5802                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
5803                       ((u64) mapping & 0xffffffff));
5804         tg3_write_mem(tp,
5805                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
5806                        maxlen_flags);
5807
5808         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
5809                 tg3_write_mem(tp,
5810                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
5811                               nic_addr);
5812 }
5813
5814 static void __tg3_set_rx_mode(struct net_device *);
5815 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
5816 {
5817         tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
5818         tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
5819         tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
5820         tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
5821         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5822                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
5823                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
5824         }
5825         tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
5826         tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
5827         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5828                 u32 val = ec->stats_block_coalesce_usecs;
5829
5830                 if (!netif_carrier_ok(tp->dev))
5831                         val = 0;
5832
5833                 tw32(HOSTCC_STAT_COAL_TICKS, val);
5834         }
5835 }
5836
5837 /* tp->lock is held. */
5838 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
5839 {
5840         u32 val, rdmac_mode;
5841         int i, err, limit;
5842
5843         tg3_disable_ints(tp);
5844
5845         tg3_stop_fw(tp);
5846
5847         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
5848
5849         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
5850                 tg3_abort_hw(tp, 1);
5851         }
5852
5853         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) && reset_phy)
5854                 tg3_phy_reset(tp);
5855
5856         err = tg3_chip_reset(tp);
5857         if (err)
5858                 return err;
5859
5860         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
5861
5862         /* This works around an issue with Athlon chipsets on
5863          * B3 tigon3 silicon.  This bit has no effect on any
5864          * other revision.  But do not set this on PCI Express
5865          * chips.
5866          */
5867         if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
5868                 tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
5869         tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
5870
5871         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
5872             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
5873                 val = tr32(TG3PCI_PCISTATE);
5874                 val |= PCISTATE_RETRY_SAME_DMA;
5875                 tw32(TG3PCI_PCISTATE, val);
5876         }
5877
5878         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
5879                 /* Enable some hw fixes.  */
5880                 val = tr32(TG3PCI_MSI_DATA);
5881                 val |= (1 << 26) | (1 << 28) | (1 << 29);
5882                 tw32(TG3PCI_MSI_DATA, val);
5883         }
5884
5885         /* Descriptor ring init may make accesses to the
5886          * NIC SRAM area to setup the TX descriptors, so we
5887          * can only do this after the hardware has been
5888          * successfully reset.
5889          */
5890         tg3_init_rings(tp);
5891
5892         /* This value is determined during the probe time DMA
5893          * engine test, tg3_test_dma.
5894          */
5895         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
5896
5897         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
5898                           GRC_MODE_4X_NIC_SEND_RINGS |
5899                           GRC_MODE_NO_TX_PHDR_CSUM |
5900                           GRC_MODE_NO_RX_PHDR_CSUM);
5901         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
5902
5903         /* Pseudo-header checksum is done by hardware logic and not
5904          * the offload processers, so make the chip do the pseudo-
5905          * header checksums on receive.  For transmit it is more
5906          * convenient to do the pseudo-header checksum in software
5907          * as Linux does that on transmit for us in all cases.
5908          */
5909         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
5910
5911         tw32(GRC_MODE,
5912              tp->grc_mode |
5913              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
5914
5915         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
5916         val = tr32(GRC_MISC_CFG);
5917         val &= ~0xff;
5918         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
5919         tw32(GRC_MISC_CFG, val);
5920
5921         /* Initialize MBUF/DESC pool. */
5922         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
5923                 /* Do nothing.  */
5924         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
5925                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
5926                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
5927                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
5928                 else
5929                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
5930                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
5931                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
5932         }
5933 #if TG3_TSO_SUPPORT != 0
5934         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
5935                 int fw_len;
5936
5937                 fw_len = (TG3_TSO5_FW_TEXT_LEN +
5938                           TG3_TSO5_FW_RODATA_LEN +
5939                           TG3_TSO5_FW_DATA_LEN +
5940                           TG3_TSO5_FW_SBSS_LEN +
5941                           TG3_TSO5_FW_BSS_LEN);
5942                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
5943                 tw32(BUFMGR_MB_POOL_ADDR,
5944                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
5945                 tw32(BUFMGR_MB_POOL_SIZE,
5946                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
5947         }
5948 #endif
5949
5950         if (tp->dev->mtu <= ETH_DATA_LEN) {
5951                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
5952                      tp->bufmgr_config.mbuf_read_dma_low_water);
5953                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
5954                      tp->bufmgr_config.mbuf_mac_rx_low_water);
5955                 tw32(BUFMGR_MB_HIGH_WATER,
5956                      tp->bufmgr_config.mbuf_high_water);
5957         } else {
5958                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
5959                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
5960                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
5961                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
5962                 tw32(BUFMGR_MB_HIGH_WATER,
5963                      tp->bufmgr_config.mbuf_high_water_jumbo);
5964         }
5965         tw32(BUFMGR_DMA_LOW_WATER,
5966              tp->bufmgr_config.dma_low_water);
5967         tw32(BUFMGR_DMA_HIGH_WATER,
5968              tp->bufmgr_config.dma_high_water);
5969
5970         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
5971         for (i = 0; i < 2000; i++) {
5972                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
5973                         break;
5974                 udelay(10);
5975         }
5976         if (i >= 2000) {
5977                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
5978                        tp->dev->name);
5979                 return -ENODEV;
5980         }
5981
5982         /* Setup replenish threshold. */
5983         tw32(RCVBDI_STD_THRESH, tp->rx_pending / 8);
5984
5985         /* Initialize TG3_BDINFO's at:
5986          *  RCVDBDI_STD_BD:     standard eth size rx ring
5987          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
5988          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
5989          *
5990          * like so:
5991          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
5992          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
5993          *                              ring attribute flags
5994          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
5995          *
5996          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
5997          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
5998          *
5999          * The size of each ring is fixed in the firmware, but the location is
6000          * configurable.
6001          */
6002         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6003              ((u64) tp->rx_std_mapping >> 32));
6004         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6005              ((u64) tp->rx_std_mapping & 0xffffffff));
6006         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
6007              NIC_SRAM_RX_BUFFER_DESC);
6008
6009         /* Don't even try to program the JUMBO/MINI buffer descriptor
6010          * configs on 5705.
6011          */
6012         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6013                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6014                      RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT);
6015         } else {
6016                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6017                      RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6018
6019                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
6020                      BDINFO_FLAGS_DISABLED);
6021
6022                 /* Setup replenish threshold. */
6023                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
6024
6025                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
6026                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6027                              ((u64) tp->rx_jumbo_mapping >> 32));
6028                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6029                              ((u64) tp->rx_jumbo_mapping & 0xffffffff));
6030                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6031                              RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6032                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
6033                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
6034                 } else {
6035                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6036                              BDINFO_FLAGS_DISABLED);
6037                 }
6038
6039         }
6040
6041         /* There is only one send ring on 5705/5750, no need to explicitly
6042          * disable the others.
6043          */
6044         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6045                 /* Clear out send RCB ring in SRAM. */
6046                 for (i = NIC_SRAM_SEND_RCB; i < NIC_SRAM_RCV_RET_RCB; i += TG3_BDINFO_SIZE)
6047                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6048                                       BDINFO_FLAGS_DISABLED);
6049         }
6050
6051         tp->tx_prod = 0;
6052         tp->tx_cons = 0;
6053         tw32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6054         tw32_tx_mbox(MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6055
6056         tg3_set_bdinfo(tp, NIC_SRAM_SEND_RCB,
6057                        tp->tx_desc_mapping,
6058                        (TG3_TX_RING_SIZE <<
6059                         BDINFO_FLAGS_MAXLEN_SHIFT),
6060                        NIC_SRAM_TX_BUFFER_DESC);
6061
6062         /* There is only one receive return ring on 5705/5750, no need
6063          * to explicitly disable the others.
6064          */
6065         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6066                 for (i = NIC_SRAM_RCV_RET_RCB; i < NIC_SRAM_STATS_BLK;
6067                      i += TG3_BDINFO_SIZE) {
6068                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6069                                       BDINFO_FLAGS_DISABLED);
6070                 }
6071         }
6072
6073         tp->rx_rcb_ptr = 0;
6074         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, 0);
6075
6076         tg3_set_bdinfo(tp, NIC_SRAM_RCV_RET_RCB,
6077                        tp->rx_rcb_mapping,
6078                        (TG3_RX_RCB_RING_SIZE(tp) <<
6079                         BDINFO_FLAGS_MAXLEN_SHIFT),
6080                        0);
6081
6082         tp->rx_std_ptr = tp->rx_pending;
6083         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
6084                      tp->rx_std_ptr);
6085
6086         tp->rx_jumbo_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
6087                                                 tp->rx_jumbo_pending : 0;
6088         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
6089                      tp->rx_jumbo_ptr);
6090
6091         /* Initialize MAC address and backoff seed. */
6092         __tg3_set_mac_addr(tp);
6093
6094         /* MTU + ethernet header + FCS + optional VLAN tag */
6095         tw32(MAC_RX_MTU_SIZE, tp->dev->mtu + ETH_HLEN + 8);
6096
6097         /* The slot time is changed by tg3_setup_phy if we
6098          * run at gigabit with half duplex.
6099          */
6100         tw32(MAC_TX_LENGTHS,
6101              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6102              (6 << TX_LENGTHS_IPG_SHIFT) |
6103              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6104
6105         /* Receive rules. */
6106         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
6107         tw32(RCVLPC_CONFIG, 0x0181);
6108
6109         /* Calculate RDMAC_MODE setting early, we need it to determine
6110          * the RCVLPC_STATE_ENABLE mask.
6111          */
6112         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
6113                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
6114                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
6115                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
6116                       RDMAC_MODE_LNGREAD_ENAB);
6117         if (tp->tg3_flags & TG3_FLAG_SPLIT_MODE)
6118                 rdmac_mode |= RDMAC_MODE_SPLIT_ENABLE;
6119
6120         /* If statement applies to 5705 and 5750 PCI devices only */
6121         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6122              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6123             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
6124                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
6125                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6126                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6127                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
6128                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6129                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
6130                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6131                 }
6132         }
6133
6134         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6135                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6136
6137 #if TG3_TSO_SUPPORT != 0
6138         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6139                 rdmac_mode |= (1 << 27);
6140 #endif
6141
6142         /* Receive/send statistics. */
6143         if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
6144             (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
6145                 val = tr32(RCVLPC_STATS_ENABLE);
6146                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
6147                 tw32(RCVLPC_STATS_ENABLE, val);
6148         } else {
6149                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
6150         }
6151         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
6152         tw32(SNDDATAI_STATSENAB, 0xffffff);
6153         tw32(SNDDATAI_STATSCTRL,
6154              (SNDDATAI_SCTRL_ENABLE |
6155               SNDDATAI_SCTRL_FASTUPD));
6156
6157         /* Setup host coalescing engine. */
6158         tw32(HOSTCC_MODE, 0);
6159         for (i = 0; i < 2000; i++) {
6160                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
6161                         break;
6162                 udelay(10);
6163         }
6164
6165         __tg3_set_coalesce(tp, &tp->coal);
6166
6167         /* set status block DMA address */
6168         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6169              ((u64) tp->status_mapping >> 32));
6170         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6171              ((u64) tp->status_mapping & 0xffffffff));
6172
6173         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6174                 /* Status/statistics block address.  See tg3_timer,
6175                  * the tg3_periodic_fetch_stats call there, and
6176                  * tg3_get_stats to see how this works for 5705/5750 chips.
6177                  */
6178                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6179                      ((u64) tp->stats_mapping >> 32));
6180                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6181                      ((u64) tp->stats_mapping & 0xffffffff));
6182                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
6183                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
6184         }
6185
6186         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
6187
6188         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
6189         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
6190         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6191                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
6192
6193         /* Clear statistics/status block in chip, and status block in ram. */
6194         for (i = NIC_SRAM_STATS_BLK;
6195              i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
6196              i += sizeof(u32)) {
6197                 tg3_write_mem(tp, i, 0);
6198                 udelay(40);
6199         }
6200         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
6201
6202         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6203                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
6204                 /* reset to prevent losing 1st rx packet intermittently */
6205                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6206                 udelay(10);
6207         }
6208
6209         tp->mac_mode = MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
6210                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
6211         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
6212         udelay(40);
6213
6214         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
6215          * If TG3_FLAG_EEPROM_WRITE_PROT is set, we should read the
6216          * register to preserve the GPIO settings for LOMs. The GPIOs,
6217          * whether used as inputs or outputs, are set by boot code after
6218          * reset.
6219          */
6220         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
6221                 u32 gpio_mask;
6222
6223                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE2 |
6224                             GRC_LCLCTRL_GPIO_OUTPUT0 | GRC_LCLCTRL_GPIO_OUTPUT2;
6225
6226                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
6227                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
6228                                      GRC_LCLCTRL_GPIO_OUTPUT3;
6229
6230                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6231                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
6232
6233                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
6234
6235                 /* GPIO1 must be driven high for eeprom write protect */
6236                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
6237                                        GRC_LCLCTRL_GPIO_OUTPUT1);
6238         }
6239         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6240         udelay(100);
6241
6242         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0);
6243         tp->last_tag = 0;
6244
6245         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6246                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
6247                 udelay(40);
6248         }
6249
6250         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
6251                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
6252                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
6253                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
6254                WDMAC_MODE_LNGREAD_ENAB);
6255
6256         /* If statement applies to 5705 and 5750 PCI devices only */
6257         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6258              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6259             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
6260                 if ((tp->tg3_flags & TG3_FLG2_TSO_CAPABLE) &&
6261                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6262                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6263                         /* nothing */
6264                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6265                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
6266                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
6267                         val |= WDMAC_MODE_RX_ACCEL;
6268                 }
6269         }
6270
6271         /* Enable host coalescing bug fix */
6272         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755) ||
6273             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787))
6274                 val |= (1 << 29);
6275
6276         tw32_f(WDMAC_MODE, val);
6277         udelay(40);
6278
6279         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0) {
6280                 val = tr32(TG3PCI_X_CAPS);
6281                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
6282                         val &= ~PCIX_CAPS_BURST_MASK;
6283                         val |= (PCIX_CAPS_MAX_BURST_CPIOB << PCIX_CAPS_BURST_SHIFT);
6284                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6285                         val &= ~(PCIX_CAPS_SPLIT_MASK | PCIX_CAPS_BURST_MASK);
6286                         val |= (PCIX_CAPS_MAX_BURST_CPIOB << PCIX_CAPS_BURST_SHIFT);
6287                         if (tp->tg3_flags & TG3_FLAG_SPLIT_MODE)
6288                                 val |= (tp->split_mode_max_reqs <<
6289                                         PCIX_CAPS_SPLIT_SHIFT);
6290                 }
6291                 tw32(TG3PCI_X_CAPS, val);
6292         }
6293
6294         tw32_f(RDMAC_MODE, rdmac_mode);
6295         udelay(40);
6296
6297         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
6298         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6299                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
6300         tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
6301         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
6302         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
6303         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
6304         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
6305 #if TG3_TSO_SUPPORT != 0
6306         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6307                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
6308 #endif
6309         tw32(SNDBDI_MODE, SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE);
6310         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
6311
6312         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
6313                 err = tg3_load_5701_a0_firmware_fix(tp);
6314                 if (err)
6315                         return err;
6316         }
6317
6318 #if TG3_TSO_SUPPORT != 0
6319         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6320                 err = tg3_load_tso_firmware(tp);
6321                 if (err)
6322                         return err;
6323         }
6324 #endif
6325
6326         tp->tx_mode = TX_MODE_ENABLE;
6327         tw32_f(MAC_TX_MODE, tp->tx_mode);
6328         udelay(100);
6329
6330         tp->rx_mode = RX_MODE_ENABLE;
6331         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6332                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
6333
6334         tw32_f(MAC_RX_MODE, tp->rx_mode);
6335         udelay(10);
6336
6337         if (tp->link_config.phy_is_low_power) {
6338                 tp->link_config.phy_is_low_power = 0;
6339                 tp->link_config.speed = tp->link_config.orig_speed;
6340                 tp->link_config.duplex = tp->link_config.orig_duplex;
6341                 tp->link_config.autoneg = tp->link_config.orig_autoneg;
6342         }
6343
6344         tp->mi_mode = MAC_MI_MODE_BASE;
6345         tw32_f(MAC_MI_MODE, tp->mi_mode);
6346         udelay(80);
6347
6348         tw32(MAC_LED_CTRL, tp->led_ctrl);
6349
6350         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
6351         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6352                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6353                 udelay(10);
6354         }
6355         tw32_f(MAC_RX_MODE, tp->rx_mode);
6356         udelay(10);
6357
6358         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6359                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
6360                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
6361                         /* Set drive transmission level to 1.2V  */
6362                         /* only if the signal pre-emphasis bit is not set  */
6363                         val = tr32(MAC_SERDES_CFG);
6364                         val &= 0xfffff000;
6365                         val |= 0x880;
6366                         tw32(MAC_SERDES_CFG, val);
6367                 }
6368                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
6369                         tw32(MAC_SERDES_CFG, 0x616000);
6370         }
6371
6372         /* Prevent chip from dropping frames when flow control
6373          * is enabled.
6374          */
6375         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
6376
6377         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
6378             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
6379                 /* Use hardware link auto-negotiation */
6380                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
6381         }
6382
6383         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
6384             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
6385                 u32 tmp;
6386
6387                 tmp = tr32(SERDES_RX_CTRL);
6388                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
6389                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
6390                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
6391                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6392         }
6393
6394         err = tg3_setup_phy(tp, reset_phy);
6395         if (err)
6396                 return err;
6397
6398         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
6399                 u32 tmp;
6400
6401                 /* Clear CRC stats. */
6402                 if (!tg3_readphy(tp, 0x1e, &tmp)) {
6403                         tg3_writephy(tp, 0x1e, tmp | 0x8000);
6404                         tg3_readphy(tp, 0x14, &tmp);
6405                 }
6406         }
6407
6408         __tg3_set_rx_mode(tp->dev);
6409
6410         /* Initialize receive rules. */
6411         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
6412         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
6413         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
6414         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
6415
6416         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6417             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
6418                 limit = 8;
6419         else
6420                 limit = 16;
6421         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
6422                 limit -= 4;
6423         switch (limit) {
6424         case 16:
6425                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
6426         case 15:
6427                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
6428         case 14:
6429                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
6430         case 13:
6431                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
6432         case 12:
6433                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
6434         case 11:
6435                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
6436         case 10:
6437                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
6438         case 9:
6439                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
6440         case 8:
6441                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
6442         case 7:
6443                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
6444         case 6:
6445                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
6446         case 5:
6447                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
6448         case 4:
6449                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
6450         case 3:
6451                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
6452         case 2:
6453         case 1:
6454
6455         default:
6456                 break;
6457         };
6458
6459         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
6460
6461         return 0;
6462 }
6463
6464 /* Called at device open time to get the chip ready for
6465  * packet processing.  Invoked with tp->lock held.
6466  */
6467 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
6468 {
6469         int err;
6470
6471         /* Force the chip into D0. */
6472         err = tg3_set_power_state(tp, PCI_D0);
6473         if (err)
6474                 goto out;
6475
6476         tg3_switch_clocks(tp);
6477
6478         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
6479
6480         err = tg3_reset_hw(tp, reset_phy);
6481
6482 out:
6483         return err;
6484 }
6485
6486 #define TG3_STAT_ADD32(PSTAT, REG) \
6487 do {    u32 __val = tr32(REG); \
6488         (PSTAT)->low += __val; \
6489         if ((PSTAT)->low < __val) \
6490                 (PSTAT)->high += 1; \
6491 } while (0)
6492
6493 static void tg3_periodic_fetch_stats(struct tg3 *tp)
6494 {
6495         struct tg3_hw_stats *sp = tp->hw_stats;
6496
6497         if (!netif_carrier_ok(tp->dev))
6498                 return;
6499
6500         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
6501         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
6502         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
6503         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
6504         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
6505         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
6506         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
6507         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
6508         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
6509         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
6510         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
6511         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
6512         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
6513
6514         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
6515         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
6516         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
6517         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
6518         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
6519         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
6520         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
6521         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
6522         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
6523         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
6524         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
6525         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
6526         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
6527         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
6528
6529         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
6530         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
6531         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
6532 }
6533
6534 static void tg3_timer(unsigned long __opaque)
6535 {
6536         struct tg3 *tp = (struct tg3 *) __opaque;
6537
6538         if (tp->irq_sync)
6539                 goto restart_timer;
6540
6541         spin_lock(&tp->lock);
6542
6543         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
6544                 /* All of this garbage is because when using non-tagged
6545                  * IRQ status the mailbox/status_block protocol the chip
6546                  * uses with the cpu is race prone.
6547                  */
6548                 if (tp->hw_status->status & SD_STATUS_UPDATED) {
6549                         tw32(GRC_LOCAL_CTRL,
6550                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
6551                 } else {
6552                         tw32(HOSTCC_MODE, tp->coalesce_mode |
6553                              (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
6554                 }
6555
6556                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
6557                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
6558                         spin_unlock(&tp->lock);
6559                         schedule_work(&tp->reset_task);
6560                         return;
6561                 }
6562         }
6563
6564         /* This part only runs once per second. */
6565         if (!--tp->timer_counter) {
6566                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
6567                         tg3_periodic_fetch_stats(tp);
6568
6569                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
6570                         u32 mac_stat;
6571                         int phy_event;
6572
6573                         mac_stat = tr32(MAC_STATUS);
6574
6575                         phy_event = 0;
6576                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
6577                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
6578                                         phy_event = 1;
6579                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
6580                                 phy_event = 1;
6581
6582                         if (phy_event)
6583                                 tg3_setup_phy(tp, 0);
6584                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
6585                         u32 mac_stat = tr32(MAC_STATUS);
6586                         int need_setup = 0;
6587
6588                         if (netif_carrier_ok(tp->dev) &&
6589                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
6590                                 need_setup = 1;
6591                         }
6592                         if (! netif_carrier_ok(tp->dev) &&
6593                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
6594                                          MAC_STATUS_SIGNAL_DET))) {
6595                                 need_setup = 1;
6596                         }
6597                         if (need_setup) {
6598                                 tw32_f(MAC_MODE,
6599                                      (tp->mac_mode &
6600                                       ~MAC_MODE_PORT_MODE_MASK));
6601                                 udelay(40);
6602                                 tw32_f(MAC_MODE, tp->mac_mode);
6603                                 udelay(40);
6604                                 tg3_setup_phy(tp, 0);
6605                         }
6606                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
6607                         tg3_serdes_parallel_detect(tp);
6608
6609                 tp->timer_counter = tp->timer_multiplier;
6610         }
6611
6612         /* Heartbeat is only sent once every 2 seconds.  */
6613         if (!--tp->asf_counter) {
6614                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6615                         u32 val;
6616
6617                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
6618                                       FWCMD_NICDRV_ALIVE2);
6619                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
6620                         /* 5 seconds timeout */
6621                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
6622                         val = tr32(GRC_RX_CPU_EVENT);
6623                         val |= (1 << 14);
6624                         tw32(GRC_RX_CPU_EVENT, val);
6625                 }
6626                 tp->asf_counter = tp->asf_multiplier;
6627         }
6628
6629         spin_unlock(&tp->lock);
6630
6631 restart_timer:
6632         tp->timer.expires = jiffies + tp->timer_offset;
6633         add_timer(&tp->timer);
6634 }
6635
6636 static int tg3_request_irq(struct tg3 *tp)
6637 {
6638         irqreturn_t (*fn)(int, void *, struct pt_regs *);
6639         unsigned long flags;
6640         struct net_device *dev = tp->dev;
6641
6642         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6643                 fn = tg3_msi;
6644                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
6645                         fn = tg3_msi_1shot;
6646                 flags = SA_SAMPLE_RANDOM;
6647         } else {
6648                 fn = tg3_interrupt;
6649                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
6650                         fn = tg3_interrupt_tagged;
6651                 flags = SA_SHIRQ | SA_SAMPLE_RANDOM;
6652         }
6653         return (request_irq(tp->pdev->irq, fn, flags, dev->name, dev));
6654 }
6655
6656 static int tg3_test_interrupt(struct tg3 *tp)
6657 {
6658         struct net_device *dev = tp->dev;
6659         int err, i;
6660         u32 int_mbox = 0;
6661
6662         if (!netif_running(dev))
6663                 return -ENODEV;
6664
6665         tg3_disable_ints(tp);
6666
6667         free_irq(tp->pdev->irq, dev);
6668
6669         err = request_irq(tp->pdev->irq, tg3_test_isr,
6670                           SA_SHIRQ | SA_SAMPLE_RANDOM, dev->name, dev);
6671         if (err)
6672                 return err;
6673
6674         tp->hw_status->status &= ~SD_STATUS_UPDATED;
6675         tg3_enable_ints(tp);
6676
6677         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
6678                HOSTCC_MODE_NOW);
6679
6680         for (i = 0; i < 5; i++) {
6681                 int_mbox = tr32_mailbox(MAILBOX_INTERRUPT_0 +
6682                                         TG3_64BIT_REG_LOW);
6683                 if (int_mbox != 0)
6684                         break;
6685                 msleep(10);
6686         }
6687
6688         tg3_disable_ints(tp);
6689
6690         free_irq(tp->pdev->irq, dev);
6691
6692         err = tg3_request_irq(tp);
6693
6694         if (err)
6695                 return err;
6696
6697         if (int_mbox != 0)
6698                 return 0;
6699
6700         return -EIO;
6701 }
6702
6703 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
6704  * successfully restored
6705  */
6706 static int tg3_test_msi(struct tg3 *tp)
6707 {
6708         struct net_device *dev = tp->dev;
6709         int err;
6710         u16 pci_cmd;
6711
6712         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
6713                 return 0;
6714
6715         /* Turn off SERR reporting in case MSI terminates with Master
6716          * Abort.
6717          */
6718         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
6719         pci_write_config_word(tp->pdev, PCI_COMMAND,
6720                               pci_cmd & ~PCI_COMMAND_SERR);
6721
6722         err = tg3_test_interrupt(tp);
6723
6724         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
6725
6726         if (!err)
6727                 return 0;
6728
6729         /* other failures */
6730         if (err != -EIO)
6731                 return err;
6732
6733         /* MSI test failed, go back to INTx mode */
6734         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
6735                "switching to INTx mode. Please report this failure to "
6736                "the PCI maintainer and include system chipset information.\n",
6737                        tp->dev->name);
6738
6739         free_irq(tp->pdev->irq, dev);
6740         pci_disable_msi(tp->pdev);
6741
6742         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6743
6744         err = tg3_request_irq(tp);
6745         if (err)
6746                 return err;
6747
6748         /* Need to reset the chip because the MSI cycle may have terminated
6749          * with Master Abort.
6750          */
6751         tg3_full_lock(tp, 1);
6752
6753         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6754         err = tg3_init_hw(tp, 1);
6755
6756         tg3_full_unlock(tp);
6757
6758         if (err)
6759                 free_irq(tp->pdev->irq, dev);
6760
6761         return err;
6762 }
6763
6764 static int tg3_open(struct net_device *dev)
6765 {
6766         struct tg3 *tp = netdev_priv(dev);
6767         int err;
6768
6769         tg3_full_lock(tp, 0);
6770
6771         err = tg3_set_power_state(tp, PCI_D0);
6772         if (err)
6773                 return err;
6774
6775         tg3_disable_ints(tp);
6776         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
6777
6778         tg3_full_unlock(tp);
6779
6780         /* The placement of this call is tied
6781          * to the setup and use of Host TX descriptors.
6782          */
6783         err = tg3_alloc_consistent(tp);
6784         if (err)
6785                 return err;
6786
6787         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
6788             (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5750_AX) &&
6789             (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5750_BX) &&
6790             !((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) &&
6791               (tp->pdev_peer == tp->pdev))) {
6792                 /* All MSI supporting chips should support tagged
6793                  * status.  Assert that this is the case.
6794                  */
6795                 if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
6796                         printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
6797                                "Not using MSI.\n", tp->dev->name);
6798                 } else if (pci_enable_msi(tp->pdev) == 0) {
6799                         u32 msi_mode;
6800
6801                         msi_mode = tr32(MSGINT_MODE);
6802                         tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
6803                         tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
6804                 }
6805         }
6806         err = tg3_request_irq(tp);
6807
6808         if (err) {
6809                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6810                         pci_disable_msi(tp->pdev);
6811                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6812                 }
6813                 tg3_free_consistent(tp);
6814                 return err;
6815         }
6816
6817         tg3_full_lock(tp, 0);
6818
6819         err = tg3_init_hw(tp, 1);
6820         if (err) {
6821                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6822                 tg3_free_rings(tp);
6823         } else {
6824                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
6825                         tp->timer_offset = HZ;
6826                 else
6827                         tp->timer_offset = HZ / 10;
6828
6829                 BUG_ON(tp->timer_offset > HZ);
6830                 tp->timer_counter = tp->timer_multiplier =
6831                         (HZ / tp->timer_offset);
6832                 tp->asf_counter = tp->asf_multiplier =
6833                         ((HZ / tp->timer_offset) * 2);
6834
6835                 init_timer(&tp->timer);
6836                 tp->timer.expires = jiffies + tp->timer_offset;
6837                 tp->timer.data = (unsigned long) tp;
6838                 tp->timer.function = tg3_timer;
6839         }
6840
6841         tg3_full_unlock(tp);
6842
6843         if (err) {
6844                 free_irq(tp->pdev->irq, dev);
6845                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6846                         pci_disable_msi(tp->pdev);
6847                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6848                 }
6849                 tg3_free_consistent(tp);
6850                 return err;
6851         }
6852
6853         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6854                 err = tg3_test_msi(tp);
6855
6856                 if (err) {
6857                         tg3_full_lock(tp, 0);
6858
6859                         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6860                                 pci_disable_msi(tp->pdev);
6861                                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6862                         }
6863                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6864                         tg3_free_rings(tp);
6865                         tg3_free_consistent(tp);
6866
6867                         tg3_full_unlock(tp);
6868
6869                         return err;
6870                 }
6871
6872                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6873                         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI) {
6874                                 u32 val = tr32(0x7c04);
6875
6876                                 tw32(0x7c04, val | (1 << 29));
6877                         }
6878                 }
6879         }
6880
6881         tg3_full_lock(tp, 0);
6882
6883         add_timer(&tp->timer);
6884         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
6885         tg3_enable_ints(tp);
6886
6887         tg3_full_unlock(tp);
6888
6889         netif_start_queue(dev);
6890
6891         return 0;
6892 }
6893
6894 #if 0
6895 /*static*/ void tg3_dump_state(struct tg3 *tp)
6896 {
6897         u32 val32, val32_2, val32_3, val32_4, val32_5;
6898         u16 val16;
6899         int i;
6900
6901         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
6902         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
6903         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
6904                val16, val32);
6905
6906         /* MAC block */
6907         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
6908                tr32(MAC_MODE), tr32(MAC_STATUS));
6909         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
6910                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
6911         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
6912                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
6913         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
6914                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
6915
6916         /* Send data initiator control block */
6917         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
6918                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
6919         printk("       SNDDATAI_STATSCTRL[%08x]\n",
6920                tr32(SNDDATAI_STATSCTRL));
6921
6922         /* Send data completion control block */
6923         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
6924
6925         /* Send BD ring selector block */
6926         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
6927                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
6928
6929         /* Send BD initiator control block */
6930         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
6931                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
6932
6933         /* Send BD completion control block */
6934         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
6935
6936         /* Receive list placement control block */
6937         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
6938                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
6939         printk("       RCVLPC_STATSCTRL[%08x]\n",
6940                tr32(RCVLPC_STATSCTRL));
6941
6942         /* Receive data and receive BD initiator control block */
6943         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
6944                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
6945
6946         /* Receive data completion control block */
6947         printk("DEBUG: RCVDCC_MODE[%08x]\n",
6948                tr32(RCVDCC_MODE));
6949
6950         /* Receive BD initiator control block */
6951         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
6952                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
6953
6954         /* Receive BD completion control block */
6955         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
6956                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
6957
6958         /* Receive list selector control block */
6959         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
6960                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
6961
6962         /* Mbuf cluster free block */
6963         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
6964                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
6965
6966         /* Host coalescing control block */
6967         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
6968                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
6969         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
6970                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
6971                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
6972         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
6973                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
6974                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
6975         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
6976                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
6977         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
6978                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
6979
6980         /* Memory arbiter control block */
6981         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
6982                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
6983
6984         /* Buffer manager control block */
6985         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
6986                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
6987         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
6988                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
6989         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
6990                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
6991                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
6992                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
6993
6994         /* Read DMA control block */
6995         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
6996                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
6997
6998         /* Write DMA control block */
6999         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
7000                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
7001
7002         /* DMA completion block */
7003         printk("DEBUG: DMAC_MODE[%08x]\n",
7004                tr32(DMAC_MODE));
7005
7006         /* GRC block */
7007         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
7008                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
7009         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
7010                tr32(GRC_LOCAL_CTRL));
7011
7012         /* TG3_BDINFOs */
7013         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
7014                tr32(RCVDBDI_JUMBO_BD + 0x0),
7015                tr32(RCVDBDI_JUMBO_BD + 0x4),
7016                tr32(RCVDBDI_JUMBO_BD + 0x8),
7017                tr32(RCVDBDI_JUMBO_BD + 0xc));
7018         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
7019                tr32(RCVDBDI_STD_BD + 0x0),
7020                tr32(RCVDBDI_STD_BD + 0x4),
7021                tr32(RCVDBDI_STD_BD + 0x8),
7022                tr32(RCVDBDI_STD_BD + 0xc));
7023         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
7024                tr32(RCVDBDI_MINI_BD + 0x0),
7025                tr32(RCVDBDI_MINI_BD + 0x4),
7026                tr32(RCVDBDI_MINI_BD + 0x8),
7027                tr32(RCVDBDI_MINI_BD + 0xc));
7028
7029         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
7030         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
7031         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
7032         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
7033         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
7034                val32, val32_2, val32_3, val32_4);
7035
7036         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
7037         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
7038         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
7039         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
7040         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
7041                val32, val32_2, val32_3, val32_4);
7042
7043         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
7044         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
7045         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
7046         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
7047         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
7048         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
7049                val32, val32_2, val32_3, val32_4, val32_5);
7050
7051         /* SW status block */
7052         printk("DEBUG: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
7053                tp->hw_status->status,
7054                tp->hw_status->status_tag,
7055                tp->hw_status->rx_jumbo_consumer,
7056                tp->hw_status->rx_consumer,
7057                tp->hw_status->rx_mini_consumer,
7058                tp->hw_status->idx[0].rx_producer,
7059                tp->hw_status->idx[0].tx_consumer);
7060
7061         /* SW statistics block */
7062         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
7063                ((u32 *)tp->hw_stats)[0],
7064                ((u32 *)tp->hw_stats)[1],
7065                ((u32 *)tp->hw_stats)[2],
7066                ((u32 *)tp->hw_stats)[3]);
7067
7068         /* Mailboxes */
7069         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
7070                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
7071                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
7072                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
7073                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
7074
7075         /* NIC side send descriptors. */
7076         for (i = 0; i < 6; i++) {
7077                 unsigned long txd;
7078
7079                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
7080                         + (i * sizeof(struct tg3_tx_buffer_desc));
7081                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
7082                        i,
7083                        readl(txd + 0x0), readl(txd + 0x4),
7084                        readl(txd + 0x8), readl(txd + 0xc));
7085         }
7086
7087         /* NIC side RX descriptors. */
7088         for (i = 0; i < 6; i++) {
7089                 unsigned long rxd;
7090
7091                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
7092                         + (i * sizeof(struct tg3_rx_buffer_desc));
7093                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
7094                        i,
7095                        readl(rxd + 0x0), readl(rxd + 0x4),
7096                        readl(rxd + 0x8), readl(rxd + 0xc));
7097                 rxd += (4 * sizeof(u32));
7098                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
7099                        i,
7100                        readl(rxd + 0x0), readl(rxd + 0x4),
7101                        readl(rxd + 0x8), readl(rxd + 0xc));
7102         }
7103
7104         for (i = 0; i < 6; i++) {
7105                 unsigned long rxd;
7106
7107                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
7108                         + (i * sizeof(struct tg3_rx_buffer_desc));
7109                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
7110                        i,
7111                        readl(rxd + 0x0), readl(rxd + 0x4),
7112                        readl(rxd + 0x8), readl(rxd + 0xc));
7113                 rxd += (4 * sizeof(u32));
7114                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
7115                        i,
7116                        readl(rxd + 0x0), readl(rxd + 0x4),
7117                        readl(rxd + 0x8), readl(rxd + 0xc));
7118         }
7119 }
7120 #endif
7121
7122 static struct net_device_stats *tg3_get_stats(struct net_device *);
7123 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
7124
7125 static int tg3_close(struct net_device *dev)
7126 {
7127         struct tg3 *tp = netdev_priv(dev);
7128
7129         /* Calling flush_scheduled_work() may deadlock because
7130          * linkwatch_event() may be on the workqueue and it will try to get
7131          * the rtnl_lock which we are holding.
7132          */
7133         while (tp->tg3_flags & TG3_FLAG_IN_RESET_TASK)
7134                 msleep(1);
7135
7136         netif_stop_queue(dev);
7137
7138         del_timer_sync(&tp->timer);
7139
7140         tg3_full_lock(tp, 1);
7141 #if 0
7142         tg3_dump_state(tp);
7143 #endif
7144
7145         tg3_disable_ints(tp);
7146
7147         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7148         tg3_free_rings(tp);
7149         tp->tg3_flags &=
7150                 ~(TG3_FLAG_INIT_COMPLETE |
7151                   TG3_FLAG_GOT_SERDES_FLOWCTL);
7152
7153         tg3_full_unlock(tp);
7154
7155         free_irq(tp->pdev->irq, dev);
7156         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7157                 pci_disable_msi(tp->pdev);
7158                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7159         }
7160
7161         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
7162                sizeof(tp->net_stats_prev));
7163         memcpy(&tp->estats_prev, tg3_get_estats(tp),
7164                sizeof(tp->estats_prev));
7165
7166         tg3_free_consistent(tp);
7167
7168         tg3_set_power_state(tp, PCI_D3hot);
7169
7170         netif_carrier_off(tp->dev);
7171
7172         return 0;
7173 }
7174
7175 static inline unsigned long get_stat64(tg3_stat64_t *val)
7176 {
7177         unsigned long ret;
7178
7179 #if (BITS_PER_LONG == 32)
7180         ret = val->low;
7181 #else
7182         ret = ((u64)val->high << 32) | ((u64)val->low);
7183 #endif
7184         return ret;
7185 }
7186
7187 static unsigned long calc_crc_errors(struct tg3 *tp)
7188 {
7189         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7190
7191         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7192             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
7193              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
7194                 u32 val;
7195
7196                 spin_lock_bh(&tp->lock);
7197                 if (!tg3_readphy(tp, 0x1e, &val)) {
7198                         tg3_writephy(tp, 0x1e, val | 0x8000);
7199                         tg3_readphy(tp, 0x14, &val);
7200                 } else
7201                         val = 0;
7202                 spin_unlock_bh(&tp->lock);
7203
7204                 tp->phy_crc_errors += val;
7205
7206                 return tp->phy_crc_errors;
7207         }
7208
7209         return get_stat64(&hw_stats->rx_fcs_errors);
7210 }
7211
7212 #define ESTAT_ADD(member) \
7213         estats->member =        old_estats->member + \
7214                                 get_stat64(&hw_stats->member)
7215
7216 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
7217 {
7218         struct tg3_ethtool_stats *estats = &tp->estats;
7219         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
7220         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7221
7222         if (!hw_stats)
7223                 return old_estats;
7224
7225         ESTAT_ADD(rx_octets);
7226         ESTAT_ADD(rx_fragments);
7227         ESTAT_ADD(rx_ucast_packets);
7228         ESTAT_ADD(rx_mcast_packets);
7229         ESTAT_ADD(rx_bcast_packets);
7230         ESTAT_ADD(rx_fcs_errors);
7231         ESTAT_ADD(rx_align_errors);
7232         ESTAT_ADD(rx_xon_pause_rcvd);
7233         ESTAT_ADD(rx_xoff_pause_rcvd);
7234         ESTAT_ADD(rx_mac_ctrl_rcvd);
7235         ESTAT_ADD(rx_xoff_entered);
7236         ESTAT_ADD(rx_frame_too_long_errors);
7237         ESTAT_ADD(rx_jabbers);
7238         ESTAT_ADD(rx_undersize_packets);
7239         ESTAT_ADD(rx_in_length_errors);
7240         ESTAT_ADD(rx_out_length_errors);
7241         ESTAT_ADD(rx_64_or_less_octet_packets);
7242         ESTAT_ADD(rx_65_to_127_octet_packets);
7243         ESTAT_ADD(rx_128_to_255_octet_packets);
7244         ESTAT_ADD(rx_256_to_511_octet_packets);
7245         ESTAT_ADD(rx_512_to_1023_octet_packets);
7246         ESTAT_ADD(rx_1024_to_1522_octet_packets);
7247         ESTAT_ADD(rx_1523_to_2047_octet_packets);
7248         ESTAT_ADD(rx_2048_to_4095_octet_packets);
7249         ESTAT_ADD(rx_4096_to_8191_octet_packets);
7250         ESTAT_ADD(rx_8192_to_9022_octet_packets);
7251
7252         ESTAT_ADD(tx_octets);
7253         ESTAT_ADD(tx_collisions);
7254         ESTAT_ADD(tx_xon_sent);
7255         ESTAT_ADD(tx_xoff_sent);
7256         ESTAT_ADD(tx_flow_control);
7257         ESTAT_ADD(tx_mac_errors);
7258         ESTAT_ADD(tx_single_collisions);
7259         ESTAT_ADD(tx_mult_collisions);
7260         ESTAT_ADD(tx_deferred);
7261         ESTAT_ADD(tx_excessive_collisions);
7262         ESTAT_ADD(tx_late_collisions);
7263         ESTAT_ADD(tx_collide_2times);
7264         ESTAT_ADD(tx_collide_3times);
7265         ESTAT_ADD(tx_collide_4times);
7266         ESTAT_ADD(tx_collide_5times);
7267         ESTAT_ADD(tx_collide_6times);
7268         ESTAT_ADD(tx_collide_7times);
7269         ESTAT_ADD(tx_collide_8times);
7270         ESTAT_ADD(tx_collide_9times);
7271         ESTAT_ADD(tx_collide_10times);
7272         ESTAT_ADD(tx_collide_11times);
7273         ESTAT_ADD(tx_collide_12times);
7274         ESTAT_ADD(tx_collide_13times);
7275         ESTAT_ADD(tx_collide_14times);
7276         ESTAT_ADD(tx_collide_15times);
7277         ESTAT_ADD(tx_ucast_packets);
7278         ESTAT_ADD(tx_mcast_packets);
7279         ESTAT_ADD(tx_bcast_packets);
7280         ESTAT_ADD(tx_carrier_sense_errors);
7281         ESTAT_ADD(tx_discards);
7282         ESTAT_ADD(tx_errors);
7283
7284         ESTAT_ADD(dma_writeq_full);
7285         ESTAT_ADD(dma_write_prioq_full);
7286         ESTAT_ADD(rxbds_empty);
7287         ESTAT_ADD(rx_discards);
7288         ESTAT_ADD(rx_errors);
7289         ESTAT_ADD(rx_threshold_hit);
7290
7291         ESTAT_ADD(dma_readq_full);
7292         ESTAT_ADD(dma_read_prioq_full);
7293         ESTAT_ADD(tx_comp_queue_full);
7294
7295         ESTAT_ADD(ring_set_send_prod_index);
7296         ESTAT_ADD(ring_status_update);
7297         ESTAT_ADD(nic_irqs);
7298         ESTAT_ADD(nic_avoided_irqs);
7299         ESTAT_ADD(nic_tx_threshold_hit);
7300
7301         return estats;
7302 }
7303
7304 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
7305 {
7306         struct tg3 *tp = netdev_priv(dev);
7307         struct net_device_stats *stats = &tp->net_stats;
7308         struct net_device_stats *old_stats = &tp->net_stats_prev;
7309         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7310
7311         if (!hw_stats)
7312                 return old_stats;
7313
7314         stats->rx_packets = old_stats->rx_packets +
7315                 get_stat64(&hw_stats->rx_ucast_packets) +
7316                 get_stat64(&hw_stats->rx_mcast_packets) +
7317                 get_stat64(&hw_stats->rx_bcast_packets);
7318
7319         stats->tx_packets = old_stats->tx_packets +
7320                 get_stat64(&hw_stats->tx_ucast_packets) +
7321                 get_stat64(&hw_stats->tx_mcast_packets) +
7322                 get_stat64(&hw_stats->tx_bcast_packets);
7323
7324         stats->rx_bytes = old_stats->rx_bytes +
7325                 get_stat64(&hw_stats->rx_octets);
7326         stats->tx_bytes = old_stats->tx_bytes +
7327                 get_stat64(&hw_stats->tx_octets);
7328
7329         stats->rx_errors = old_stats->rx_errors +
7330                 get_stat64(&hw_stats->rx_errors);
7331         stats->tx_errors = old_stats->tx_errors +
7332                 get_stat64(&hw_stats->tx_errors) +
7333                 get_stat64(&hw_stats->tx_mac_errors) +
7334                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
7335                 get_stat64(&hw_stats->tx_discards);
7336
7337         stats->multicast = old_stats->multicast +
7338                 get_stat64(&hw_stats->rx_mcast_packets);
7339         stats->collisions = old_stats->collisions +
7340                 get_stat64(&hw_stats->tx_collisions);
7341
7342         stats->rx_length_errors = old_stats->rx_length_errors +
7343                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
7344                 get_stat64(&hw_stats->rx_undersize_packets);
7345
7346         stats->rx_over_errors = old_stats->rx_over_errors +
7347                 get_stat64(&hw_stats->rxbds_empty);
7348         stats->rx_frame_errors = old_stats->rx_frame_errors +
7349                 get_stat64(&hw_stats->rx_align_errors);
7350         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
7351                 get_stat64(&hw_stats->tx_discards);
7352         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
7353                 get_stat64(&hw_stats->tx_carrier_sense_errors);
7354
7355         stats->rx_crc_errors = old_stats->rx_crc_errors +
7356                 calc_crc_errors(tp);
7357
7358         stats->rx_missed_errors = old_stats->rx_missed_errors +
7359                 get_stat64(&hw_stats->rx_discards);
7360
7361         return stats;
7362 }
7363
7364 static inline u32 calc_crc(unsigned char *buf, int len)
7365 {
7366         u32 reg;
7367         u32 tmp;
7368         int j, k;
7369
7370         reg = 0xffffffff;
7371
7372         for (j = 0; j < len; j++) {
7373                 reg ^= buf[j];
7374
7375                 for (k = 0; k < 8; k++) {
7376                         tmp = reg & 0x01;
7377
7378                         reg >>= 1;
7379
7380                         if (tmp) {
7381                                 reg ^= 0xedb88320;
7382                         }
7383                 }
7384         }
7385
7386         return ~reg;
7387 }
7388
7389 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
7390 {
7391         /* accept or reject all multicast frames */
7392         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
7393         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
7394         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
7395         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
7396 }
7397
7398 static void __tg3_set_rx_mode(struct net_device *dev)
7399 {
7400         struct tg3 *tp = netdev_priv(dev);
7401         u32 rx_mode;
7402
7403         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
7404                                   RX_MODE_KEEP_VLAN_TAG);
7405
7406         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
7407          * flag clear.
7408          */
7409 #if TG3_VLAN_TAG_USED
7410         if (!tp->vlgrp &&
7411             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
7412                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
7413 #else
7414         /* By definition, VLAN is disabled always in this
7415          * case.
7416          */
7417         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
7418                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
7419 #endif
7420
7421         if (dev->flags & IFF_PROMISC) {
7422                 /* Promiscuous mode. */
7423                 rx_mode |= RX_MODE_PROMISC;
7424         } else if (dev->flags & IFF_ALLMULTI) {
7425                 /* Accept all multicast. */
7426                 tg3_set_multi (tp, 1);
7427         } else if (dev->mc_count < 1) {
7428                 /* Reject all multicast. */
7429                 tg3_set_multi (tp, 0);
7430         } else {
7431                 /* Accept one or more multicast(s). */
7432                 struct dev_mc_list *mclist;
7433                 unsigned int i;
7434                 u32 mc_filter[4] = { 0, };
7435                 u32 regidx;
7436                 u32 bit;
7437                 u32 crc;
7438
7439                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
7440                      i++, mclist = mclist->next) {
7441
7442                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
7443                         bit = ~crc & 0x7f;
7444                         regidx = (bit & 0x60) >> 5;
7445                         bit &= 0x1f;
7446                         mc_filter[regidx] |= (1 << bit);
7447                 }
7448
7449                 tw32(MAC_HASH_REG_0, mc_filter[0]);
7450                 tw32(MAC_HASH_REG_1, mc_filter[1]);
7451                 tw32(MAC_HASH_REG_2, mc_filter[2]);
7452                 tw32(MAC_HASH_REG_3, mc_filter[3]);
7453         }
7454
7455         if (rx_mode != tp->rx_mode) {
7456                 tp->rx_mode = rx_mode;
7457                 tw32_f(MAC_RX_MODE, rx_mode);
7458                 udelay(10);
7459         }
7460 }
7461
7462 static void tg3_set_rx_mode(struct net_device *dev)
7463 {
7464         struct tg3 *tp = netdev_priv(dev);
7465
7466         if (!netif_running(dev))
7467                 return;
7468
7469         tg3_full_lock(tp, 0);
7470         __tg3_set_rx_mode(dev);
7471         tg3_full_unlock(tp);
7472 }
7473
7474 #define TG3_REGDUMP_LEN         (32 * 1024)
7475
7476 static int tg3_get_regs_len(struct net_device *dev)
7477 {
7478         return TG3_REGDUMP_LEN;
7479 }
7480
7481 static void tg3_get_regs(struct net_device *dev,
7482                 struct ethtool_regs *regs, void *_p)
7483 {
7484         u32 *p = _p;
7485         struct tg3 *tp = netdev_priv(dev);
7486         u8 *orig_p = _p;
7487         int i;
7488
7489         regs->version = 0;
7490
7491         memset(p, 0, TG3_REGDUMP_LEN);
7492
7493         if (tp->link_config.phy_is_low_power)
7494                 return;
7495
7496         tg3_full_lock(tp, 0);
7497
7498 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
7499 #define GET_REG32_LOOP(base,len)                \
7500 do {    p = (u32 *)(orig_p + (base));           \
7501         for (i = 0; i < len; i += 4)            \
7502                 __GET_REG32((base) + i);        \
7503 } while (0)
7504 #define GET_REG32_1(reg)                        \
7505 do {    p = (u32 *)(orig_p + (reg));            \
7506         __GET_REG32((reg));                     \
7507 } while (0)
7508
7509         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
7510         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
7511         GET_REG32_LOOP(MAC_MODE, 0x4f0);
7512         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
7513         GET_REG32_1(SNDDATAC_MODE);
7514         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
7515         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
7516         GET_REG32_1(SNDBDC_MODE);
7517         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
7518         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
7519         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
7520         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
7521         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
7522         GET_REG32_1(RCVDCC_MODE);
7523         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
7524         GET_REG32_LOOP(RCVCC_MODE, 0x14);
7525         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
7526         GET_REG32_1(MBFREE_MODE);
7527         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
7528         GET_REG32_LOOP(MEMARB_MODE, 0x10);
7529         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
7530         GET_REG32_LOOP(RDMAC_MODE, 0x08);
7531         GET_REG32_LOOP(WDMAC_MODE, 0x08);
7532         GET_REG32_1(RX_CPU_MODE);
7533         GET_REG32_1(RX_CPU_STATE);
7534         GET_REG32_1(RX_CPU_PGMCTR);
7535         GET_REG32_1(RX_CPU_HWBKPT);
7536         GET_REG32_1(TX_CPU_MODE);
7537         GET_REG32_1(TX_CPU_STATE);
7538         GET_REG32_1(TX_CPU_PGMCTR);
7539         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
7540         GET_REG32_LOOP(FTQ_RESET, 0x120);
7541         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
7542         GET_REG32_1(DMAC_MODE);
7543         GET_REG32_LOOP(GRC_MODE, 0x4c);
7544         if (tp->tg3_flags & TG3_FLAG_NVRAM)
7545                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
7546
7547 #undef __GET_REG32
7548 #undef GET_REG32_LOOP
7549 #undef GET_REG32_1
7550
7551         tg3_full_unlock(tp);
7552 }
7553
7554 static int tg3_get_eeprom_len(struct net_device *dev)
7555 {
7556         struct tg3 *tp = netdev_priv(dev);
7557
7558         return tp->nvram_size;
7559 }
7560
7561 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val);
7562 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val);
7563
7564 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
7565 {
7566         struct tg3 *tp = netdev_priv(dev);
7567         int ret;
7568         u8  *pd;
7569         u32 i, offset, len, val, b_offset, b_count;
7570
7571         if (tp->link_config.phy_is_low_power)
7572                 return -EAGAIN;
7573
7574         offset = eeprom->offset;
7575         len = eeprom->len;
7576         eeprom->len = 0;
7577
7578         eeprom->magic = TG3_EEPROM_MAGIC;
7579
7580         if (offset & 3) {
7581                 /* adjustments to start on required 4 byte boundary */
7582                 b_offset = offset & 3;
7583                 b_count = 4 - b_offset;
7584                 if (b_count > len) {
7585                         /* i.e. offset=1 len=2 */
7586                         b_count = len;
7587                 }
7588                 ret = tg3_nvram_read(tp, offset-b_offset, &val);
7589                 if (ret)
7590                         return ret;
7591                 val = cpu_to_le32(val);
7592                 memcpy(data, ((char*)&val) + b_offset, b_count);
7593                 len -= b_count;
7594                 offset += b_count;
7595                 eeprom->len += b_count;
7596         }
7597
7598         /* read bytes upto the last 4 byte boundary */
7599         pd = &data[eeprom->len];
7600         for (i = 0; i < (len - (len & 3)); i += 4) {
7601                 ret = tg3_nvram_read(tp, offset + i, &val);
7602                 if (ret) {
7603                         eeprom->len += i;
7604                         return ret;
7605                 }
7606                 val = cpu_to_le32(val);
7607                 memcpy(pd + i, &val, 4);
7608         }
7609         eeprom->len += i;
7610
7611         if (len & 3) {
7612                 /* read last bytes not ending on 4 byte boundary */
7613                 pd = &data[eeprom->len];
7614                 b_count = len & 3;
7615                 b_offset = offset + len - b_count;
7616                 ret = tg3_nvram_read(tp, b_offset, &val);
7617                 if (ret)
7618                         return ret;
7619                 val = cpu_to_le32(val);
7620                 memcpy(pd, ((char*)&val), b_count);
7621                 eeprom->len += b_count;
7622         }
7623         return 0;
7624 }
7625
7626 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
7627
7628 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
7629 {
7630         struct tg3 *tp = netdev_priv(dev);
7631         int ret;
7632         u32 offset, len, b_offset, odd_len, start, end;
7633         u8 *buf;
7634
7635         if (tp->link_config.phy_is_low_power)
7636                 return -EAGAIN;
7637
7638         if (eeprom->magic != TG3_EEPROM_MAGIC)
7639                 return -EINVAL;
7640
7641         offset = eeprom->offset;
7642         len = eeprom->len;
7643
7644         if ((b_offset = (offset & 3))) {
7645                 /* adjustments to start on required 4 byte boundary */
7646                 ret = tg3_nvram_read(tp, offset-b_offset, &start);
7647                 if (ret)
7648                         return ret;
7649                 start = cpu_to_le32(start);
7650                 len += b_offset;
7651                 offset &= ~3;
7652                 if (len < 4)
7653                         len = 4;
7654         }
7655
7656         odd_len = 0;
7657         if (len & 3) {
7658                 /* adjustments to end on required 4 byte boundary */
7659                 odd_len = 1;
7660                 len = (len + 3) & ~3;
7661                 ret = tg3_nvram_read(tp, offset+len-4, &end);
7662                 if (ret)
7663                         return ret;
7664                 end = cpu_to_le32(end);
7665         }
7666
7667         buf = data;
7668         if (b_offset || odd_len) {
7669                 buf = kmalloc(len, GFP_KERNEL);
7670                 if (buf == 0)
7671                         return -ENOMEM;
7672                 if (b_offset)
7673                         memcpy(buf, &start, 4);
7674                 if (odd_len)
7675                         memcpy(buf+len-4, &end, 4);
7676                 memcpy(buf + b_offset, data, eeprom->len);
7677         }
7678
7679         ret = tg3_nvram_write_block(tp, offset, len, buf);
7680
7681         if (buf != data)
7682                 kfree(buf);
7683
7684         return ret;
7685 }
7686
7687 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
7688 {
7689         struct tg3 *tp = netdev_priv(dev);
7690
7691         cmd->supported = (SUPPORTED_Autoneg);
7692
7693         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
7694                 cmd->supported |= (SUPPORTED_1000baseT_Half |
7695                                    SUPPORTED_1000baseT_Full);
7696
7697         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
7698                 cmd->supported |= (SUPPORTED_100baseT_Half |
7699                                   SUPPORTED_100baseT_Full |
7700                                   SUPPORTED_10baseT_Half |
7701                                   SUPPORTED_10baseT_Full |
7702                                   SUPPORTED_MII);
7703                 cmd->port = PORT_TP;
7704         } else {
7705                 cmd->supported |= SUPPORTED_FIBRE;
7706                 cmd->port = PORT_FIBRE;
7707         }
7708
7709         cmd->advertising = tp->link_config.advertising;
7710         if (netif_running(dev)) {
7711                 cmd->speed = tp->link_config.active_speed;
7712                 cmd->duplex = tp->link_config.active_duplex;
7713         }
7714         cmd->phy_address = PHY_ADDR;
7715         cmd->transceiver = 0;
7716         cmd->autoneg = tp->link_config.autoneg;
7717         cmd->maxtxpkt = 0;
7718         cmd->maxrxpkt = 0;
7719         return 0;
7720 }
7721
7722 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
7723 {
7724         struct tg3 *tp = netdev_priv(dev);
7725
7726         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
7727                 /* These are the only valid advertisement bits allowed.  */
7728                 if (cmd->autoneg == AUTONEG_ENABLE &&
7729                     (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
7730                                           ADVERTISED_1000baseT_Full |
7731                                           ADVERTISED_Autoneg |
7732                                           ADVERTISED_FIBRE)))
7733                         return -EINVAL;
7734                 /* Fiber can only do SPEED_1000.  */
7735                 else if ((cmd->autoneg != AUTONEG_ENABLE) &&
7736                          (cmd->speed != SPEED_1000))
7737                         return -EINVAL;
7738         /* Copper cannot force SPEED_1000.  */
7739         } else if ((cmd->autoneg != AUTONEG_ENABLE) &&
7740                    (cmd->speed == SPEED_1000))
7741                 return -EINVAL;
7742         else if ((cmd->speed == SPEED_1000) &&
7743                  (tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
7744                 return -EINVAL;
7745
7746         tg3_full_lock(tp, 0);
7747
7748         tp->link_config.autoneg = cmd->autoneg;
7749         if (cmd->autoneg == AUTONEG_ENABLE) {
7750                 tp->link_config.advertising = cmd->advertising;
7751                 tp->link_config.speed = SPEED_INVALID;
7752                 tp->link_config.duplex = DUPLEX_INVALID;
7753         } else {
7754                 tp->link_config.advertising = 0;
7755                 tp->link_config.speed = cmd->speed;
7756                 tp->link_config.duplex = cmd->duplex;
7757         }
7758
7759         if (netif_running(dev))
7760                 tg3_setup_phy(tp, 1);
7761
7762         tg3_full_unlock(tp);
7763
7764         return 0;
7765 }
7766
7767 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
7768 {
7769         struct tg3 *tp = netdev_priv(dev);
7770
7771         strcpy(info->driver, DRV_MODULE_NAME);
7772         strcpy(info->version, DRV_MODULE_VERSION);
7773         strcpy(info->fw_version, tp->fw_ver);
7774         strcpy(info->bus_info, pci_name(tp->pdev));
7775 }
7776
7777 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
7778 {
7779         struct tg3 *tp = netdev_priv(dev);
7780
7781         wol->supported = WAKE_MAGIC;
7782         wol->wolopts = 0;
7783         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)
7784                 wol->wolopts = WAKE_MAGIC;
7785         memset(&wol->sopass, 0, sizeof(wol->sopass));
7786 }
7787
7788 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
7789 {
7790         struct tg3 *tp = netdev_priv(dev);
7791
7792         if (wol->wolopts & ~WAKE_MAGIC)
7793                 return -EINVAL;
7794         if ((wol->wolopts & WAKE_MAGIC) &&
7795             tp->tg3_flags2 & TG3_FLG2_PHY_SERDES &&
7796             !(tp->tg3_flags & TG3_FLAG_SERDES_WOL_CAP))
7797                 return -EINVAL;
7798
7799         spin_lock_bh(&tp->lock);
7800         if (wol->wolopts & WAKE_MAGIC)
7801                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
7802         else
7803                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
7804         spin_unlock_bh(&tp->lock);
7805
7806         return 0;
7807 }
7808
7809 static u32 tg3_get_msglevel(struct net_device *dev)
7810 {
7811         struct tg3 *tp = netdev_priv(dev);
7812         return tp->msg_enable;
7813 }
7814
7815 static void tg3_set_msglevel(struct net_device *dev, u32 value)
7816 {
7817         struct tg3 *tp = netdev_priv(dev);
7818         tp->msg_enable = value;
7819 }
7820
7821 #if TG3_TSO_SUPPORT != 0
7822 static int tg3_set_tso(struct net_device *dev, u32 value)
7823 {
7824         struct tg3 *tp = netdev_priv(dev);
7825
7826         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
7827                 if (value)
7828                         return -EINVAL;
7829                 return 0;
7830         }
7831         return ethtool_op_set_tso(dev, value);
7832 }
7833 #endif
7834
7835 static int tg3_nway_reset(struct net_device *dev)
7836 {
7837         struct tg3 *tp = netdev_priv(dev);
7838         u32 bmcr;
7839         int r;
7840
7841         if (!netif_running(dev))
7842                 return -EAGAIN;
7843
7844         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
7845                 return -EINVAL;
7846
7847         spin_lock_bh(&tp->lock);
7848         r = -EINVAL;
7849         tg3_readphy(tp, MII_BMCR, &bmcr);
7850         if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
7851             ((bmcr & BMCR_ANENABLE) ||
7852              (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
7853                 tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
7854                                            BMCR_ANENABLE);
7855                 r = 0;
7856         }
7857         spin_unlock_bh(&tp->lock);
7858
7859         return r;
7860 }
7861
7862 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
7863 {
7864         struct tg3 *tp = netdev_priv(dev);
7865
7866         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
7867         ering->rx_mini_max_pending = 0;
7868         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
7869                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
7870         else
7871                 ering->rx_jumbo_max_pending = 0;
7872
7873         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
7874
7875         ering->rx_pending = tp->rx_pending;
7876         ering->rx_mini_pending = 0;
7877         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
7878                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
7879         else
7880                 ering->rx_jumbo_pending = 0;
7881
7882         ering->tx_pending = tp->tx_pending;
7883 }
7884
7885 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
7886 {
7887         struct tg3 *tp = netdev_priv(dev);
7888         int irq_sync = 0;
7889
7890         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
7891             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
7892             (ering->tx_pending > TG3_TX_RING_SIZE - 1))
7893                 return -EINVAL;
7894
7895         if (netif_running(dev)) {
7896                 tg3_netif_stop(tp);
7897                 irq_sync = 1;
7898         }
7899
7900         tg3_full_lock(tp, irq_sync);
7901
7902         tp->rx_pending = ering->rx_pending;
7903
7904         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
7905             tp->rx_pending > 63)
7906                 tp->rx_pending = 63;
7907         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
7908         tp->tx_pending = ering->tx_pending;
7909
7910         if (netif_running(dev)) {
7911                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7912                 tg3_init_hw(tp, 1);
7913                 tg3_netif_start(tp);
7914         }
7915
7916         tg3_full_unlock(tp);
7917
7918         return 0;
7919 }
7920
7921 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
7922 {
7923         struct tg3 *tp = netdev_priv(dev);
7924
7925         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
7926         epause->rx_pause = (tp->tg3_flags & TG3_FLAG_RX_PAUSE) != 0;
7927         epause->tx_pause = (tp->tg3_flags & TG3_FLAG_TX_PAUSE) != 0;
7928 }
7929
7930 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
7931 {
7932         struct tg3 *tp = netdev_priv(dev);
7933         int irq_sync = 0;
7934
7935         if (netif_running(dev)) {
7936                 tg3_netif_stop(tp);
7937                 irq_sync = 1;
7938         }
7939
7940         tg3_full_lock(tp, irq_sync);
7941
7942         if (epause->autoneg)
7943                 tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
7944         else
7945                 tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
7946         if (epause->rx_pause)
7947                 tp->tg3_flags |= TG3_FLAG_RX_PAUSE;
7948         else
7949                 tp->tg3_flags &= ~TG3_FLAG_RX_PAUSE;
7950         if (epause->tx_pause)
7951                 tp->tg3_flags |= TG3_FLAG_TX_PAUSE;
7952         else
7953                 tp->tg3_flags &= ~TG3_FLAG_TX_PAUSE;
7954
7955         if (netif_running(dev)) {
7956                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7957                 tg3_init_hw(tp, 1);
7958                 tg3_netif_start(tp);
7959         }
7960
7961         tg3_full_unlock(tp);
7962
7963         return 0;
7964 }
7965
7966 static u32 tg3_get_rx_csum(struct net_device *dev)
7967 {
7968         struct tg3 *tp = netdev_priv(dev);
7969         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
7970 }
7971
7972 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
7973 {
7974         struct tg3 *tp = netdev_priv(dev);
7975
7976         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
7977                 if (data != 0)
7978                         return -EINVAL;
7979                 return 0;
7980         }
7981
7982         spin_lock_bh(&tp->lock);
7983         if (data)
7984                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
7985         else
7986                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
7987         spin_unlock_bh(&tp->lock);
7988
7989         return 0;
7990 }
7991
7992 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
7993 {
7994         struct tg3 *tp = netdev_priv(dev);
7995
7996         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
7997                 if (data != 0)
7998                         return -EINVAL;
7999                 return 0;
8000         }
8001
8002         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8003             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
8004                 ethtool_op_set_tx_hw_csum(dev, data);
8005         else
8006                 ethtool_op_set_tx_csum(dev, data);
8007
8008         return 0;
8009 }
8010
8011 static int tg3_get_stats_count (struct net_device *dev)
8012 {
8013         return TG3_NUM_STATS;
8014 }
8015
8016 static int tg3_get_test_count (struct net_device *dev)
8017 {
8018         return TG3_NUM_TEST;
8019 }
8020
8021 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
8022 {
8023         switch (stringset) {
8024         case ETH_SS_STATS:
8025                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
8026                 break;
8027         case ETH_SS_TEST:
8028                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
8029                 break;
8030         default:
8031                 WARN_ON(1);     /* we need a WARN() */
8032                 break;
8033         }
8034 }
8035
8036 static int tg3_phys_id(struct net_device *dev, u32 data)
8037 {
8038         struct tg3 *tp = netdev_priv(dev);
8039         int i;
8040
8041         if (!netif_running(tp->dev))
8042                 return -EAGAIN;
8043
8044         if (data == 0)
8045                 data = 2;
8046
8047         for (i = 0; i < (data * 2); i++) {
8048                 if ((i % 2) == 0)
8049                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8050                                            LED_CTRL_1000MBPS_ON |
8051                                            LED_CTRL_100MBPS_ON |
8052                                            LED_CTRL_10MBPS_ON |
8053                                            LED_CTRL_TRAFFIC_OVERRIDE |
8054                                            LED_CTRL_TRAFFIC_BLINK |
8055                                            LED_CTRL_TRAFFIC_LED);
8056
8057                 else
8058                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8059                                            LED_CTRL_TRAFFIC_OVERRIDE);
8060
8061                 if (msleep_interruptible(500))
8062                         break;
8063         }
8064         tw32(MAC_LED_CTRL, tp->led_ctrl);
8065         return 0;
8066 }
8067
8068 static void tg3_get_ethtool_stats (struct net_device *dev,
8069                                    struct ethtool_stats *estats, u64 *tmp_stats)
8070 {
8071         struct tg3 *tp = netdev_priv(dev);
8072         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
8073 }
8074
8075 #define NVRAM_TEST_SIZE 0x100
8076 #define NVRAM_SELFBOOT_FORMAT1_SIZE 0x14
8077
8078 static int tg3_test_nvram(struct tg3 *tp)
8079 {
8080         u32 *buf, csum, magic;
8081         int i, j, err = 0, size;
8082
8083         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8084                 return -EIO;
8085
8086         if (magic == TG3_EEPROM_MAGIC)
8087                 size = NVRAM_TEST_SIZE;
8088         else if ((magic & 0xff000000) == 0xa5000000) {
8089                 if ((magic & 0xe00000) == 0x200000)
8090                         size = NVRAM_SELFBOOT_FORMAT1_SIZE;
8091                 else
8092                         return 0;
8093         } else
8094                 return -EIO;
8095
8096         buf = kmalloc(size, GFP_KERNEL);
8097         if (buf == NULL)
8098                 return -ENOMEM;
8099
8100         err = -EIO;
8101         for (i = 0, j = 0; i < size; i += 4, j++) {
8102                 u32 val;
8103
8104                 if ((err = tg3_nvram_read(tp, i, &val)) != 0)
8105                         break;
8106                 buf[j] = cpu_to_le32(val);
8107         }
8108         if (i < size)
8109                 goto out;
8110
8111         /* Selfboot format */
8112         if (cpu_to_be32(buf[0]) != TG3_EEPROM_MAGIC) {
8113                 u8 *buf8 = (u8 *) buf, csum8 = 0;
8114
8115                 for (i = 0; i < size; i++)
8116                         csum8 += buf8[i];
8117
8118                 if (csum8 == 0) {
8119                         err = 0;
8120                         goto out;
8121                 }
8122
8123                 err = -EIO;
8124                 goto out;
8125         }
8126
8127         /* Bootstrap checksum at offset 0x10 */
8128         csum = calc_crc((unsigned char *) buf, 0x10);
8129         if(csum != cpu_to_le32(buf[0x10/4]))
8130                 goto out;
8131
8132         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
8133         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
8134         if (csum != cpu_to_le32(buf[0xfc/4]))
8135                  goto out;
8136
8137         err = 0;
8138
8139 out:
8140         kfree(buf);
8141         return err;
8142 }
8143
8144 #define TG3_SERDES_TIMEOUT_SEC  2
8145 #define TG3_COPPER_TIMEOUT_SEC  6
8146
8147 static int tg3_test_link(struct tg3 *tp)
8148 {
8149         int i, max;
8150
8151         if (!netif_running(tp->dev))
8152                 return -ENODEV;
8153
8154         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
8155                 max = TG3_SERDES_TIMEOUT_SEC;
8156         else
8157                 max = TG3_COPPER_TIMEOUT_SEC;
8158
8159         for (i = 0; i < max; i++) {
8160                 if (netif_carrier_ok(tp->dev))
8161                         return 0;
8162
8163                 if (msleep_interruptible(1000))
8164                         break;
8165         }
8166
8167         return -EIO;
8168 }
8169
8170 /* Only test the commonly used registers */
8171 static int tg3_test_registers(struct tg3 *tp)
8172 {
8173         int i, is_5705;
8174         u32 offset, read_mask, write_mask, val, save_val, read_val;
8175         static struct {
8176                 u16 offset;
8177                 u16 flags;
8178 #define TG3_FL_5705     0x1
8179 #define TG3_FL_NOT_5705 0x2
8180 #define TG3_FL_NOT_5788 0x4
8181                 u32 read_mask;
8182                 u32 write_mask;
8183         } reg_tbl[] = {
8184                 /* MAC Control Registers */
8185                 { MAC_MODE, TG3_FL_NOT_5705,
8186                         0x00000000, 0x00ef6f8c },
8187                 { MAC_MODE, TG3_FL_5705,
8188                         0x00000000, 0x01ef6b8c },
8189                 { MAC_STATUS, TG3_FL_NOT_5705,
8190                         0x03800107, 0x00000000 },
8191                 { MAC_STATUS, TG3_FL_5705,
8192                         0x03800100, 0x00000000 },
8193                 { MAC_ADDR_0_HIGH, 0x0000,
8194                         0x00000000, 0x0000ffff },
8195                 { MAC_ADDR_0_LOW, 0x0000,
8196                         0x00000000, 0xffffffff },
8197                 { MAC_RX_MTU_SIZE, 0x0000,
8198                         0x00000000, 0x0000ffff },
8199                 { MAC_TX_MODE, 0x0000,
8200                         0x00000000, 0x00000070 },
8201                 { MAC_TX_LENGTHS, 0x0000,
8202                         0x00000000, 0x00003fff },
8203                 { MAC_RX_MODE, TG3_FL_NOT_5705,
8204                         0x00000000, 0x000007fc },
8205                 { MAC_RX_MODE, TG3_FL_5705,
8206                         0x00000000, 0x000007dc },
8207                 { MAC_HASH_REG_0, 0x0000,
8208                         0x00000000, 0xffffffff },
8209                 { MAC_HASH_REG_1, 0x0000,
8210                         0x00000000, 0xffffffff },
8211                 { MAC_HASH_REG_2, 0x0000,
8212                         0x00000000, 0xffffffff },
8213                 { MAC_HASH_REG_3, 0x0000,
8214                         0x00000000, 0xffffffff },
8215
8216                 /* Receive Data and Receive BD Initiator Control Registers. */
8217                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
8218                         0x00000000, 0xffffffff },
8219                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
8220                         0x00000000, 0xffffffff },
8221                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
8222                         0x00000000, 0x00000003 },
8223                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
8224                         0x00000000, 0xffffffff },
8225                 { RCVDBDI_STD_BD+0, 0x0000,
8226                         0x00000000, 0xffffffff },
8227                 { RCVDBDI_STD_BD+4, 0x0000,
8228                         0x00000000, 0xffffffff },
8229                 { RCVDBDI_STD_BD+8, 0x0000,
8230                         0x00000000, 0xffff0002 },
8231                 { RCVDBDI_STD_BD+0xc, 0x0000,
8232                         0x00000000, 0xffffffff },
8233
8234                 /* Receive BD Initiator Control Registers. */
8235                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
8236                         0x00000000, 0xffffffff },
8237                 { RCVBDI_STD_THRESH, TG3_FL_5705,
8238                         0x00000000, 0x000003ff },
8239                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
8240                         0x00000000, 0xffffffff },
8241
8242                 /* Host Coalescing Control Registers. */
8243                 { HOSTCC_MODE, TG3_FL_NOT_5705,
8244                         0x00000000, 0x00000004 },
8245                 { HOSTCC_MODE, TG3_FL_5705,
8246                         0x00000000, 0x000000f6 },
8247                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
8248                         0x00000000, 0xffffffff },
8249                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
8250                         0x00000000, 0x000003ff },
8251                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
8252                         0x00000000, 0xffffffff },
8253                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
8254                         0x00000000, 0x000003ff },
8255                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
8256                         0x00000000, 0xffffffff },
8257                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8258                         0x00000000, 0x000000ff },
8259                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
8260                         0x00000000, 0xffffffff },
8261                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8262                         0x00000000, 0x000000ff },
8263                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
8264                         0x00000000, 0xffffffff },
8265                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
8266                         0x00000000, 0xffffffff },
8267                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8268                         0x00000000, 0xffffffff },
8269                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8270                         0x00000000, 0x000000ff },
8271                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8272                         0x00000000, 0xffffffff },
8273                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8274                         0x00000000, 0x000000ff },
8275                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
8276                         0x00000000, 0xffffffff },
8277                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
8278                         0x00000000, 0xffffffff },
8279                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
8280                         0x00000000, 0xffffffff },
8281                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
8282                         0x00000000, 0xffffffff },
8283                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
8284                         0x00000000, 0xffffffff },
8285                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
8286                         0xffffffff, 0x00000000 },
8287                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
8288                         0xffffffff, 0x00000000 },
8289
8290                 /* Buffer Manager Control Registers. */
8291                 { BUFMGR_MB_POOL_ADDR, 0x0000,
8292                         0x00000000, 0x007fff80 },
8293                 { BUFMGR_MB_POOL_SIZE, 0x0000,
8294                         0x00000000, 0x007fffff },
8295                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
8296                         0x00000000, 0x0000003f },
8297                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
8298                         0x00000000, 0x000001ff },
8299                 { BUFMGR_MB_HIGH_WATER, 0x0000,
8300                         0x00000000, 0x000001ff },
8301                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
8302                         0xffffffff, 0x00000000 },
8303                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
8304                         0xffffffff, 0x00000000 },
8305
8306                 /* Mailbox Registers */
8307                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
8308                         0x00000000, 0x000001ff },
8309                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
8310                         0x00000000, 0x000001ff },
8311                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
8312                         0x00000000, 0x000007ff },
8313                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
8314                         0x00000000, 0x000001ff },
8315
8316                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
8317         };
8318
8319         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
8320                 is_5705 = 1;
8321         else
8322                 is_5705 = 0;
8323
8324         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
8325                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
8326                         continue;
8327
8328                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
8329                         continue;
8330
8331                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
8332                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
8333                         continue;
8334
8335                 offset = (u32) reg_tbl[i].offset;
8336                 read_mask = reg_tbl[i].read_mask;
8337                 write_mask = reg_tbl[i].write_mask;
8338
8339                 /* Save the original register content */
8340                 save_val = tr32(offset);
8341
8342                 /* Determine the read-only value. */
8343                 read_val = save_val & read_mask;
8344
8345                 /* Write zero to the register, then make sure the read-only bits
8346                  * are not changed and the read/write bits are all zeros.
8347                  */
8348                 tw32(offset, 0);
8349
8350                 val = tr32(offset);
8351
8352                 /* Test the read-only and read/write bits. */
8353                 if (((val & read_mask) != read_val) || (val & write_mask))
8354                         goto out;
8355
8356                 /* Write ones to all the bits defined by RdMask and WrMask, then
8357                  * make sure the read-only bits are not changed and the
8358                  * read/write bits are all ones.
8359                  */
8360                 tw32(offset, read_mask | write_mask);
8361
8362                 val = tr32(offset);
8363
8364                 /* Test the read-only bits. */
8365                 if ((val & read_mask) != read_val)
8366                         goto out;
8367
8368                 /* Test the read/write bits. */
8369                 if ((val & write_mask) != write_mask)
8370                         goto out;
8371
8372                 tw32(offset, save_val);
8373         }
8374
8375         return 0;
8376
8377 out:
8378         printk(KERN_ERR PFX "Register test failed at offset %x\n", offset);
8379         tw32(offset, save_val);
8380         return -EIO;
8381 }
8382
8383 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
8384 {
8385         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
8386         int i;
8387         u32 j;
8388
8389         for (i = 0; i < sizeof(test_pattern)/sizeof(u32); i++) {
8390                 for (j = 0; j < len; j += 4) {
8391                         u32 val;
8392
8393                         tg3_write_mem(tp, offset + j, test_pattern[i]);
8394                         tg3_read_mem(tp, offset + j, &val);
8395                         if (val != test_pattern[i])
8396                                 return -EIO;
8397                 }
8398         }
8399         return 0;
8400 }
8401
8402 static int tg3_test_memory(struct tg3 *tp)
8403 {
8404         static struct mem_entry {
8405                 u32 offset;
8406                 u32 len;
8407         } mem_tbl_570x[] = {
8408                 { 0x00000000, 0x00b50},
8409                 { 0x00002000, 0x1c000},
8410                 { 0xffffffff, 0x00000}
8411         }, mem_tbl_5705[] = {
8412                 { 0x00000100, 0x0000c},
8413                 { 0x00000200, 0x00008},
8414                 { 0x00004000, 0x00800},
8415                 { 0x00006000, 0x01000},
8416                 { 0x00008000, 0x02000},
8417                 { 0x00010000, 0x0e000},
8418                 { 0xffffffff, 0x00000}
8419         }, mem_tbl_5755[] = {
8420                 { 0x00000200, 0x00008},
8421                 { 0x00004000, 0x00800},
8422                 { 0x00006000, 0x00800},
8423                 { 0x00008000, 0x02000},
8424                 { 0x00010000, 0x0c000},
8425                 { 0xffffffff, 0x00000}
8426         };
8427         struct mem_entry *mem_tbl;
8428         int err = 0;
8429         int i;
8430
8431         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
8432                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8433                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
8434                         mem_tbl = mem_tbl_5755;
8435                 else
8436                         mem_tbl = mem_tbl_5705;
8437         } else
8438                 mem_tbl = mem_tbl_570x;
8439
8440         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
8441                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
8442                     mem_tbl[i].len)) != 0)
8443                         break;
8444         }
8445
8446         return err;
8447 }
8448
8449 #define TG3_MAC_LOOPBACK        0
8450 #define TG3_PHY_LOOPBACK        1
8451
8452 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
8453 {
8454         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
8455         u32 desc_idx;
8456         struct sk_buff *skb, *rx_skb;
8457         u8 *tx_data;
8458         dma_addr_t map;
8459         int num_pkts, tx_len, rx_len, i, err;
8460         struct tg3_rx_buffer_desc *desc;
8461
8462         if (loopback_mode == TG3_MAC_LOOPBACK) {
8463                 /* HW errata - mac loopback fails in some cases on 5780.
8464                  * Normal traffic and PHY loopback are not affected by
8465                  * errata.
8466                  */
8467                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
8468                         return 0;
8469
8470                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
8471                            MAC_MODE_PORT_INT_LPBACK | MAC_MODE_LINK_POLARITY |
8472                            MAC_MODE_PORT_MODE_GMII;
8473                 tw32(MAC_MODE, mac_mode);
8474         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
8475                 tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK | BMCR_FULLDPLX |
8476                                            BMCR_SPEED1000);
8477                 udelay(40);
8478                 /* reset to prevent losing 1st rx packet intermittently */
8479                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
8480                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8481                         udelay(10);
8482                         tw32_f(MAC_RX_MODE, tp->rx_mode);
8483                 }
8484                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
8485                            MAC_MODE_LINK_POLARITY | MAC_MODE_PORT_MODE_GMII;
8486                 if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
8487                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
8488                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
8489                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
8490                 }
8491                 tw32(MAC_MODE, mac_mode);
8492         }
8493         else
8494                 return -EINVAL;
8495
8496         err = -EIO;
8497
8498         tx_len = 1514;
8499         skb = dev_alloc_skb(tx_len);
8500         if (!skb)
8501                 return -ENOMEM;
8502
8503         tx_data = skb_put(skb, tx_len);
8504         memcpy(tx_data, tp->dev->dev_addr, 6);
8505         memset(tx_data + 6, 0x0, 8);
8506
8507         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
8508
8509         for (i = 14; i < tx_len; i++)
8510                 tx_data[i] = (u8) (i & 0xff);
8511
8512         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
8513
8514         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8515              HOSTCC_MODE_NOW);
8516
8517         udelay(10);
8518
8519         rx_start_idx = tp->hw_status->idx[0].rx_producer;
8520
8521         num_pkts = 0;
8522
8523         tg3_set_txd(tp, tp->tx_prod, map, tx_len, 0, 1);
8524
8525         tp->tx_prod++;
8526         num_pkts++;
8527
8528         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW,
8529                      tp->tx_prod);
8530         tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW);
8531
8532         udelay(10);
8533
8534         for (i = 0; i < 10; i++) {
8535                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8536                        HOSTCC_MODE_NOW);
8537
8538                 udelay(10);
8539
8540                 tx_idx = tp->hw_status->idx[0].tx_consumer;
8541                 rx_idx = tp->hw_status->idx[0].rx_producer;
8542                 if ((tx_idx == tp->tx_prod) &&
8543                     (rx_idx == (rx_start_idx + num_pkts)))
8544                         break;
8545         }
8546
8547         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
8548         dev_kfree_skb(skb);
8549
8550         if (tx_idx != tp->tx_prod)
8551                 goto out;
8552
8553         if (rx_idx != rx_start_idx + num_pkts)
8554                 goto out;
8555
8556         desc = &tp->rx_rcb[rx_start_idx];
8557         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
8558         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
8559         if (opaque_key != RXD_OPAQUE_RING_STD)
8560                 goto out;
8561
8562         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
8563             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
8564                 goto out;
8565
8566         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
8567         if (rx_len != tx_len)
8568                 goto out;
8569
8570         rx_skb = tp->rx_std_buffers[desc_idx].skb;
8571
8572         map = pci_unmap_addr(&tp->rx_std_buffers[desc_idx], mapping);
8573         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
8574
8575         for (i = 14; i < tx_len; i++) {
8576                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
8577                         goto out;
8578         }
8579         err = 0;
8580
8581         /* tg3_free_rings will unmap and free the rx_skb */
8582 out:
8583         return err;
8584 }
8585
8586 #define TG3_MAC_LOOPBACK_FAILED         1
8587 #define TG3_PHY_LOOPBACK_FAILED         2
8588 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
8589                                          TG3_PHY_LOOPBACK_FAILED)
8590
8591 static int tg3_test_loopback(struct tg3 *tp)
8592 {
8593         int err = 0;
8594
8595         if (!netif_running(tp->dev))
8596                 return TG3_LOOPBACK_FAILED;
8597
8598         tg3_reset_hw(tp, 1);
8599
8600         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
8601                 err |= TG3_MAC_LOOPBACK_FAILED;
8602         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
8603                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
8604                         err |= TG3_PHY_LOOPBACK_FAILED;
8605         }
8606
8607         return err;
8608 }
8609
8610 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
8611                           u64 *data)
8612 {
8613         struct tg3 *tp = netdev_priv(dev);
8614
8615         if (tp->link_config.phy_is_low_power)
8616                 tg3_set_power_state(tp, PCI_D0);
8617
8618         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
8619
8620         if (tg3_test_nvram(tp) != 0) {
8621                 etest->flags |= ETH_TEST_FL_FAILED;
8622                 data[0] = 1;
8623         }
8624         if (tg3_test_link(tp) != 0) {
8625                 etest->flags |= ETH_TEST_FL_FAILED;
8626                 data[1] = 1;
8627         }
8628         if (etest->flags & ETH_TEST_FL_OFFLINE) {
8629                 int err, irq_sync = 0;
8630
8631                 if (netif_running(dev)) {
8632                         tg3_netif_stop(tp);
8633                         irq_sync = 1;
8634                 }
8635
8636                 tg3_full_lock(tp, irq_sync);
8637
8638                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
8639                 err = tg3_nvram_lock(tp);
8640                 tg3_halt_cpu(tp, RX_CPU_BASE);
8641                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
8642                         tg3_halt_cpu(tp, TX_CPU_BASE);
8643                 if (!err)
8644                         tg3_nvram_unlock(tp);
8645
8646                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
8647                         tg3_phy_reset(tp);
8648
8649                 if (tg3_test_registers(tp) != 0) {
8650                         etest->flags |= ETH_TEST_FL_FAILED;
8651                         data[2] = 1;
8652                 }
8653                 if (tg3_test_memory(tp) != 0) {
8654                         etest->flags |= ETH_TEST_FL_FAILED;
8655                         data[3] = 1;
8656                 }
8657                 if ((data[4] = tg3_test_loopback(tp)) != 0)
8658                         etest->flags |= ETH_TEST_FL_FAILED;
8659
8660                 tg3_full_unlock(tp);
8661
8662                 if (tg3_test_interrupt(tp) != 0) {
8663                         etest->flags |= ETH_TEST_FL_FAILED;
8664                         data[5] = 1;
8665                 }
8666
8667                 tg3_full_lock(tp, 0);
8668
8669                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8670                 if (netif_running(dev)) {
8671                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
8672                         tg3_init_hw(tp, 1);
8673                         tg3_netif_start(tp);
8674                 }
8675
8676                 tg3_full_unlock(tp);
8677         }
8678         if (tp->link_config.phy_is_low_power)
8679                 tg3_set_power_state(tp, PCI_D3hot);
8680
8681 }
8682
8683 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
8684 {
8685         struct mii_ioctl_data *data = if_mii(ifr);
8686         struct tg3 *tp = netdev_priv(dev);
8687         int err;
8688
8689         switch(cmd) {
8690         case SIOCGMIIPHY:
8691                 data->phy_id = PHY_ADDR;
8692
8693                 /* fallthru */
8694         case SIOCGMIIREG: {
8695                 u32 mii_regval;
8696
8697                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8698                         break;                  /* We have no PHY */
8699
8700                 if (tp->link_config.phy_is_low_power)
8701                         return -EAGAIN;
8702
8703                 spin_lock_bh(&tp->lock);
8704                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
8705                 spin_unlock_bh(&tp->lock);
8706
8707                 data->val_out = mii_regval;
8708
8709                 return err;
8710         }
8711
8712         case SIOCSMIIREG:
8713                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8714                         break;                  /* We have no PHY */
8715
8716                 if (!capable(CAP_NET_ADMIN))
8717                         return -EPERM;
8718
8719                 if (tp->link_config.phy_is_low_power)
8720                         return -EAGAIN;
8721
8722                 spin_lock_bh(&tp->lock);
8723                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
8724                 spin_unlock_bh(&tp->lock);
8725
8726                 return err;
8727
8728         default:
8729                 /* do nothing */
8730                 break;
8731         }
8732         return -EOPNOTSUPP;
8733 }
8734
8735 #if TG3_VLAN_TAG_USED
8736 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
8737 {
8738         struct tg3 *tp = netdev_priv(dev);
8739
8740         tg3_full_lock(tp, 0);
8741
8742         tp->vlgrp = grp;
8743
8744         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
8745         __tg3_set_rx_mode(dev);
8746
8747         tg3_full_unlock(tp);
8748 }
8749
8750 static void tg3_vlan_rx_kill_vid(struct net_device *dev, unsigned short vid)
8751 {
8752         struct tg3 *tp = netdev_priv(dev);
8753
8754         tg3_full_lock(tp, 0);
8755         if (tp->vlgrp)
8756                 tp->vlgrp->vlan_devices[vid] = NULL;
8757         tg3_full_unlock(tp);
8758 }
8759 #endif
8760
8761 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
8762 {
8763         struct tg3 *tp = netdev_priv(dev);
8764
8765         memcpy(ec, &tp->coal, sizeof(*ec));
8766         return 0;
8767 }
8768
8769 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
8770 {
8771         struct tg3 *tp = netdev_priv(dev);
8772         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
8773         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
8774
8775         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
8776                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
8777                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
8778                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
8779                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
8780         }
8781
8782         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
8783             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
8784             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
8785             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
8786             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
8787             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
8788             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
8789             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
8790             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
8791             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
8792                 return -EINVAL;
8793
8794         /* No rx interrupts will be generated if both are zero */
8795         if ((ec->rx_coalesce_usecs == 0) &&
8796             (ec->rx_max_coalesced_frames == 0))
8797                 return -EINVAL;
8798
8799         /* No tx interrupts will be generated if both are zero */
8800         if ((ec->tx_coalesce_usecs == 0) &&
8801             (ec->tx_max_coalesced_frames == 0))
8802                 return -EINVAL;
8803
8804         /* Only copy relevant parameters, ignore all others. */
8805         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
8806         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
8807         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
8808         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
8809         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
8810         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
8811         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
8812         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
8813         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
8814
8815         if (netif_running(dev)) {
8816                 tg3_full_lock(tp, 0);
8817                 __tg3_set_coalesce(tp, &tp->coal);
8818                 tg3_full_unlock(tp);
8819         }
8820         return 0;
8821 }
8822
8823 static struct ethtool_ops tg3_ethtool_ops = {
8824         .get_settings           = tg3_get_settings,
8825         .set_settings           = tg3_set_settings,
8826         .get_drvinfo            = tg3_get_drvinfo,
8827         .get_regs_len           = tg3_get_regs_len,
8828         .get_regs               = tg3_get_regs,
8829         .get_wol                = tg3_get_wol,
8830         .set_wol                = tg3_set_wol,
8831         .get_msglevel           = tg3_get_msglevel,
8832         .set_msglevel           = tg3_set_msglevel,
8833         .nway_reset             = tg3_nway_reset,
8834         .get_link               = ethtool_op_get_link,
8835         .get_eeprom_len         = tg3_get_eeprom_len,
8836         .get_eeprom             = tg3_get_eeprom,
8837         .set_eeprom             = tg3_set_eeprom,
8838         .get_ringparam          = tg3_get_ringparam,
8839         .set_ringparam          = tg3_set_ringparam,
8840         .get_pauseparam         = tg3_get_pauseparam,
8841         .set_pauseparam         = tg3_set_pauseparam,
8842         .get_rx_csum            = tg3_get_rx_csum,
8843         .set_rx_csum            = tg3_set_rx_csum,
8844         .get_tx_csum            = ethtool_op_get_tx_csum,
8845         .set_tx_csum            = tg3_set_tx_csum,
8846         .get_sg                 = ethtool_op_get_sg,
8847         .set_sg                 = ethtool_op_set_sg,
8848 #if TG3_TSO_SUPPORT != 0
8849         .get_tso                = ethtool_op_get_tso,
8850         .set_tso                = tg3_set_tso,
8851 #endif
8852         .self_test_count        = tg3_get_test_count,
8853         .self_test              = tg3_self_test,
8854         .get_strings            = tg3_get_strings,
8855         .phys_id                = tg3_phys_id,
8856         .get_stats_count        = tg3_get_stats_count,
8857         .get_ethtool_stats      = tg3_get_ethtool_stats,
8858         .get_coalesce           = tg3_get_coalesce,
8859         .set_coalesce           = tg3_set_coalesce,
8860         .get_perm_addr          = ethtool_op_get_perm_addr,
8861 };
8862
8863 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
8864 {
8865         u32 cursize, val, magic;
8866
8867         tp->nvram_size = EEPROM_CHIP_SIZE;
8868
8869         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8870                 return;
8871
8872         if ((magic != TG3_EEPROM_MAGIC) && ((magic & 0xff000000) != 0xa5000000))
8873                 return;
8874
8875         /*
8876          * Size the chip by reading offsets at increasing powers of two.
8877          * When we encounter our validation signature, we know the addressing
8878          * has wrapped around, and thus have our chip size.
8879          */
8880         cursize = 0x10;
8881
8882         while (cursize < tp->nvram_size) {
8883                 if (tg3_nvram_read_swab(tp, cursize, &val) != 0)
8884                         return;
8885
8886                 if (val == magic)
8887                         break;
8888
8889                 cursize <<= 1;
8890         }
8891
8892         tp->nvram_size = cursize;
8893 }
8894
8895 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
8896 {
8897         u32 val;
8898
8899         if (tg3_nvram_read_swab(tp, 0, &val) != 0)
8900                 return;
8901
8902         /* Selfboot format */
8903         if (val != TG3_EEPROM_MAGIC) {
8904                 tg3_get_eeprom_size(tp);
8905                 return;
8906         }
8907
8908         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
8909                 if (val != 0) {
8910                         tp->nvram_size = (val >> 16) * 1024;
8911                         return;
8912                 }
8913         }
8914         tp->nvram_size = 0x20000;
8915 }
8916
8917 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
8918 {
8919         u32 nvcfg1;
8920
8921         nvcfg1 = tr32(NVRAM_CFG1);
8922         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
8923                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
8924         }
8925         else {
8926                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
8927                 tw32(NVRAM_CFG1, nvcfg1);
8928         }
8929
8930         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
8931             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
8932                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
8933                         case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
8934                                 tp->nvram_jedecnum = JEDEC_ATMEL;
8935                                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
8936                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8937                                 break;
8938                         case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
8939                                 tp->nvram_jedecnum = JEDEC_ATMEL;
8940                                 tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
8941                                 break;
8942                         case FLASH_VENDOR_ATMEL_EEPROM:
8943                                 tp->nvram_jedecnum = JEDEC_ATMEL;
8944                                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
8945                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8946                                 break;
8947                         case FLASH_VENDOR_ST:
8948                                 tp->nvram_jedecnum = JEDEC_ST;
8949                                 tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
8950                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8951                                 break;
8952                         case FLASH_VENDOR_SAIFUN:
8953                                 tp->nvram_jedecnum = JEDEC_SAIFUN;
8954                                 tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
8955                                 break;
8956                         case FLASH_VENDOR_SST_SMALL:
8957                         case FLASH_VENDOR_SST_LARGE:
8958                                 tp->nvram_jedecnum = JEDEC_SST;
8959                                 tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
8960                                 break;
8961                 }
8962         }
8963         else {
8964                 tp->nvram_jedecnum = JEDEC_ATMEL;
8965                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
8966                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8967         }
8968 }
8969
8970 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
8971 {
8972         u32 nvcfg1;
8973
8974         nvcfg1 = tr32(NVRAM_CFG1);
8975
8976         /* NVRAM protection for TPM */
8977         if (nvcfg1 & (1 << 27))
8978                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
8979
8980         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
8981                 case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
8982                 case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
8983                         tp->nvram_jedecnum = JEDEC_ATMEL;
8984                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8985                         break;
8986                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
8987                         tp->nvram_jedecnum = JEDEC_ATMEL;
8988                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8989                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
8990                         break;
8991                 case FLASH_5752VENDOR_ST_M45PE10:
8992                 case FLASH_5752VENDOR_ST_M45PE20:
8993                 case FLASH_5752VENDOR_ST_M45PE40:
8994                         tp->nvram_jedecnum = JEDEC_ST;
8995                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
8996                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
8997                         break;
8998         }
8999
9000         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
9001                 switch (nvcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
9002                         case FLASH_5752PAGE_SIZE_256:
9003                                 tp->nvram_pagesize = 256;
9004                                 break;
9005                         case FLASH_5752PAGE_SIZE_512:
9006                                 tp->nvram_pagesize = 512;
9007                                 break;
9008                         case FLASH_5752PAGE_SIZE_1K:
9009                                 tp->nvram_pagesize = 1024;
9010                                 break;
9011                         case FLASH_5752PAGE_SIZE_2K:
9012                                 tp->nvram_pagesize = 2048;
9013                                 break;
9014                         case FLASH_5752PAGE_SIZE_4K:
9015                                 tp->nvram_pagesize = 4096;
9016                                 break;
9017                         case FLASH_5752PAGE_SIZE_264:
9018                                 tp->nvram_pagesize = 264;
9019                                 break;
9020                 }
9021         }
9022         else {
9023                 /* For eeprom, set pagesize to maximum eeprom size */
9024                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9025
9026                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9027                 tw32(NVRAM_CFG1, nvcfg1);
9028         }
9029 }
9030
9031 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
9032 {
9033         u32 nvcfg1;
9034
9035         nvcfg1 = tr32(NVRAM_CFG1);
9036
9037         /* NVRAM protection for TPM */
9038         if (nvcfg1 & (1 << 27))
9039                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9040
9041         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9042                 case FLASH_5755VENDOR_ATMEL_EEPROM_64KHZ:
9043                 case FLASH_5755VENDOR_ATMEL_EEPROM_376KHZ:
9044                         tp->nvram_jedecnum = JEDEC_ATMEL;
9045                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9046                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9047
9048                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9049                         tw32(NVRAM_CFG1, nvcfg1);
9050                         break;
9051                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9052                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9053                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9054                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9055                 case FLASH_5755VENDOR_ATMEL_FLASH_4:
9056                         tp->nvram_jedecnum = JEDEC_ATMEL;
9057                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9058                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9059                         tp->nvram_pagesize = 264;
9060                         break;
9061                 case FLASH_5752VENDOR_ST_M45PE10:
9062                 case FLASH_5752VENDOR_ST_M45PE20:
9063                 case FLASH_5752VENDOR_ST_M45PE40:
9064                         tp->nvram_jedecnum = JEDEC_ST;
9065                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9066                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9067                         tp->nvram_pagesize = 256;
9068                         break;
9069         }
9070 }
9071
9072 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
9073 {
9074         u32 nvcfg1;
9075
9076         nvcfg1 = tr32(NVRAM_CFG1);
9077
9078         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9079                 case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
9080                 case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
9081                 case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
9082                 case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
9083                         tp->nvram_jedecnum = JEDEC_ATMEL;
9084                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9085                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9086
9087                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9088                         tw32(NVRAM_CFG1, nvcfg1);
9089                         break;
9090                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9091                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9092                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9093                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9094                         tp->nvram_jedecnum = JEDEC_ATMEL;
9095                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9096                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9097                         tp->nvram_pagesize = 264;
9098                         break;
9099                 case FLASH_5752VENDOR_ST_M45PE10:
9100                 case FLASH_5752VENDOR_ST_M45PE20:
9101                 case FLASH_5752VENDOR_ST_M45PE40:
9102                         tp->nvram_jedecnum = JEDEC_ST;
9103                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9104                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9105                         tp->nvram_pagesize = 256;
9106                         break;
9107         }
9108 }
9109
9110 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
9111 static void __devinit tg3_nvram_init(struct tg3 *tp)
9112 {
9113         int j;
9114
9115         tw32_f(GRC_EEPROM_ADDR,
9116              (EEPROM_ADDR_FSM_RESET |
9117               (EEPROM_DEFAULT_CLOCK_PERIOD <<
9118                EEPROM_ADDR_CLKPERD_SHIFT)));
9119
9120         /* XXX schedule_timeout() ... */
9121         for (j = 0; j < 100; j++)
9122                 udelay(10);
9123
9124         /* Enable seeprom accesses. */
9125         tw32_f(GRC_LOCAL_CTRL,
9126              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
9127         udelay(100);
9128
9129         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
9130             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
9131                 tp->tg3_flags |= TG3_FLAG_NVRAM;
9132
9133                 if (tg3_nvram_lock(tp)) {
9134                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
9135                                "tg3_nvram_init failed.\n", tp->dev->name);
9136                         return;
9137                 }
9138                 tg3_enable_nvram_access(tp);
9139
9140                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
9141                         tg3_get_5752_nvram_info(tp);
9142                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
9143                         tg3_get_5755_nvram_info(tp);
9144                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
9145                         tg3_get_5787_nvram_info(tp);
9146                 else
9147                         tg3_get_nvram_info(tp);
9148
9149                 tg3_get_nvram_size(tp);
9150
9151                 tg3_disable_nvram_access(tp);
9152                 tg3_nvram_unlock(tp);
9153
9154         } else {
9155                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
9156
9157                 tg3_get_eeprom_size(tp);
9158         }
9159 }
9160
9161 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
9162                                         u32 offset, u32 *val)
9163 {
9164         u32 tmp;
9165         int i;
9166
9167         if (offset > EEPROM_ADDR_ADDR_MASK ||
9168             (offset % 4) != 0)
9169                 return -EINVAL;
9170
9171         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
9172                                         EEPROM_ADDR_DEVID_MASK |
9173                                         EEPROM_ADDR_READ);
9174         tw32(GRC_EEPROM_ADDR,
9175              tmp |
9176              (0 << EEPROM_ADDR_DEVID_SHIFT) |
9177              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
9178               EEPROM_ADDR_ADDR_MASK) |
9179              EEPROM_ADDR_READ | EEPROM_ADDR_START);
9180
9181         for (i = 0; i < 10000; i++) {
9182                 tmp = tr32(GRC_EEPROM_ADDR);
9183
9184                 if (tmp & EEPROM_ADDR_COMPLETE)
9185                         break;
9186                 udelay(100);
9187         }
9188         if (!(tmp & EEPROM_ADDR_COMPLETE))
9189                 return -EBUSY;
9190
9191         *val = tr32(GRC_EEPROM_DATA);
9192         return 0;
9193 }
9194
9195 #define NVRAM_CMD_TIMEOUT 10000
9196
9197 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
9198 {
9199         int i;
9200
9201         tw32(NVRAM_CMD, nvram_cmd);
9202         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
9203                 udelay(10);
9204                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
9205                         udelay(10);
9206                         break;
9207                 }
9208         }
9209         if (i == NVRAM_CMD_TIMEOUT) {
9210                 return -EBUSY;
9211         }
9212         return 0;
9213 }
9214
9215 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
9216 {
9217         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
9218             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
9219             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
9220             (tp->nvram_jedecnum == JEDEC_ATMEL))
9221
9222                 addr = ((addr / tp->nvram_pagesize) <<
9223                         ATMEL_AT45DB0X1B_PAGE_POS) +
9224                        (addr % tp->nvram_pagesize);
9225
9226         return addr;
9227 }
9228
9229 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
9230 {
9231         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
9232             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
9233             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
9234             (tp->nvram_jedecnum == JEDEC_ATMEL))
9235
9236                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
9237                         tp->nvram_pagesize) +
9238                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
9239
9240         return addr;
9241 }
9242
9243 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
9244 {
9245         int ret;
9246
9247         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
9248                 return tg3_nvram_read_using_eeprom(tp, offset, val);
9249
9250         offset = tg3_nvram_phys_addr(tp, offset);
9251
9252         if (offset > NVRAM_ADDR_MSK)
9253                 return -EINVAL;
9254
9255         ret = tg3_nvram_lock(tp);
9256         if (ret)
9257                 return ret;
9258
9259         tg3_enable_nvram_access(tp);
9260
9261         tw32(NVRAM_ADDR, offset);
9262         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
9263                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
9264
9265         if (ret == 0)
9266                 *val = swab32(tr32(NVRAM_RDDATA));
9267
9268         tg3_disable_nvram_access(tp);
9269
9270         tg3_nvram_unlock(tp);
9271
9272         return ret;
9273 }
9274
9275 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val)
9276 {
9277         int err;
9278         u32 tmp;
9279
9280         err = tg3_nvram_read(tp, offset, &tmp);
9281         *val = swab32(tmp);
9282         return err;
9283 }
9284
9285 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
9286                                     u32 offset, u32 len, u8 *buf)
9287 {
9288         int i, j, rc = 0;
9289         u32 val;
9290
9291         for (i = 0; i < len; i += 4) {
9292                 u32 addr, data;
9293
9294                 addr = offset + i;
9295
9296                 memcpy(&data, buf + i, 4);
9297
9298                 tw32(GRC_EEPROM_DATA, cpu_to_le32(data));
9299
9300                 val = tr32(GRC_EEPROM_ADDR);
9301                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
9302
9303                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
9304                         EEPROM_ADDR_READ);
9305                 tw32(GRC_EEPROM_ADDR, val |
9306                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
9307                         (addr & EEPROM_ADDR_ADDR_MASK) |
9308                         EEPROM_ADDR_START |
9309                         EEPROM_ADDR_WRITE);
9310
9311                 for (j = 0; j < 10000; j++) {
9312                         val = tr32(GRC_EEPROM_ADDR);
9313
9314                         if (val & EEPROM_ADDR_COMPLETE)
9315                                 break;
9316                         udelay(100);
9317                 }
9318                 if (!(val & EEPROM_ADDR_COMPLETE)) {
9319                         rc = -EBUSY;
9320                         break;
9321                 }
9322         }
9323
9324         return rc;
9325 }
9326
9327 /* offset and length are dword aligned */
9328 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
9329                 u8 *buf)
9330 {
9331         int ret = 0;
9332         u32 pagesize = tp->nvram_pagesize;
9333         u32 pagemask = pagesize - 1;
9334         u32 nvram_cmd;
9335         u8 *tmp;
9336
9337         tmp = kmalloc(pagesize, GFP_KERNEL);
9338         if (tmp == NULL)
9339                 return -ENOMEM;
9340
9341         while (len) {
9342                 int j;
9343                 u32 phy_addr, page_off, size;
9344
9345                 phy_addr = offset & ~pagemask;
9346
9347                 for (j = 0; j < pagesize; j += 4) {
9348                         if ((ret = tg3_nvram_read(tp, phy_addr + j,
9349                                                 (u32 *) (tmp + j))))
9350                                 break;
9351                 }
9352                 if (ret)
9353                         break;
9354
9355                 page_off = offset & pagemask;
9356                 size = pagesize;
9357                 if (len < size)
9358                         size = len;
9359
9360                 len -= size;
9361
9362                 memcpy(tmp + page_off, buf, size);
9363
9364                 offset = offset + (pagesize - page_off);
9365
9366                 tg3_enable_nvram_access(tp);
9367
9368                 /*
9369                  * Before we can erase the flash page, we need
9370                  * to issue a special "write enable" command.
9371                  */
9372                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9373
9374                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9375                         break;
9376
9377                 /* Erase the target page */
9378                 tw32(NVRAM_ADDR, phy_addr);
9379
9380                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
9381                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
9382
9383                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9384                         break;
9385
9386                 /* Issue another write enable to start the write. */
9387                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9388
9389                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9390                         break;
9391
9392                 for (j = 0; j < pagesize; j += 4) {
9393                         u32 data;
9394
9395                         data = *((u32 *) (tmp + j));
9396                         tw32(NVRAM_WRDATA, cpu_to_be32(data));
9397
9398                         tw32(NVRAM_ADDR, phy_addr + j);
9399
9400                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
9401                                 NVRAM_CMD_WR;
9402
9403                         if (j == 0)
9404                                 nvram_cmd |= NVRAM_CMD_FIRST;
9405                         else if (j == (pagesize - 4))
9406                                 nvram_cmd |= NVRAM_CMD_LAST;
9407
9408                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
9409                                 break;
9410                 }
9411                 if (ret)
9412                         break;
9413         }
9414
9415         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9416         tg3_nvram_exec_cmd(tp, nvram_cmd);
9417
9418         kfree(tmp);
9419
9420         return ret;
9421 }
9422
9423 /* offset and length are dword aligned */
9424 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
9425                 u8 *buf)
9426 {
9427         int i, ret = 0;
9428
9429         for (i = 0; i < len; i += 4, offset += 4) {
9430                 u32 data, page_off, phy_addr, nvram_cmd;
9431
9432                 memcpy(&data, buf + i, 4);
9433                 tw32(NVRAM_WRDATA, cpu_to_be32(data));
9434
9435                 page_off = offset % tp->nvram_pagesize;
9436
9437                 phy_addr = tg3_nvram_phys_addr(tp, offset);
9438
9439                 tw32(NVRAM_ADDR, phy_addr);
9440
9441                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
9442
9443                 if ((page_off == 0) || (i == 0))
9444                         nvram_cmd |= NVRAM_CMD_FIRST;
9445                 if (page_off == (tp->nvram_pagesize - 4))
9446                         nvram_cmd |= NVRAM_CMD_LAST;
9447
9448                 if (i == (len - 4))
9449                         nvram_cmd |= NVRAM_CMD_LAST;
9450
9451                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752) &&
9452                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755) &&
9453                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787) &&
9454                     (tp->nvram_jedecnum == JEDEC_ST) &&
9455                     (nvram_cmd & NVRAM_CMD_FIRST)) {
9456
9457                         if ((ret = tg3_nvram_exec_cmd(tp,
9458                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
9459                                 NVRAM_CMD_DONE)))
9460
9461                                 break;
9462                 }
9463                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
9464                         /* We always do complete word writes to eeprom. */
9465                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
9466                 }
9467
9468                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
9469                         break;
9470         }
9471         return ret;
9472 }
9473
9474 /* offset and length are dword aligned */
9475 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
9476 {
9477         int ret;
9478
9479         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
9480                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
9481                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
9482                 udelay(40);
9483         }
9484
9485         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
9486                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
9487         }
9488         else {
9489                 u32 grc_mode;
9490
9491                 ret = tg3_nvram_lock(tp);
9492                 if (ret)
9493                         return ret;
9494
9495                 tg3_enable_nvram_access(tp);
9496                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
9497                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
9498                         tw32(NVRAM_WRITE1, 0x406);
9499
9500                 grc_mode = tr32(GRC_MODE);
9501                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
9502
9503                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
9504                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
9505
9506                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
9507                                 buf);
9508                 }
9509                 else {
9510                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
9511                                 buf);
9512                 }
9513
9514                 grc_mode = tr32(GRC_MODE);
9515                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
9516
9517                 tg3_disable_nvram_access(tp);
9518                 tg3_nvram_unlock(tp);
9519         }
9520
9521         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
9522                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
9523                 udelay(40);
9524         }
9525
9526         return ret;
9527 }
9528
9529 struct subsys_tbl_ent {
9530         u16 subsys_vendor, subsys_devid;
9531         u32 phy_id;
9532 };
9533
9534 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
9535         /* Broadcom boards. */
9536         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
9537         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
9538         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
9539         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
9540         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
9541         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
9542         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
9543         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
9544         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
9545         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
9546         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
9547
9548         /* 3com boards. */
9549         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
9550         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
9551         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
9552         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
9553         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
9554
9555         /* DELL boards. */
9556         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
9557         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
9558         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
9559         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
9560
9561         /* Compaq boards. */
9562         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
9563         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
9564         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
9565         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
9566         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
9567
9568         /* IBM boards. */
9569         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
9570 };
9571
9572 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
9573 {
9574         int i;
9575
9576         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
9577                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
9578                      tp->pdev->subsystem_vendor) &&
9579                     (subsys_id_to_phy_id[i].subsys_devid ==
9580                      tp->pdev->subsystem_device))
9581                         return &subsys_id_to_phy_id[i];
9582         }
9583         return NULL;
9584 }
9585
9586 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
9587 {
9588         u32 val;
9589         u16 pmcsr;
9590
9591         /* On some early chips the SRAM cannot be accessed in D3hot state,
9592          * so need make sure we're in D0.
9593          */
9594         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
9595         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
9596         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
9597         msleep(1);
9598
9599         /* Make sure register accesses (indirect or otherwise)
9600          * will function correctly.
9601          */
9602         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
9603                                tp->misc_host_ctrl);
9604
9605         /* The memory arbiter has to be enabled in order for SRAM accesses
9606          * to succeed.  Normally on powerup the tg3 chip firmware will make
9607          * sure it is enabled, but other entities such as system netboot
9608          * code might disable it.
9609          */
9610         val = tr32(MEMARB_MODE);
9611         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
9612
9613         tp->phy_id = PHY_ID_INVALID;
9614         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9615
9616         /* Assume an onboard device by default.  */
9617         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
9618
9619         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
9620         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
9621                 u32 nic_cfg, led_cfg;
9622                 u32 nic_phy_id, ver, cfg2 = 0, eeprom_phy_id;
9623                 int eeprom_phy_serdes = 0;
9624
9625                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
9626                 tp->nic_sram_data_cfg = nic_cfg;
9627
9628                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
9629                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
9630                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
9631                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
9632                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
9633                     (ver > 0) && (ver < 0x100))
9634                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
9635
9636                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
9637                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
9638                         eeprom_phy_serdes = 1;
9639
9640                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
9641                 if (nic_phy_id != 0) {
9642                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
9643                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
9644
9645                         eeprom_phy_id  = (id1 >> 16) << 10;
9646                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
9647                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
9648                 } else
9649                         eeprom_phy_id = 0;
9650
9651                 tp->phy_id = eeprom_phy_id;
9652                 if (eeprom_phy_serdes) {
9653                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
9654                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
9655                         else
9656                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9657                 }
9658
9659                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9660                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
9661                                     SHASTA_EXT_LED_MODE_MASK);
9662                 else
9663                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
9664
9665                 switch (led_cfg) {
9666                 default:
9667                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
9668                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9669                         break;
9670
9671                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
9672                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
9673                         break;
9674
9675                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
9676                         tp->led_ctrl = LED_CTRL_MODE_MAC;
9677
9678                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
9679                          * read on some older 5700/5701 bootcode.
9680                          */
9681                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
9682                             ASIC_REV_5700 ||
9683                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
9684                             ASIC_REV_5701)
9685                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9686
9687                         break;
9688
9689                 case SHASTA_EXT_LED_SHARED:
9690                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
9691                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
9692                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
9693                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
9694                                                  LED_CTRL_MODE_PHY_2);
9695                         break;
9696
9697                 case SHASTA_EXT_LED_MAC:
9698                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
9699                         break;
9700
9701                 case SHASTA_EXT_LED_COMBO:
9702                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
9703                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
9704                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
9705                                                  LED_CTRL_MODE_PHY_2);
9706                         break;
9707
9708                 };
9709
9710                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
9711                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
9712                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
9713                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
9714
9715                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP)
9716                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
9717                 else
9718                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
9719
9720                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
9721                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
9722                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9723                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
9724                 }
9725                 if (nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL)
9726                         tp->tg3_flags |= TG3_FLAG_SERDES_WOL_CAP;
9727
9728                 if (cfg2 & (1 << 17))
9729                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
9730
9731                 /* serdes signal pre-emphasis in register 0x590 set by */
9732                 /* bootcode if bit 18 is set */
9733                 if (cfg2 & (1 << 18))
9734                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
9735         }
9736 }
9737
9738 static int __devinit tg3_phy_probe(struct tg3 *tp)
9739 {
9740         u32 hw_phy_id_1, hw_phy_id_2;
9741         u32 hw_phy_id, hw_phy_id_masked;
9742         int err;
9743
9744         /* Reading the PHY ID register can conflict with ASF
9745          * firwmare access to the PHY hardware.
9746          */
9747         err = 0;
9748         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
9749                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
9750         } else {
9751                 /* Now read the physical PHY_ID from the chip and verify
9752                  * that it is sane.  If it doesn't look good, we fall back
9753                  * to either the hard-coded table based PHY_ID and failing
9754                  * that the value found in the eeprom area.
9755                  */
9756                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
9757                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
9758
9759                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
9760                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
9761                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
9762
9763                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
9764         }
9765
9766         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
9767                 tp->phy_id = hw_phy_id;
9768                 if (hw_phy_id_masked == PHY_ID_BCM8002)
9769                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9770                 else
9771                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
9772         } else {
9773                 if (tp->phy_id != PHY_ID_INVALID) {
9774                         /* Do nothing, phy ID already set up in
9775                          * tg3_get_eeprom_hw_cfg().
9776                          */
9777                 } else {
9778                         struct subsys_tbl_ent *p;
9779
9780                         /* No eeprom signature?  Try the hardcoded
9781                          * subsys device table.
9782                          */
9783                         p = lookup_by_subsys(tp);
9784                         if (!p)
9785                                 return -ENODEV;
9786
9787                         tp->phy_id = p->phy_id;
9788                         if (!tp->phy_id ||
9789                             tp->phy_id == PHY_ID_BCM8002)
9790                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9791                 }
9792         }
9793
9794         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
9795             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
9796                 u32 bmsr, adv_reg, tg3_ctrl;
9797
9798                 tg3_readphy(tp, MII_BMSR, &bmsr);
9799                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
9800                     (bmsr & BMSR_LSTATUS))
9801                         goto skip_phy_reset;
9802
9803                 err = tg3_phy_reset(tp);
9804                 if (err)
9805                         return err;
9806
9807                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
9808                            ADVERTISE_100HALF | ADVERTISE_100FULL |
9809                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
9810                 tg3_ctrl = 0;
9811                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
9812                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
9813                                     MII_TG3_CTRL_ADV_1000_FULL);
9814                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
9815                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
9816                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
9817                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
9818                 }
9819
9820                 if (!tg3_copper_is_advertising_all(tp)) {
9821                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
9822
9823                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
9824                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
9825
9826                         tg3_writephy(tp, MII_BMCR,
9827                                      BMCR_ANENABLE | BMCR_ANRESTART);
9828                 }
9829                 tg3_phy_set_wirespeed(tp);
9830
9831                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
9832                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
9833                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
9834         }
9835
9836 skip_phy_reset:
9837         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
9838                 err = tg3_init_5401phy_dsp(tp);
9839                 if (err)
9840                         return err;
9841         }
9842
9843         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
9844                 err = tg3_init_5401phy_dsp(tp);
9845         }
9846
9847         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
9848                 tp->link_config.advertising =
9849                         (ADVERTISED_1000baseT_Half |
9850                          ADVERTISED_1000baseT_Full |
9851                          ADVERTISED_Autoneg |
9852                          ADVERTISED_FIBRE);
9853         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
9854                 tp->link_config.advertising &=
9855                         ~(ADVERTISED_1000baseT_Half |
9856                           ADVERTISED_1000baseT_Full);
9857
9858         return err;
9859 }
9860
9861 static void __devinit tg3_read_partno(struct tg3 *tp)
9862 {
9863         unsigned char vpd_data[256];
9864         int i;
9865         u32 magic;
9866
9867         if (tg3_nvram_read_swab(tp, 0x0, &magic))
9868                 goto out_not_found;
9869
9870         if (magic == TG3_EEPROM_MAGIC) {
9871                 for (i = 0; i < 256; i += 4) {
9872                         u32 tmp;
9873
9874                         if (tg3_nvram_read(tp, 0x100 + i, &tmp))
9875                                 goto out_not_found;
9876
9877                         vpd_data[i + 0] = ((tmp >>  0) & 0xff);
9878                         vpd_data[i + 1] = ((tmp >>  8) & 0xff);
9879                         vpd_data[i + 2] = ((tmp >> 16) & 0xff);
9880                         vpd_data[i + 3] = ((tmp >> 24) & 0xff);
9881                 }
9882         } else {
9883                 int vpd_cap;
9884
9885                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
9886                 for (i = 0; i < 256; i += 4) {
9887                         u32 tmp, j = 0;
9888                         u16 tmp16;
9889
9890                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
9891                                               i);
9892                         while (j++ < 100) {
9893                                 pci_read_config_word(tp->pdev, vpd_cap +
9894                                                      PCI_VPD_ADDR, &tmp16);
9895                                 if (tmp16 & 0x8000)
9896                                         break;
9897                                 msleep(1);
9898                         }
9899                         if (!(tmp16 & 0x8000))
9900                                 goto out_not_found;
9901
9902                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
9903                                               &tmp);
9904                         tmp = cpu_to_le32(tmp);
9905                         memcpy(&vpd_data[i], &tmp, 4);
9906                 }
9907         }
9908
9909         /* Now parse and find the part number. */
9910         for (i = 0; i < 256; ) {
9911                 unsigned char val = vpd_data[i];
9912                 int block_end;
9913
9914                 if (val == 0x82 || val == 0x91) {
9915                         i = (i + 3 +
9916                              (vpd_data[i + 1] +
9917                               (vpd_data[i + 2] << 8)));
9918                         continue;
9919                 }
9920
9921                 if (val != 0x90)
9922                         goto out_not_found;
9923
9924                 block_end = (i + 3 +
9925                              (vpd_data[i + 1] +
9926                               (vpd_data[i + 2] << 8)));
9927                 i += 3;
9928                 while (i < block_end) {
9929                         if (vpd_data[i + 0] == 'P' &&
9930                             vpd_data[i + 1] == 'N') {
9931                                 int partno_len = vpd_data[i + 2];
9932
9933                                 if (partno_len > 24)
9934                                         goto out_not_found;
9935
9936                                 memcpy(tp->board_part_number,
9937                                        &vpd_data[i + 3],
9938                                        partno_len);
9939
9940                                 /* Success. */
9941                                 return;
9942                         }
9943                 }
9944
9945                 /* Part number not found. */
9946                 goto out_not_found;
9947         }
9948
9949 out_not_found:
9950         strcpy(tp->board_part_number, "none");
9951 }
9952
9953 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
9954 {
9955         u32 val, offset, start;
9956
9957         if (tg3_nvram_read_swab(tp, 0, &val))
9958                 return;
9959
9960         if (val != TG3_EEPROM_MAGIC)
9961                 return;
9962
9963         if (tg3_nvram_read_swab(tp, 0xc, &offset) ||
9964             tg3_nvram_read_swab(tp, 0x4, &start))
9965                 return;
9966
9967         offset = tg3_nvram_logical_addr(tp, offset);
9968         if (tg3_nvram_read_swab(tp, offset, &val))
9969                 return;
9970
9971         if ((val & 0xfc000000) == 0x0c000000) {
9972                 u32 ver_offset, addr;
9973                 int i;
9974
9975                 if (tg3_nvram_read_swab(tp, offset + 4, &val) ||
9976                     tg3_nvram_read_swab(tp, offset + 8, &ver_offset))
9977                         return;
9978
9979                 if (val != 0)
9980                         return;
9981
9982                 addr = offset + ver_offset - start;
9983                 for (i = 0; i < 16; i += 4) {
9984                         if (tg3_nvram_read(tp, addr + i, &val))
9985                                 return;
9986
9987                         val = cpu_to_le32(val);
9988                         memcpy(tp->fw_ver + i, &val, 4);
9989                 }
9990         }
9991 }
9992
9993 static int __devinit tg3_get_invariants(struct tg3 *tp)
9994 {
9995         static struct pci_device_id write_reorder_chipsets[] = {
9996                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
9997                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
9998                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
9999                              PCI_DEVICE_ID_VIA_8385_0) },
10000                 { },
10001         };
10002         u32 misc_ctrl_reg;
10003         u32 cacheline_sz_reg;
10004         u32 pci_state_reg, grc_misc_cfg;
10005         u32 val;
10006         u16 pci_cmd;
10007         int err;
10008
10009         /* Force memory write invalidate off.  If we leave it on,
10010          * then on 5700_BX chips we have to enable a workaround.
10011          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
10012          * to match the cacheline size.  The Broadcom driver have this
10013          * workaround but turns MWI off all the times so never uses
10014          * it.  This seems to suggest that the workaround is insufficient.
10015          */
10016         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10017         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
10018         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10019
10020         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
10021          * has the register indirect write enable bit set before
10022          * we try to access any of the MMIO registers.  It is also
10023          * critical that the PCI-X hw workaround situation is decided
10024          * before that as well.
10025          */
10026         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10027                               &misc_ctrl_reg);
10028
10029         tp->pci_chip_rev_id = (misc_ctrl_reg >>
10030                                MISC_HOST_CTRL_CHIPREV_SHIFT);
10031
10032         /* Wrong chip ID in 5752 A0. This code can be removed later
10033          * as A0 is not in production.
10034          */
10035         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
10036                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
10037
10038         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
10039          * we need to disable memory and use config. cycles
10040          * only to access all registers. The 5702/03 chips
10041          * can mistakenly decode the special cycles from the
10042          * ICH chipsets as memory write cycles, causing corruption
10043          * of register and memory space. Only certain ICH bridges
10044          * will drive special cycles with non-zero data during the
10045          * address phase which can fall within the 5703's address
10046          * range. This is not an ICH bug as the PCI spec allows
10047          * non-zero address during special cycles. However, only
10048          * these ICH bridges are known to drive non-zero addresses
10049          * during special cycles.
10050          *
10051          * Since special cycles do not cross PCI bridges, we only
10052          * enable this workaround if the 5703 is on the secondary
10053          * bus of these ICH bridges.
10054          */
10055         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
10056             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
10057                 static struct tg3_dev_id {
10058                         u32     vendor;
10059                         u32     device;
10060                         u32     rev;
10061                 } ich_chipsets[] = {
10062                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
10063                           PCI_ANY_ID },
10064                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
10065                           PCI_ANY_ID },
10066                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
10067                           0xa },
10068                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
10069                           PCI_ANY_ID },
10070                         { },
10071                 };
10072                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
10073                 struct pci_dev *bridge = NULL;
10074
10075                 while (pci_id->vendor != 0) {
10076                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
10077                                                 bridge);
10078                         if (!bridge) {
10079                                 pci_id++;
10080                                 continue;
10081                         }
10082                         if (pci_id->rev != PCI_ANY_ID) {
10083                                 u8 rev;
10084
10085                                 pci_read_config_byte(bridge, PCI_REVISION_ID,
10086                                                      &rev);
10087                                 if (rev > pci_id->rev)
10088                                         continue;
10089                         }
10090                         if (bridge->subordinate &&
10091                             (bridge->subordinate->number ==
10092                              tp->pdev->bus->number)) {
10093
10094                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
10095                                 pci_dev_put(bridge);
10096                                 break;
10097                         }
10098                 }
10099         }
10100
10101         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
10102          * DMA addresses > 40-bit. This bridge may have other additional
10103          * 57xx devices behind it in some 4-port NIC designs for example.
10104          * Any tg3 device found behind the bridge will also need the 40-bit
10105          * DMA workaround.
10106          */
10107         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
10108             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
10109                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
10110                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
10111                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
10112         }
10113         else {
10114                 struct pci_dev *bridge = NULL;
10115
10116                 do {
10117                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
10118                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
10119                                                 bridge);
10120                         if (bridge && bridge->subordinate &&
10121                             (bridge->subordinate->number <=
10122                              tp->pdev->bus->number) &&
10123                             (bridge->subordinate->subordinate >=
10124                              tp->pdev->bus->number)) {
10125                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
10126                                 pci_dev_put(bridge);
10127                                 break;
10128                         }
10129                 } while (bridge);
10130         }
10131
10132         /* Initialize misc host control in PCI block. */
10133         tp->misc_host_ctrl |= (misc_ctrl_reg &
10134                                MISC_HOST_CTRL_CHIPREV);
10135         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10136                                tp->misc_host_ctrl);
10137
10138         pci_read_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
10139                               &cacheline_sz_reg);
10140
10141         tp->pci_cacheline_sz = (cacheline_sz_reg >>  0) & 0xff;
10142         tp->pci_lat_timer    = (cacheline_sz_reg >>  8) & 0xff;
10143         tp->pci_hdr_type     = (cacheline_sz_reg >> 16) & 0xff;
10144         tp->pci_bist         = (cacheline_sz_reg >> 24) & 0xff;
10145
10146         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
10147             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
10148             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10149             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
10150             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
10151                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
10152
10153         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
10154             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
10155                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
10156
10157         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
10158                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10159                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787) {
10160                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
10161                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
10162                 } else
10163                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1;
10164         }
10165
10166         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
10167             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
10168             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
10169             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755 &&
10170             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787)
10171                 tp->tg3_flags2 |= TG3_FLG2_JUMBO_CAPABLE;
10172
10173         if (pci_find_capability(tp->pdev, PCI_CAP_ID_EXP) != 0)
10174                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
10175
10176         /* If we have an AMD 762 or VIA K8T800 chipset, write
10177          * reordering to the mailbox registers done by the host
10178          * controller can cause major troubles.  We read back from
10179          * every mailbox register write to force the writes to be
10180          * posted to the chip in order.
10181          */
10182         if (pci_dev_present(write_reorder_chipsets) &&
10183             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
10184                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
10185
10186         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
10187             tp->pci_lat_timer < 64) {
10188                 tp->pci_lat_timer = 64;
10189
10190                 cacheline_sz_reg  = ((tp->pci_cacheline_sz & 0xff) <<  0);
10191                 cacheline_sz_reg |= ((tp->pci_lat_timer    & 0xff) <<  8);
10192                 cacheline_sz_reg |= ((tp->pci_hdr_type     & 0xff) << 16);
10193                 cacheline_sz_reg |= ((tp->pci_bist         & 0xff) << 24);
10194
10195                 pci_write_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
10196                                        cacheline_sz_reg);
10197         }
10198
10199         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
10200                               &pci_state_reg);
10201
10202         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0) {
10203                 tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
10204
10205                 /* If this is a 5700 BX chipset, and we are in PCI-X
10206                  * mode, enable register write workaround.
10207                  *
10208                  * The workaround is to use indirect register accesses
10209                  * for all chip writes not to mailbox registers.
10210                  */
10211                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
10212                         u32 pm_reg;
10213                         u16 pci_cmd;
10214
10215                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
10216
10217                         /* The chip can have it's power management PCI config
10218                          * space registers clobbered due to this bug.
10219                          * So explicitly force the chip into D0 here.
10220                          */
10221                         pci_read_config_dword(tp->pdev, TG3PCI_PM_CTRL_STAT,
10222                                               &pm_reg);
10223                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
10224                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
10225                         pci_write_config_dword(tp->pdev, TG3PCI_PM_CTRL_STAT,
10226                                                pm_reg);
10227
10228                         /* Also, force SERR#/PERR# in PCI command. */
10229                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10230                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
10231                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10232                 }
10233         }
10234
10235         /* 5700 BX chips need to have their TX producer index mailboxes
10236          * written twice to workaround a bug.
10237          */
10238         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX)
10239                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
10240
10241         /* Back to back register writes can cause problems on this chip,
10242          * the workaround is to read back all reg writes except those to
10243          * mailbox regs.  See tg3_write_indirect_reg32().
10244          *
10245          * PCI Express 5750_A0 rev chips need this workaround too.
10246          */
10247         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
10248             ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
10249              tp->pci_chip_rev_id == CHIPREV_ID_5750_A0))
10250                 tp->tg3_flags |= TG3_FLAG_5701_REG_WRITE_BUG;
10251
10252         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
10253                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
10254         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
10255                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
10256
10257         /* Chip-specific fixup from Broadcom driver */
10258         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
10259             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
10260                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
10261                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
10262         }
10263
10264         /* Default fast path register access methods */
10265         tp->read32 = tg3_read32;
10266         tp->write32 = tg3_write32;
10267         tp->read32_mbox = tg3_read32;
10268         tp->write32_mbox = tg3_write32;
10269         tp->write32_tx_mbox = tg3_write32;
10270         tp->write32_rx_mbox = tg3_write32;
10271
10272         /* Various workaround register access methods */
10273         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
10274                 tp->write32 = tg3_write_indirect_reg32;
10275         else if (tp->tg3_flags & TG3_FLAG_5701_REG_WRITE_BUG)
10276                 tp->write32 = tg3_write_flush_reg32;
10277
10278         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
10279             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
10280                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
10281                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
10282                         tp->write32_rx_mbox = tg3_write_flush_reg32;
10283         }
10284
10285         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
10286                 tp->read32 = tg3_read_indirect_reg32;
10287                 tp->write32 = tg3_write_indirect_reg32;
10288                 tp->read32_mbox = tg3_read_indirect_mbox;
10289                 tp->write32_mbox = tg3_write_indirect_mbox;
10290                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
10291                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
10292
10293                 iounmap(tp->regs);
10294                 tp->regs = NULL;
10295
10296                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10297                 pci_cmd &= ~PCI_COMMAND_MEMORY;
10298                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10299         }
10300
10301         if (tp->write32 == tg3_write_indirect_reg32 ||
10302             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
10303              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10304               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
10305                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
10306
10307         /* Get eeprom hw config before calling tg3_set_power_state().
10308          * In particular, the TG3_FLAG_EEPROM_WRITE_PROT flag must be
10309          * determined before calling tg3_set_power_state() so that
10310          * we know whether or not to switch out of Vaux power.
10311          * When the flag is set, it means that GPIO1 is used for eeprom
10312          * write protect and also implies that it is a LOM where GPIOs
10313          * are not used to switch power.
10314          */
10315         tg3_get_eeprom_hw_cfg(tp);
10316
10317         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
10318          * GPIO1 driven high will bring 5700's external PHY out of reset.
10319          * It is also used as eeprom write protect on LOMs.
10320          */
10321         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
10322         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
10323             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
10324                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
10325                                        GRC_LCLCTRL_GPIO_OUTPUT1);
10326         /* Unused GPIO3 must be driven as output on 5752 because there
10327          * are no pull-up resistors on unused GPIO pins.
10328          */
10329         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
10330                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
10331
10332         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10333                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
10334
10335         /* Force the chip into D0. */
10336         err = tg3_set_power_state(tp, PCI_D0);
10337         if (err) {
10338                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
10339                        pci_name(tp->pdev));
10340                 return err;
10341         }
10342
10343         /* 5700 B0 chips do not support checksumming correctly due
10344          * to hardware bugs.
10345          */
10346         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
10347                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
10348
10349         /* Derive initial jumbo mode from MTU assigned in
10350          * ether_setup() via the alloc_etherdev() call
10351          */
10352         if (tp->dev->mtu > ETH_DATA_LEN &&
10353             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
10354                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
10355
10356         /* Determine WakeOnLan speed to use. */
10357         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10358             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
10359             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
10360             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
10361                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
10362         } else {
10363                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
10364         }
10365
10366         /* A few boards don't want Ethernet@WireSpeed phy feature */
10367         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
10368             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
10369              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
10370              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
10371             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
10372                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
10373
10374         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
10375             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
10376                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
10377         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
10378                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
10379
10380         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
10381                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10382                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
10383                         tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
10384                 else
10385                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
10386         }
10387
10388         tp->coalesce_mode = 0;
10389         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
10390             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
10391                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
10392
10393         /* Initialize MAC MI mode, polling disabled. */
10394         tw32_f(MAC_MI_MODE, tp->mi_mode);
10395         udelay(80);
10396
10397         /* Initialize data/descriptor byte/word swapping. */
10398         val = tr32(GRC_MODE);
10399         val &= GRC_MODE_HOST_STACKUP;
10400         tw32(GRC_MODE, val | tp->grc_mode);
10401
10402         tg3_switch_clocks(tp);
10403
10404         /* Clear this out for sanity. */
10405         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
10406
10407         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
10408                               &pci_state_reg);
10409         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
10410             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
10411                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
10412
10413                 if (chiprevid == CHIPREV_ID_5701_A0 ||
10414                     chiprevid == CHIPREV_ID_5701_B0 ||
10415                     chiprevid == CHIPREV_ID_5701_B2 ||
10416                     chiprevid == CHIPREV_ID_5701_B5) {
10417                         void __iomem *sram_base;
10418
10419                         /* Write some dummy words into the SRAM status block
10420                          * area, see if it reads back correctly.  If the return
10421                          * value is bad, force enable the PCIX workaround.
10422                          */
10423                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
10424
10425                         writel(0x00000000, sram_base);
10426                         writel(0x00000000, sram_base + 4);
10427                         writel(0xffffffff, sram_base + 4);
10428                         if (readl(sram_base) != 0x00000000)
10429                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
10430                 }
10431         }
10432
10433         udelay(50);
10434         tg3_nvram_init(tp);
10435
10436         grc_misc_cfg = tr32(GRC_MISC_CFG);
10437         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
10438
10439         /* Broadcom's driver says that CIOBE multisplit has a bug */
10440 #if 0
10441         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
10442             grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5704CIOBE) {
10443                 tp->tg3_flags |= TG3_FLAG_SPLIT_MODE;
10444                 tp->split_mode_max_reqs = SPLIT_MODE_5704_MAX_REQ;
10445         }
10446 #endif
10447         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
10448             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
10449              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
10450                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
10451
10452         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
10453             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
10454                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
10455         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
10456                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
10457                                       HOSTCC_MODE_CLRTICK_TXBD);
10458
10459                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
10460                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10461                                        tp->misc_host_ctrl);
10462         }
10463
10464         /* these are limited to 10/100 only */
10465         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
10466              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
10467             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
10468              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
10469              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
10470               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
10471               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
10472             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
10473              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
10474               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F)))
10475                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
10476
10477         err = tg3_phy_probe(tp);
10478         if (err) {
10479                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
10480                        pci_name(tp->pdev), err);
10481                 /* ... but do not return immediately ... */
10482         }
10483
10484         tg3_read_partno(tp);
10485         tg3_read_fw_ver(tp);
10486
10487         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
10488                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
10489         } else {
10490                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
10491                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
10492                 else
10493                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
10494         }
10495
10496         /* 5700 {AX,BX} chips have a broken status block link
10497          * change bit implementation, so we must use the
10498          * status register in those cases.
10499          */
10500         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
10501                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
10502         else
10503                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
10504
10505         /* The led_ctrl is set during tg3_phy_probe, here we might
10506          * have to force the link status polling mechanism based
10507          * upon subsystem IDs.
10508          */
10509         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
10510             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
10511                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
10512                                   TG3_FLAG_USE_LINKCHG_REG);
10513         }
10514
10515         /* For all SERDES we poll the MAC status register. */
10516         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
10517                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
10518         else
10519                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
10520
10521         /* All chips before 5787 can get confused if TX buffers
10522          * straddle the 4GB address boundary in some cases.
10523          */
10524         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10525             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
10526                 tp->dev->hard_start_xmit = tg3_start_xmit;
10527         else
10528                 tp->dev->hard_start_xmit = tg3_start_xmit_dma_bug;
10529
10530         tp->rx_offset = 2;
10531         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
10532             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
10533                 tp->rx_offset = 0;
10534
10535         /* By default, disable wake-on-lan.  User can change this
10536          * using ETHTOOL_SWOL.
10537          */
10538         tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
10539
10540         return err;
10541 }
10542
10543 #ifdef CONFIG_SPARC64
10544 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
10545 {
10546         struct net_device *dev = tp->dev;
10547         struct pci_dev *pdev = tp->pdev;
10548         struct pcidev_cookie *pcp = pdev->sysdata;
10549
10550         if (pcp != NULL) {
10551                 unsigned char *addr;
10552                 int len;
10553
10554                 addr = of_get_property(pcp->prom_node, "local-mac-address",
10555                                         &len);
10556                 if (addr && len == 6) {
10557                         memcpy(dev->dev_addr, addr, 6);
10558                         memcpy(dev->perm_addr, dev->dev_addr, 6);
10559                         return 0;
10560                 }
10561         }
10562         return -ENODEV;
10563 }
10564
10565 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
10566 {
10567         struct net_device *dev = tp->dev;
10568
10569         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
10570         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
10571         return 0;
10572 }
10573 #endif
10574
10575 static int __devinit tg3_get_device_address(struct tg3 *tp)
10576 {
10577         struct net_device *dev = tp->dev;
10578         u32 hi, lo, mac_offset;
10579         int addr_ok = 0;
10580
10581 #ifdef CONFIG_SPARC64
10582         if (!tg3_get_macaddr_sparc(tp))
10583                 return 0;
10584 #endif
10585
10586         mac_offset = 0x7c;
10587         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
10588             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
10589                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
10590                         mac_offset = 0xcc;
10591                 if (tg3_nvram_lock(tp))
10592                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
10593                 else
10594                         tg3_nvram_unlock(tp);
10595         }
10596
10597         /* First try to get it from MAC address mailbox. */
10598         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
10599         if ((hi >> 16) == 0x484b) {
10600                 dev->dev_addr[0] = (hi >>  8) & 0xff;
10601                 dev->dev_addr[1] = (hi >>  0) & 0xff;
10602
10603                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
10604                 dev->dev_addr[2] = (lo >> 24) & 0xff;
10605                 dev->dev_addr[3] = (lo >> 16) & 0xff;
10606                 dev->dev_addr[4] = (lo >>  8) & 0xff;
10607                 dev->dev_addr[5] = (lo >>  0) & 0xff;
10608
10609                 /* Some old bootcode may report a 0 MAC address in SRAM */
10610                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
10611         }
10612         if (!addr_ok) {
10613                 /* Next, try NVRAM. */
10614                 if (!tg3_nvram_read(tp, mac_offset + 0, &hi) &&
10615                     !tg3_nvram_read(tp, mac_offset + 4, &lo)) {
10616                         dev->dev_addr[0] = ((hi >> 16) & 0xff);
10617                         dev->dev_addr[1] = ((hi >> 24) & 0xff);
10618                         dev->dev_addr[2] = ((lo >>  0) & 0xff);
10619                         dev->dev_addr[3] = ((lo >>  8) & 0xff);
10620                         dev->dev_addr[4] = ((lo >> 16) & 0xff);
10621                         dev->dev_addr[5] = ((lo >> 24) & 0xff);
10622                 }
10623                 /* Finally just fetch it out of the MAC control regs. */
10624                 else {
10625                         hi = tr32(MAC_ADDR_0_HIGH);
10626                         lo = tr32(MAC_ADDR_0_LOW);
10627
10628                         dev->dev_addr[5] = lo & 0xff;
10629                         dev->dev_addr[4] = (lo >> 8) & 0xff;
10630                         dev->dev_addr[3] = (lo >> 16) & 0xff;
10631                         dev->dev_addr[2] = (lo >> 24) & 0xff;
10632                         dev->dev_addr[1] = hi & 0xff;
10633                         dev->dev_addr[0] = (hi >> 8) & 0xff;
10634                 }
10635         }
10636
10637         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
10638 #ifdef CONFIG_SPARC64
10639                 if (!tg3_get_default_macaddr_sparc(tp))
10640                         return 0;
10641 #endif
10642                 return -EINVAL;
10643         }
10644         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10645         return 0;
10646 }
10647
10648 #define BOUNDARY_SINGLE_CACHELINE       1
10649 #define BOUNDARY_MULTI_CACHELINE        2
10650
10651 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
10652 {
10653         int cacheline_size;
10654         u8 byte;
10655         int goal;
10656
10657         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
10658         if (byte == 0)
10659                 cacheline_size = 1024;
10660         else
10661                 cacheline_size = (int) byte * 4;
10662
10663         /* On 5703 and later chips, the boundary bits have no
10664          * effect.
10665          */
10666         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
10667             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
10668             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
10669                 goto out;
10670
10671 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
10672         goal = BOUNDARY_MULTI_CACHELINE;
10673 #else
10674 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
10675         goal = BOUNDARY_SINGLE_CACHELINE;
10676 #else
10677         goal = 0;
10678 #endif
10679 #endif
10680
10681         if (!goal)
10682                 goto out;
10683
10684         /* PCI controllers on most RISC systems tend to disconnect
10685          * when a device tries to burst across a cache-line boundary.
10686          * Therefore, letting tg3 do so just wastes PCI bandwidth.
10687          *
10688          * Unfortunately, for PCI-E there are only limited
10689          * write-side controls for this, and thus for reads
10690          * we will still get the disconnects.  We'll also waste
10691          * these PCI cycles for both read and write for chips
10692          * other than 5700 and 5701 which do not implement the
10693          * boundary bits.
10694          */
10695         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
10696             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
10697                 switch (cacheline_size) {
10698                 case 16:
10699                 case 32:
10700                 case 64:
10701                 case 128:
10702                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10703                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
10704                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
10705                         } else {
10706                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
10707                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
10708                         }
10709                         break;
10710
10711                 case 256:
10712                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
10713                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
10714                         break;
10715
10716                 default:
10717                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
10718                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
10719                         break;
10720                 };
10721         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10722                 switch (cacheline_size) {
10723                 case 16:
10724                 case 32:
10725                 case 64:
10726                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10727                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
10728                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
10729                                 break;
10730                         }
10731                         /* fallthrough */
10732                 case 128:
10733                 default:
10734                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
10735                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
10736                         break;
10737                 };
10738         } else {
10739                 switch (cacheline_size) {
10740                 case 16:
10741                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10742                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
10743                                         DMA_RWCTRL_WRITE_BNDRY_16);
10744                                 break;
10745                         }
10746                         /* fallthrough */
10747                 case 32:
10748                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10749                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
10750                                         DMA_RWCTRL_WRITE_BNDRY_32);
10751                                 break;
10752                         }
10753                         /* fallthrough */
10754                 case 64:
10755                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10756                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
10757                                         DMA_RWCTRL_WRITE_BNDRY_64);
10758                                 break;
10759                         }
10760                         /* fallthrough */
10761                 case 128:
10762                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10763                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
10764                                         DMA_RWCTRL_WRITE_BNDRY_128);
10765                                 break;
10766                         }
10767                         /* fallthrough */
10768                 case 256:
10769                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
10770                                 DMA_RWCTRL_WRITE_BNDRY_256);
10771                         break;
10772                 case 512:
10773                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
10774                                 DMA_RWCTRL_WRITE_BNDRY_512);
10775                         break;
10776                 case 1024:
10777                 default:
10778                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
10779                                 DMA_RWCTRL_WRITE_BNDRY_1024);
10780                         break;
10781                 };
10782         }
10783
10784 out:
10785         return val;
10786 }
10787
10788 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
10789 {
10790         struct tg3_internal_buffer_desc test_desc;
10791         u32 sram_dma_descs;
10792         int i, ret;
10793
10794         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
10795
10796         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
10797         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
10798         tw32(RDMAC_STATUS, 0);
10799         tw32(WDMAC_STATUS, 0);
10800
10801         tw32(BUFMGR_MODE, 0);
10802         tw32(FTQ_RESET, 0);
10803
10804         test_desc.addr_hi = ((u64) buf_dma) >> 32;
10805         test_desc.addr_lo = buf_dma & 0xffffffff;
10806         test_desc.nic_mbuf = 0x00002100;
10807         test_desc.len = size;
10808
10809         /*
10810          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
10811          * the *second* time the tg3 driver was getting loaded after an
10812          * initial scan.
10813          *
10814          * Broadcom tells me:
10815          *   ...the DMA engine is connected to the GRC block and a DMA
10816          *   reset may affect the GRC block in some unpredictable way...
10817          *   The behavior of resets to individual blocks has not been tested.
10818          *
10819          * Broadcom noted the GRC reset will also reset all sub-components.
10820          */
10821         if (to_device) {
10822                 test_desc.cqid_sqid = (13 << 8) | 2;
10823
10824                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
10825                 udelay(40);
10826         } else {
10827                 test_desc.cqid_sqid = (16 << 8) | 7;
10828
10829                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
10830                 udelay(40);
10831         }
10832         test_desc.flags = 0x00000005;
10833
10834         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
10835                 u32 val;
10836
10837                 val = *(((u32 *)&test_desc) + i);
10838                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
10839                                        sram_dma_descs + (i * sizeof(u32)));
10840                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
10841         }
10842         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
10843
10844         if (to_device) {
10845                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
10846         } else {
10847                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
10848         }
10849
10850         ret = -ENODEV;
10851         for (i = 0; i < 40; i++) {
10852                 u32 val;
10853
10854                 if (to_device)
10855                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
10856                 else
10857                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
10858                 if ((val & 0xffff) == sram_dma_descs) {
10859                         ret = 0;
10860                         break;
10861                 }
10862
10863                 udelay(100);
10864         }
10865
10866         return ret;
10867 }
10868
10869 #define TEST_BUFFER_SIZE        0x2000
10870
10871 static int __devinit tg3_test_dma(struct tg3 *tp)
10872 {
10873         dma_addr_t buf_dma;
10874         u32 *buf, saved_dma_rwctrl;
10875         int ret;
10876
10877         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
10878         if (!buf) {
10879                 ret = -ENOMEM;
10880                 goto out_nofree;
10881         }
10882
10883         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
10884                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
10885
10886         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
10887
10888         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10889                 /* DMA read watermark not used on PCIE */
10890                 tp->dma_rwctrl |= 0x00180000;
10891         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
10892                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
10893                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
10894                         tp->dma_rwctrl |= 0x003f0000;
10895                 else
10896                         tp->dma_rwctrl |= 0x003f000f;
10897         } else {
10898                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
10899                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
10900                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
10901
10902                         /* If the 5704 is behind the EPB bridge, we can
10903                          * do the less restrictive ONE_DMA workaround for
10904                          * better performance.
10905                          */
10906                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
10907                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
10908                                 tp->dma_rwctrl |= 0x8000;
10909                         else if (ccval == 0x6 || ccval == 0x7)
10910                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
10911
10912                         /* Set bit 23 to enable PCIX hw bug fix */
10913                         tp->dma_rwctrl |= 0x009f0000;
10914                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
10915                         /* 5780 always in PCIX mode */
10916                         tp->dma_rwctrl |= 0x00144000;
10917                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
10918                         /* 5714 always in PCIX mode */
10919                         tp->dma_rwctrl |= 0x00148000;
10920                 } else {
10921                         tp->dma_rwctrl |= 0x001b000f;
10922                 }
10923         }
10924
10925         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
10926             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
10927                 tp->dma_rwctrl &= 0xfffffff0;
10928
10929         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10930             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
10931                 /* Remove this if it causes problems for some boards. */
10932                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
10933
10934                 /* On 5700/5701 chips, we need to set this bit.
10935                  * Otherwise the chip will issue cacheline transactions
10936                  * to streamable DMA memory with not all the byte
10937                  * enables turned on.  This is an error on several
10938                  * RISC PCI controllers, in particular sparc64.
10939                  *
10940                  * On 5703/5704 chips, this bit has been reassigned
10941                  * a different meaning.  In particular, it is used
10942                  * on those chips to enable a PCI-X workaround.
10943                  */
10944                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
10945         }
10946
10947         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
10948
10949 #if 0
10950         /* Unneeded, already done by tg3_get_invariants.  */
10951         tg3_switch_clocks(tp);
10952 #endif
10953
10954         ret = 0;
10955         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
10956             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
10957                 goto out;
10958
10959         /* It is best to perform DMA test with maximum write burst size
10960          * to expose the 5700/5701 write DMA bug.
10961          */
10962         saved_dma_rwctrl = tp->dma_rwctrl;
10963         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
10964         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
10965
10966         while (1) {
10967                 u32 *p = buf, i;
10968
10969                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
10970                         p[i] = i;
10971
10972                 /* Send the buffer to the chip. */
10973                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
10974                 if (ret) {
10975                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
10976                         break;
10977                 }
10978
10979 #if 0
10980                 /* validate data reached card RAM correctly. */
10981                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
10982                         u32 val;
10983                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
10984                         if (le32_to_cpu(val) != p[i]) {
10985                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
10986                                 /* ret = -ENODEV here? */
10987                         }
10988                         p[i] = 0;
10989                 }
10990 #endif
10991                 /* Now read it back. */
10992                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
10993                 if (ret) {
10994                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
10995
10996                         break;
10997                 }
10998
10999                 /* Verify it. */
11000                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
11001                         if (p[i] == i)
11002                                 continue;
11003
11004                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
11005                             DMA_RWCTRL_WRITE_BNDRY_16) {
11006                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
11007                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
11008                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11009                                 break;
11010                         } else {
11011                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
11012                                 ret = -ENODEV;
11013                                 goto out;
11014                         }
11015                 }
11016
11017                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
11018                         /* Success. */
11019                         ret = 0;
11020                         break;
11021                 }
11022         }
11023         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
11024             DMA_RWCTRL_WRITE_BNDRY_16) {
11025                 static struct pci_device_id dma_wait_state_chipsets[] = {
11026                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
11027                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
11028                         { },
11029                 };
11030
11031                 /* DMA test passed without adjusting DMA boundary,
11032                  * now look for chipsets that are known to expose the
11033                  * DMA bug without failing the test.
11034                  */
11035                 if (pci_dev_present(dma_wait_state_chipsets)) {
11036                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
11037                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
11038                 }
11039                 else
11040                         /* Safe to use the calculated DMA boundary. */
11041                         tp->dma_rwctrl = saved_dma_rwctrl;
11042
11043                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11044         }
11045
11046 out:
11047         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
11048 out_nofree:
11049         return ret;
11050 }
11051
11052 static void __devinit tg3_init_link_config(struct tg3 *tp)
11053 {
11054         tp->link_config.advertising =
11055                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
11056                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
11057                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
11058                  ADVERTISED_Autoneg | ADVERTISED_MII);
11059         tp->link_config.speed = SPEED_INVALID;
11060         tp->link_config.duplex = DUPLEX_INVALID;
11061         tp->link_config.autoneg = AUTONEG_ENABLE;
11062         tp->link_config.active_speed = SPEED_INVALID;
11063         tp->link_config.active_duplex = DUPLEX_INVALID;
11064         tp->link_config.phy_is_low_power = 0;
11065         tp->link_config.orig_speed = SPEED_INVALID;
11066         tp->link_config.orig_duplex = DUPLEX_INVALID;
11067         tp->link_config.orig_autoneg = AUTONEG_INVALID;
11068 }
11069
11070 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
11071 {
11072         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11073                 tp->bufmgr_config.mbuf_read_dma_low_water =
11074                         DEFAULT_MB_RDMA_LOW_WATER_5705;
11075                 tp->bufmgr_config.mbuf_mac_rx_low_water =
11076                         DEFAULT_MB_MACRX_LOW_WATER_5705;
11077                 tp->bufmgr_config.mbuf_high_water =
11078                         DEFAULT_MB_HIGH_WATER_5705;
11079
11080                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
11081                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
11082                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
11083                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
11084                 tp->bufmgr_config.mbuf_high_water_jumbo =
11085                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
11086         } else {
11087                 tp->bufmgr_config.mbuf_read_dma_low_water =
11088                         DEFAULT_MB_RDMA_LOW_WATER;
11089                 tp->bufmgr_config.mbuf_mac_rx_low_water =
11090                         DEFAULT_MB_MACRX_LOW_WATER;
11091                 tp->bufmgr_config.mbuf_high_water =
11092                         DEFAULT_MB_HIGH_WATER;
11093
11094                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
11095                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
11096                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
11097                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
11098                 tp->bufmgr_config.mbuf_high_water_jumbo =
11099                         DEFAULT_MB_HIGH_WATER_JUMBO;
11100         }
11101
11102         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
11103         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
11104 }
11105
11106 static char * __devinit tg3_phy_string(struct tg3 *tp)
11107 {
11108         switch (tp->phy_id & PHY_ID_MASK) {
11109         case PHY_ID_BCM5400:    return "5400";
11110         case PHY_ID_BCM5401:    return "5401";
11111         case PHY_ID_BCM5411:    return "5411";
11112         case PHY_ID_BCM5701:    return "5701";
11113         case PHY_ID_BCM5703:    return "5703";
11114         case PHY_ID_BCM5704:    return "5704";
11115         case PHY_ID_BCM5705:    return "5705";
11116         case PHY_ID_BCM5750:    return "5750";
11117         case PHY_ID_BCM5752:    return "5752";
11118         case PHY_ID_BCM5714:    return "5714";
11119         case PHY_ID_BCM5780:    return "5780";
11120         case PHY_ID_BCM5755:    return "5755";
11121         case PHY_ID_BCM5787:    return "5787";
11122         case PHY_ID_BCM8002:    return "8002/serdes";
11123         case 0:                 return "serdes";
11124         default:                return "unknown";
11125         };
11126 }
11127
11128 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
11129 {
11130         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11131                 strcpy(str, "PCI Express");
11132                 return str;
11133         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
11134                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
11135
11136                 strcpy(str, "PCIX:");
11137
11138                 if ((clock_ctrl == 7) ||
11139                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
11140                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
11141                         strcat(str, "133MHz");
11142                 else if (clock_ctrl == 0)
11143                         strcat(str, "33MHz");
11144                 else if (clock_ctrl == 2)
11145                         strcat(str, "50MHz");
11146                 else if (clock_ctrl == 4)
11147                         strcat(str, "66MHz");
11148                 else if (clock_ctrl == 6)
11149                         strcat(str, "100MHz");
11150         } else {
11151                 strcpy(str, "PCI:");
11152                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
11153                         strcat(str, "66MHz");
11154                 else
11155                         strcat(str, "33MHz");
11156         }
11157         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
11158                 strcat(str, ":32-bit");
11159         else
11160                 strcat(str, ":64-bit");
11161         return str;
11162 }
11163
11164 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
11165 {
11166         struct pci_dev *peer;
11167         unsigned int func, devnr = tp->pdev->devfn & ~7;
11168
11169         for (func = 0; func < 8; func++) {
11170                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
11171                 if (peer && peer != tp->pdev)
11172                         break;
11173                 pci_dev_put(peer);
11174         }
11175         /* 5704 can be configured in single-port mode, set peer to
11176          * tp->pdev in that case.
11177          */
11178         if (!peer) {
11179                 peer = tp->pdev;
11180                 return peer;
11181         }
11182
11183         /*
11184          * We don't need to keep the refcount elevated; there's no way
11185          * to remove one half of this device without removing the other
11186          */
11187         pci_dev_put(peer);
11188
11189         return peer;
11190 }
11191
11192 static void __devinit tg3_init_coal(struct tg3 *tp)
11193 {
11194         struct ethtool_coalesce *ec = &tp->coal;
11195
11196         memset(ec, 0, sizeof(*ec));
11197         ec->cmd = ETHTOOL_GCOALESCE;
11198         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
11199         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
11200         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
11201         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
11202         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
11203         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
11204         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
11205         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
11206         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
11207
11208         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
11209                                  HOSTCC_MODE_CLRTICK_TXBD)) {
11210                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
11211                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
11212                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
11213                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
11214         }
11215
11216         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11217                 ec->rx_coalesce_usecs_irq = 0;
11218                 ec->tx_coalesce_usecs_irq = 0;
11219                 ec->stats_block_coalesce_usecs = 0;
11220         }
11221 }
11222
11223 static int __devinit tg3_init_one(struct pci_dev *pdev,
11224                                   const struct pci_device_id *ent)
11225 {
11226         static int tg3_version_printed = 0;
11227         unsigned long tg3reg_base, tg3reg_len;
11228         struct net_device *dev;
11229         struct tg3 *tp;
11230         int i, err, pm_cap;
11231         char str[40];
11232         u64 dma_mask, persist_dma_mask;
11233
11234         if (tg3_version_printed++ == 0)
11235                 printk(KERN_INFO "%s", version);
11236
11237         err = pci_enable_device(pdev);
11238         if (err) {
11239                 printk(KERN_ERR PFX "Cannot enable PCI device, "
11240                        "aborting.\n");
11241                 return err;
11242         }
11243
11244         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11245                 printk(KERN_ERR PFX "Cannot find proper PCI device "
11246                        "base address, aborting.\n");
11247                 err = -ENODEV;
11248                 goto err_out_disable_pdev;
11249         }
11250
11251         err = pci_request_regions(pdev, DRV_MODULE_NAME);
11252         if (err) {
11253                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
11254                        "aborting.\n");
11255                 goto err_out_disable_pdev;
11256         }
11257
11258         pci_set_master(pdev);
11259
11260         /* Find power-management capability. */
11261         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
11262         if (pm_cap == 0) {
11263                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
11264                        "aborting.\n");
11265                 err = -EIO;
11266                 goto err_out_free_res;
11267         }
11268
11269         tg3reg_base = pci_resource_start(pdev, 0);
11270         tg3reg_len = pci_resource_len(pdev, 0);
11271
11272         dev = alloc_etherdev(sizeof(*tp));
11273         if (!dev) {
11274                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
11275                 err = -ENOMEM;
11276                 goto err_out_free_res;
11277         }
11278
11279         SET_MODULE_OWNER(dev);
11280         SET_NETDEV_DEV(dev, &pdev->dev);
11281
11282 #if TG3_VLAN_TAG_USED
11283         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
11284         dev->vlan_rx_register = tg3_vlan_rx_register;
11285         dev->vlan_rx_kill_vid = tg3_vlan_rx_kill_vid;
11286 #endif
11287
11288         tp = netdev_priv(dev);
11289         tp->pdev = pdev;
11290         tp->dev = dev;
11291         tp->pm_cap = pm_cap;
11292         tp->mac_mode = TG3_DEF_MAC_MODE;
11293         tp->rx_mode = TG3_DEF_RX_MODE;
11294         tp->tx_mode = TG3_DEF_TX_MODE;
11295         tp->mi_mode = MAC_MI_MODE_BASE;
11296         if (tg3_debug > 0)
11297                 tp->msg_enable = tg3_debug;
11298         else
11299                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
11300
11301         /* The word/byte swap controls here control register access byte
11302          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
11303          * setting below.
11304          */
11305         tp->misc_host_ctrl =
11306                 MISC_HOST_CTRL_MASK_PCI_INT |
11307                 MISC_HOST_CTRL_WORD_SWAP |
11308                 MISC_HOST_CTRL_INDIR_ACCESS |
11309                 MISC_HOST_CTRL_PCISTATE_RW;
11310
11311         /* The NONFRM (non-frame) byte/word swap controls take effect
11312          * on descriptor entries, anything which isn't packet data.
11313          *
11314          * The StrongARM chips on the board (one for tx, one for rx)
11315          * are running in big-endian mode.
11316          */
11317         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
11318                         GRC_MODE_WSWAP_NONFRM_DATA);
11319 #ifdef __BIG_ENDIAN
11320         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
11321 #endif
11322         spin_lock_init(&tp->lock);
11323         spin_lock_init(&tp->tx_lock);
11324         spin_lock_init(&tp->indirect_lock);
11325         INIT_WORK(&tp->reset_task, tg3_reset_task, tp);
11326
11327         tp->regs = ioremap_nocache(tg3reg_base, tg3reg_len);
11328         if (tp->regs == 0UL) {
11329                 printk(KERN_ERR PFX "Cannot map device registers, "
11330                        "aborting.\n");
11331                 err = -ENOMEM;
11332                 goto err_out_free_dev;
11333         }
11334
11335         tg3_init_link_config(tp);
11336
11337         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
11338         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
11339         tp->tx_pending = TG3_DEF_TX_RING_PENDING;
11340
11341         dev->open = tg3_open;
11342         dev->stop = tg3_close;
11343         dev->get_stats = tg3_get_stats;
11344         dev->set_multicast_list = tg3_set_rx_mode;
11345         dev->set_mac_address = tg3_set_mac_addr;
11346         dev->do_ioctl = tg3_ioctl;
11347         dev->tx_timeout = tg3_tx_timeout;
11348         dev->poll = tg3_poll;
11349         dev->ethtool_ops = &tg3_ethtool_ops;
11350         dev->weight = 64;
11351         dev->watchdog_timeo = TG3_TX_TIMEOUT;
11352         dev->change_mtu = tg3_change_mtu;
11353         dev->irq = pdev->irq;
11354 #ifdef CONFIG_NET_POLL_CONTROLLER
11355         dev->poll_controller = tg3_poll_controller;
11356 #endif
11357
11358         err = tg3_get_invariants(tp);
11359         if (err) {
11360                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
11361                        "aborting.\n");
11362                 goto err_out_iounmap;
11363         }
11364
11365         /* The EPB bridge inside 5714, 5715, and 5780 and any
11366          * device behind the EPB cannot support DMA addresses > 40-bit.
11367          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
11368          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
11369          * do DMA address check in tg3_start_xmit().
11370          */
11371         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
11372                 persist_dma_mask = dma_mask = DMA_32BIT_MASK;
11373         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
11374                 persist_dma_mask = dma_mask = DMA_40BIT_MASK;
11375 #ifdef CONFIG_HIGHMEM
11376                 dma_mask = DMA_64BIT_MASK;
11377 #endif
11378         } else
11379                 persist_dma_mask = dma_mask = DMA_64BIT_MASK;
11380
11381         /* Configure DMA attributes. */
11382         if (dma_mask > DMA_32BIT_MASK) {
11383                 err = pci_set_dma_mask(pdev, dma_mask);
11384                 if (!err) {
11385                         dev->features |= NETIF_F_HIGHDMA;
11386                         err = pci_set_consistent_dma_mask(pdev,
11387                                                           persist_dma_mask);
11388                         if (err < 0) {
11389                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
11390                                        "DMA for consistent allocations\n");
11391                                 goto err_out_iounmap;
11392                         }
11393                 }
11394         }
11395         if (err || dma_mask == DMA_32BIT_MASK) {
11396                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
11397                 if (err) {
11398                         printk(KERN_ERR PFX "No usable DMA configuration, "
11399                                "aborting.\n");
11400                         goto err_out_iounmap;
11401                 }
11402         }
11403
11404         tg3_init_bufmgr_config(tp);
11405
11406 #if TG3_TSO_SUPPORT != 0
11407         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
11408                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
11409         }
11410         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11411             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
11412             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
11413             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
11414                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
11415         } else {
11416                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
11417         }
11418
11419         /* TSO is on by default on chips that support hardware TSO.
11420          * Firmware TSO on older chips gives lower performance, so it
11421          * is off by default, but can be enabled using ethtool.
11422          */
11423         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
11424                 dev->features |= NETIF_F_TSO;
11425
11426 #endif
11427
11428         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
11429             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
11430             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
11431                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
11432                 tp->rx_pending = 63;
11433         }
11434
11435         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11436             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714))
11437                 tp->pdev_peer = tg3_find_peer(tp);
11438
11439         err = tg3_get_device_address(tp);
11440         if (err) {
11441                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
11442                        "aborting.\n");
11443                 goto err_out_iounmap;
11444         }
11445
11446         /*
11447          * Reset chip in case UNDI or EFI driver did not shutdown
11448          * DMA self test will enable WDMAC and we'll see (spurious)
11449          * pending DMA on the PCI bus at that point.
11450          */
11451         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
11452             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
11453                 pci_save_state(tp->pdev);
11454                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
11455                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11456         }
11457
11458         err = tg3_test_dma(tp);
11459         if (err) {
11460                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
11461                 goto err_out_iounmap;
11462         }
11463
11464         /* Tigon3 can do ipv4 only... and some chips have buggy
11465          * checksumming.
11466          */
11467         if ((tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) == 0) {
11468                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11469                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
11470                         dev->features |= NETIF_F_HW_CSUM;
11471                 else
11472                         dev->features |= NETIF_F_IP_CSUM;
11473                 dev->features |= NETIF_F_SG;
11474                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
11475         } else
11476                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
11477
11478         /* flow control autonegotiation is default behavior */
11479         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
11480
11481         tg3_init_coal(tp);
11482
11483         /* Now that we have fully setup the chip, save away a snapshot
11484          * of the PCI config space.  We need to restore this after
11485          * GRC_MISC_CFG core clock resets and some resume events.
11486          */
11487         pci_save_state(tp->pdev);
11488
11489         err = register_netdev(dev);
11490         if (err) {
11491                 printk(KERN_ERR PFX "Cannot register net device, "
11492                        "aborting.\n");
11493                 goto err_out_iounmap;
11494         }
11495
11496         pci_set_drvdata(pdev, dev);
11497
11498         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x PHY(%s)] (%s) %sBaseT Ethernet ",
11499                dev->name,
11500                tp->board_part_number,
11501                tp->pci_chip_rev_id,
11502                tg3_phy_string(tp),
11503                tg3_bus_string(tp, str),
11504                (tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100" : "10/100/1000");
11505
11506         for (i = 0; i < 6; i++)
11507                 printk("%2.2x%c", dev->dev_addr[i],
11508                        i == 5 ? '\n' : ':');
11509
11510         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] "
11511                "MIirq[%d] ASF[%d] Split[%d] WireSpeed[%d] "
11512                "TSOcap[%d] \n",
11513                dev->name,
11514                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
11515                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
11516                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
11517                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
11518                (tp->tg3_flags & TG3_FLAG_SPLIT_MODE) != 0,
11519                (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0,
11520                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
11521         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
11522                dev->name, tp->dma_rwctrl,
11523                (pdev->dma_mask == DMA_32BIT_MASK) ? 32 :
11524                 (((u64) pdev->dma_mask == DMA_40BIT_MASK) ? 40 : 64));
11525
11526         netif_carrier_off(tp->dev);
11527
11528         return 0;
11529
11530 err_out_iounmap:
11531         if (tp->regs) {
11532                 iounmap(tp->regs);
11533                 tp->regs = NULL;
11534         }
11535
11536 err_out_free_dev:
11537         free_netdev(dev);
11538
11539 err_out_free_res:
11540         pci_release_regions(pdev);
11541
11542 err_out_disable_pdev:
11543         pci_disable_device(pdev);
11544         pci_set_drvdata(pdev, NULL);
11545         return err;
11546 }
11547
11548 static void __devexit tg3_remove_one(struct pci_dev *pdev)
11549 {
11550         struct net_device *dev = pci_get_drvdata(pdev);
11551
11552         if (dev) {
11553                 struct tg3 *tp = netdev_priv(dev);
11554
11555                 flush_scheduled_work();
11556                 unregister_netdev(dev);
11557                 if (tp->regs) {
11558                         iounmap(tp->regs);
11559                         tp->regs = NULL;
11560                 }
11561                 free_netdev(dev);
11562                 pci_release_regions(pdev);
11563                 pci_disable_device(pdev);
11564                 pci_set_drvdata(pdev, NULL);
11565         }
11566 }
11567
11568 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
11569 {
11570         struct net_device *dev = pci_get_drvdata(pdev);
11571         struct tg3 *tp = netdev_priv(dev);
11572         int err;
11573
11574         if (!netif_running(dev))
11575                 return 0;
11576
11577         flush_scheduled_work();
11578         tg3_netif_stop(tp);
11579
11580         del_timer_sync(&tp->timer);
11581
11582         tg3_full_lock(tp, 1);
11583         tg3_disable_ints(tp);
11584         tg3_full_unlock(tp);
11585
11586         netif_device_detach(dev);
11587
11588         tg3_full_lock(tp, 0);
11589         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11590         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
11591         tg3_full_unlock(tp);
11592
11593         err = tg3_set_power_state(tp, pci_choose_state(pdev, state));
11594         if (err) {
11595                 tg3_full_lock(tp, 0);
11596
11597                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
11598                 tg3_init_hw(tp, 1);
11599
11600                 tp->timer.expires = jiffies + tp->timer_offset;
11601                 add_timer(&tp->timer);
11602
11603                 netif_device_attach(dev);
11604                 tg3_netif_start(tp);
11605
11606                 tg3_full_unlock(tp);
11607         }
11608
11609         return err;
11610 }
11611
11612 static int tg3_resume(struct pci_dev *pdev)
11613 {
11614         struct net_device *dev = pci_get_drvdata(pdev);
11615         struct tg3 *tp = netdev_priv(dev);
11616         int err;
11617
11618         if (!netif_running(dev))
11619                 return 0;
11620
11621         pci_restore_state(tp->pdev);
11622
11623         err = tg3_set_power_state(tp, PCI_D0);
11624         if (err)
11625                 return err;
11626
11627         netif_device_attach(dev);
11628
11629         tg3_full_lock(tp, 0);
11630
11631         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
11632         tg3_init_hw(tp, 1);
11633
11634         tp->timer.expires = jiffies + tp->timer_offset;
11635         add_timer(&tp->timer);
11636
11637         tg3_netif_start(tp);
11638
11639         tg3_full_unlock(tp);
11640
11641         return 0;
11642 }
11643
11644 static struct pci_driver tg3_driver = {
11645         .name           = DRV_MODULE_NAME,
11646         .id_table       = tg3_pci_tbl,
11647         .probe          = tg3_init_one,
11648         .remove         = __devexit_p(tg3_remove_one),
11649         .suspend        = tg3_suspend,
11650         .resume         = tg3_resume
11651 };
11652
11653 static int __init tg3_init(void)
11654 {
11655         return pci_module_init(&tg3_driver);
11656 }
11657
11658 static void __exit tg3_cleanup(void)
11659 {
11660         pci_unregister_driver(&tg3_driver);
11661 }
11662
11663 module_init(tg3_init);
11664 module_exit(tg3_cleanup);