drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005-2007 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43 #include <net/ip.h>
  44
  45 #include <asm/system.h>
  46 #include <asm/io.h>
  47 #include <asm/byteorder.h>
  48 #include <asm/uaccess.h>
  49
  50 #ifdef CONFIG_SPARC
  51 #include <asm/idprom.h>
  52 #include <asm/prom.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #define TG3_TSO_SUPPORT 1
  62
  63 #include "tg3.h"
  64
  65 #define DRV_MODULE_NAME         "tg3"
  66 #define PFX DRV_MODULE_NAME     ": "
  67 #define DRV_MODULE_VERSION      "3.87"
  68 #define DRV_MODULE_RELDATE      "December 20, 2007"
  69
  70 #define TG3_DEF_MAC_MODE        0
  71 #define TG3_DEF_RX_MODE         0
  72 #define TG3_DEF_TX_MODE         0
  73 #define TG3_DEF_MSG_ENABLE        \
  74         (NETIF_MSG_DRV          | \
  75          NETIF_MSG_PROBE        | \
  76          NETIF_MSG_LINK         | \
  77          NETIF_MSG_TIMER        | \
  78          NETIF_MSG_IFDOWN       | \
  79          NETIF_MSG_IFUP         | \
  80          NETIF_MSG_RX_ERR       | \
  81          NETIF_MSG_TX_ERR)
  82
  83 /* length of time before we decide the hardware is borked,
  84  * and dev->tx_timeout() should be called to fix the problem
  85  */
  86 #define TG3_TX_TIMEOUT                  (5 * HZ)
  87
  88 /* hardware minimum and maximum for a single frame's data payload */
  89 #define TG3_MIN_MTU                     60
  90 #define TG3_MAX_MTU(tp) \
  91         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  92
  93 /* These numbers seem to be hard coded in the NIC firmware somehow.
  94  * You can't change the ring sizes, but you can change where you place
  95  * them in the NIC onboard memory.
  96  */
  97 #define TG3_RX_RING_SIZE                512
  98 #define TG3_DEF_RX_RING_PENDING         200
  99 #define TG3_RX_JUMBO_RING_SIZE          256
 100 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 101
 102 /* Do not place this n-ring entries value into the tp struct itself,
 103  * we really want to expose these constants to GCC so that modulo et
 104  * al.  operations are done with shifts and masks instead of with
 105  * hw multiply/modulo instructions.  Another solution would be to
 106  * replace things like '% foo' with '& (foo - 1)'.
 107  */
 108 #define TG3_RX_RCB_RING_SIZE(tp)        \
 109         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 110
 111 #define TG3_TX_RING_SIZE                512
 112 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 113
 114 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 115                                  TG3_RX_RING_SIZE)
 116 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 117                                  TG3_RX_JUMBO_RING_SIZE)
 118 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 119                                    TG3_RX_RCB_RING_SIZE(tp))
 120 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 121                                  TG3_TX_RING_SIZE)
 122 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 123
 124 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 125 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 126
 127 /* minimum number of free TX descriptors required to wake up TX process */
 128 #define TG3_TX_WAKEUP_THRESH(tp)                ((tp)->tx_pending / 4)
 129
 130 /* number of ETHTOOL_GSTATS u64's */
 131 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 132
 133 #define TG3_NUM_TEST            6
 134
 135 static char version[] __devinitdata =
 136         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 137
 138 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 139 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 140 MODULE_LICENSE("GPL");
 141 MODULE_VERSION(DRV_MODULE_VERSION);
 142
 143 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 144 module_param(tg3_debug, int, 0);
 145 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 146
 147 static struct pci_device_id tg3_pci_tbl[] = {
 148         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 149         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 150         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 151         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 152         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 153         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 154         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 155         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 156         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 157         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 158         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 159         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 160         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 161         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 162         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 163         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 164         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 165         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 166         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 167         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 168         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 169         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 170         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
 171         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
 174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
 176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
 202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
 203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
 204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
 205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
 206         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 207         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 208         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 209         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 210         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 211         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 212         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 213         {}
 214 };
 215
 216 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 217
 218 static const struct {
 219         const char string[ETH_GSTRING_LEN];
 220 } ethtool_stats_keys[TG3_NUM_STATS] = {
 221         { "rx_octets" },
 222         { "rx_fragments" },
 223         { "rx_ucast_packets" },
 224         { "rx_mcast_packets" },
 225         { "rx_bcast_packets" },
 226         { "rx_fcs_errors" },
 227         { "rx_align_errors" },
 228         { "rx_xon_pause_rcvd" },
 229         { "rx_xoff_pause_rcvd" },
 230         { "rx_mac_ctrl_rcvd" },
 231         { "rx_xoff_entered" },
 232         { "rx_frame_too_long_errors" },
 233         { "rx_jabbers" },
 234         { "rx_undersize_packets" },
 235         { "rx_in_length_errors" },
 236         { "rx_out_length_errors" },
 237         { "rx_64_or_less_octet_packets" },
 238         { "rx_65_to_127_octet_packets" },
 239         { "rx_128_to_255_octet_packets" },
 240         { "rx_256_to_511_octet_packets" },
 241         { "rx_512_to_1023_octet_packets" },
 242         { "rx_1024_to_1522_octet_packets" },
 243         { "rx_1523_to_2047_octet_packets" },
 244         { "rx_2048_to_4095_octet_packets" },
 245         { "rx_4096_to_8191_octet_packets" },
 246         { "rx_8192_to_9022_octet_packets" },
 247
 248         { "tx_octets" },
 249         { "tx_collisions" },
 250
 251         { "tx_xon_sent" },
 252         { "tx_xoff_sent" },
 253         { "tx_flow_control" },
 254         { "tx_mac_errors" },
 255         { "tx_single_collisions" },
 256         { "tx_mult_collisions" },
 257         { "tx_deferred" },
 258         { "tx_excessive_collisions" },
 259         { "tx_late_collisions" },
 260         { "tx_collide_2times" },
 261         { "tx_collide_3times" },
 262         { "tx_collide_4times" },
 263         { "tx_collide_5times" },
 264         { "tx_collide_6times" },
 265         { "tx_collide_7times" },
 266         { "tx_collide_8times" },
 267         { "tx_collide_9times" },
 268         { "tx_collide_10times" },
 269         { "tx_collide_11times" },
 270         { "tx_collide_12times" },
 271         { "tx_collide_13times" },
 272         { "tx_collide_14times" },
 273         { "tx_collide_15times" },
 274         { "tx_ucast_packets" },
 275         { "tx_mcast_packets" },
 276         { "tx_bcast_packets" },
 277         { "tx_carrier_sense_errors" },
 278         { "tx_discards" },
 279         { "tx_errors" },
 280
 281         { "dma_writeq_full" },
 282         { "dma_write_prioq_full" },
 283         { "rxbds_empty" },
 284         { "rx_discards" },
 285         { "rx_errors" },
 286         { "rx_threshold_hit" },
 287
 288         { "dma_readq_full" },
 289         { "dma_read_prioq_full" },
 290         { "tx_comp_queue_full" },
 291
 292         { "ring_set_send_prod_index" },
 293         { "ring_status_update" },
 294         { "nic_irqs" },
 295         { "nic_avoided_irqs" },
 296         { "nic_tx_threshold_hit" }
 297 };
 298
 299 static const struct {
 300         const char string[ETH_GSTRING_LEN];
 301 } ethtool_test_keys[TG3_NUM_TEST] = {
 302         { "nvram test     (online) " },
 303         { "link test      (online) " },
 304         { "register test  (offline)" },
 305         { "memory test    (offline)" },
 306         { "loopback test  (offline)" },
 307         { "interrupt test (offline)" },
 308 };
 309
 310 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 311 {
 312         writel(val, tp->regs + off);
 313 }
 314
 315 static u32 tg3_read32(struct tg3 *tp, u32 off)
 316 {
 317         return (readl(tp->regs + off));
 318 }
 319
 320 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
 321 {
 322         writel(val, tp->aperegs + off);
 323 }
 324
 325 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
 326 {
 327         return (readl(tp->aperegs + off));
 328 }
 329
 330 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 331 {
 332         unsigned long flags;
 333
 334         spin_lock_irqsave(&tp->indirect_lock, flags);
 335         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 336         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 337         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 338 }
 339
 340 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 341 {
 342         writel(val, tp->regs + off);
 343         readl(tp->regs + off);
 344 }
 345
 346 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 347 {
 348         unsigned long flags;
 349         u32 val;
 350
 351         spin_lock_irqsave(&tp->indirect_lock, flags);
 352         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 353         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 354         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 355         return val;
 356 }
 357
 358 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 359 {
 360         unsigned long flags;
 361
 362         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 363                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 364                                        TG3_64BIT_REG_LOW, val);
 365                 return;
 366         }
 367         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 368                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 369                                        TG3_64BIT_REG_LOW, val);
 370                 return;
 371         }
 372
 373         spin_lock_irqsave(&tp->indirect_lock, flags);
 374         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 375         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 376         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 377
 378         /* In indirect mode when disabling interrupts, we also need
 379          * to clear the interrupt bit in the GRC local ctrl register.
 380          */
 381         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 382             (val == 0x1)) {
 383                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 384                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 385         }
 386 }
 387
 388 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 389 {
 390         unsigned long flags;
 391         u32 val;
 392
 393         spin_lock_irqsave(&tp->indirect_lock, flags);
 394         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 395         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 396         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 397         return val;
 398 }
 399
 400 /* usec_wait specifies the wait time in usec when writing to certain registers
 401  * where it is unsafe to read back the register without some delay.
 402  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 403  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 404  */
 405 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 406 {
 407         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 408             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 409                 /* Non-posted methods */
 410                 tp->write32(tp, off, val);
 411         else {
 412                 /* Posted method */
 413                 tg3_write32(tp, off, val);
 414                 if (usec_wait)
 415                         udelay(usec_wait);
 416                 tp->read32(tp, off);
 417         }
 418         /* Wait again after the read for the posted method to guarantee that
 419          * the wait time is met.
 420          */
 421         if (usec_wait)
 422                 udelay(usec_wait);
 423 }
 424
 425 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 426 {
 427         tp->write32_mbox(tp, off, val);
 428         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 429             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 430                 tp->read32_mbox(tp, off);
 431 }
 432
 433 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 434 {
 435         void __iomem *mbox = tp->regs + off;
 436         writel(val, mbox);
 437         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 438                 writel(val, mbox);
 439         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 440                 readl(mbox);
 441 }
 442
 443 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 444 {
 445         return (readl(tp->regs + off + GRCMBOX_BASE));
 446 }
 447
 448 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 449 {
 450         writel(val, tp->regs + off + GRCMBOX_BASE);
 451 }
 452
 453 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 454 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 455 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 456 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 457 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 458
 459 #define tw32(reg,val)           tp->write32(tp, reg, val)
 460 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 461 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 462 #define tr32(reg)               tp->read32(tp, reg)
 463
 464 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 465 {
 466         unsigned long flags;
 467
 468         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 469             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 470                 return;
 471
 472         spin_lock_irqsave(&tp->indirect_lock, flags);
 473         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 474                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 475                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 476
 477                 /* Always leave this as zero. */
 478                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 479         } else {
 480                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 481                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 482
 483                 /* Always leave this as zero. */
 484                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 485         }
 486         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 487 }
 488
 489 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 490 {
 491         unsigned long flags;
 492
 493         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 494             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 495                 *val = 0;
 496                 return;
 497         }
 498
 499         spin_lock_irqsave(&tp->indirect_lock, flags);
 500         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 501                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 502                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 503
 504                 /* Always leave this as zero. */
 505                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 506         } else {
 507                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 508                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 509
 510                 /* Always leave this as zero. */
 511                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 512         }
 513         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 514 }
 515
 516 static void tg3_ape_lock_init(struct tg3 *tp)
 517 {
 518         int i;
 519
 520         /* Make sure the driver hasn't any stale locks. */
 521         for (i = 0; i < 8; i++)
 522                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
 523                                 APE_LOCK_GRANT_DRIVER);
 524 }
 525
 526 static int tg3_ape_lock(struct tg3 *tp, int locknum)
 527 {
 528         int i, off;
 529         int ret = 0;
 530         u32 status;
 531
 532         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 533                 return 0;
 534
 535         switch (locknum) {
 536                 case TG3_APE_LOCK_MEM:
 537                         break;
 538                 default:
 539                         return -EINVAL;
 540         }
 541
 542         off = 4 * locknum;
 543
 544         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
 545
 546         /* Wait for up to 1 millisecond to acquire lock. */
 547         for (i = 0; i < 100; i++) {
 548                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
 549                 if (status == APE_LOCK_GRANT_DRIVER)
 550                         break;
 551                 udelay(10);
 552         }
 553
 554         if (status != APE_LOCK_GRANT_DRIVER) {
 555                 /* Revoke the lock request. */
 556                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
 557                                 APE_LOCK_GRANT_DRIVER);
 558
 559                 ret = -EBUSY;
 560         }
 561
 562         return ret;
 563 }
 564
 565 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
 566 {
 567         int off;
 568
 569         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 570                 return;
 571
 572         switch (locknum) {
 573                 case TG3_APE_LOCK_MEM:
 574                         break;
 575                 default:
 576                         return;
 577         }
 578
 579         off = 4 * locknum;
 580         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
 581 }
 582
 583 static void tg3_disable_ints(struct tg3 *tp)
 584 {
 585         tw32(TG3PCI_MISC_HOST_CTRL,
 586              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 587         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 588 }
 589
 590 static inline void tg3_cond_int(struct tg3 *tp)
 591 {
 592         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 593             (tp->hw_status->status & SD_STATUS_UPDATED))
 594                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 595         else
 596                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 597                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 598 }
 599
 600 static void tg3_enable_ints(struct tg3 *tp)
 601 {
 602         tp->irq_sync = 0;
 603         wmb();
 604
 605         tw32(TG3PCI_MISC_HOST_CTRL,
 606              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 607         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 608                        (tp->last_tag << 24));
 609         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 610                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 611                                (tp->last_tag << 24));
 612         tg3_cond_int(tp);
 613 }
 614
 615 static inline unsigned int tg3_has_work(struct tg3 *tp)
 616 {
 617         struct tg3_hw_status *sblk = tp->hw_status;
 618         unsigned int work_exists = 0;
 619
 620         /* check for phy events */
 621         if (!(tp->tg3_flags &
 622               (TG3_FLAG_USE_LINKCHG_REG |
 623                TG3_FLAG_POLL_SERDES))) {
 624                 if (sblk->status & SD_STATUS_LINK_CHG)
 625                         work_exists = 1;
 626         }
 627         /* check for RX/TX work to do */
 628         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 629             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 630                 work_exists = 1;
 631
 632         return work_exists;
 633 }
 634
 635 /* tg3_restart_ints
 636  *  similar to tg3_enable_ints, but it accurately determines whether there
 637  *  is new work pending and can return without flushing the PIO write
 638  *  which reenables interrupts
 639  */
 640 static void tg3_restart_ints(struct tg3 *tp)
 641 {
 642         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 643                      tp->last_tag << 24);
 644         mmiowb();
 645
 646         /* When doing tagged status, this work check is unnecessary.
 647          * The last_tag we write above tells the chip which piece of
 648          * work we've completed.
 649          */
 650         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 651             tg3_has_work(tp))
 652                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 653                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 654 }
 655
 656 static inline void tg3_netif_stop(struct tg3 *tp)
 657 {
 658         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 659         napi_disable(&tp->napi);
 660         netif_tx_disable(tp->dev);
 661 }
 662
 663 static inline void tg3_netif_start(struct tg3 *tp)
 664 {
 665         netif_wake_queue(tp->dev);
 666         /* NOTE: unconditional netif_wake_queue is only appropriate
 667          * so long as all callers are assured to have free tx slots
 668          * (such as after tg3_init_hw)
 669          */
 670         napi_enable(&tp->napi);
 671         tp->hw_status->status |= SD_STATUS_UPDATED;
 672         tg3_enable_ints(tp);
 673 }
 674
 675 static void tg3_switch_clocks(struct tg3 *tp)
 676 {
 677         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 678         u32 orig_clock_ctrl;
 679
 680         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
 681             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
 682                 return;
 683
 684         orig_clock_ctrl = clock_ctrl;
 685         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 686                        CLOCK_CTRL_CLKRUN_OENABLE |
 687                        0x1f);
 688         tp->pci_clock_ctrl = clock_ctrl;
 689
 690         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 691                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 692                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 693                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 694                 }
 695         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 696                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 697                             clock_ctrl |
 698                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 699                             40);
 700                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 701                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 702                             40);
 703         }
 704         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 705 }
 706
 707 #define PHY_BUSY_LOOPS  5000
 708
 709 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 710 {
 711         u32 frame_val;
 712         unsigned int loops;
 713         int ret;
 714
 715         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 716                 tw32_f(MAC_MI_MODE,
 717                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 718                 udelay(80);
 719         }
 720
 721         *val = 0x0;
 722
 723         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 724                       MI_COM_PHY_ADDR_MASK);
 725         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 726                       MI_COM_REG_ADDR_MASK);
 727         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 728
 729         tw32_f(MAC_MI_COM, frame_val);
 730
 731         loops = PHY_BUSY_LOOPS;
 732         while (loops != 0) {
 733                 udelay(10);
 734                 frame_val = tr32(MAC_MI_COM);
 735
 736                 if ((frame_val & MI_COM_BUSY) == 0) {
 737                         udelay(5);
 738                         frame_val = tr32(MAC_MI_COM);
 739                         break;
 740                 }
 741                 loops -= 1;
 742         }
 743
 744         ret = -EBUSY;
 745         if (loops != 0) {
 746                 *val = frame_val & MI_COM_DATA_MASK;
 747                 ret = 0;
 748         }
 749
 750         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 751                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 752                 udelay(80);
 753         }
 754
 755         return ret;
 756 }
 757
 758 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 759 {
 760         u32 frame_val;
 761         unsigned int loops;
 762         int ret;
 763
 764         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
 765             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 766                 return 0;
 767
 768         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 769                 tw32_f(MAC_MI_MODE,
 770                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 771                 udelay(80);
 772         }
 773
 774         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 775                       MI_COM_PHY_ADDR_MASK);
 776         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 777                       MI_COM_REG_ADDR_MASK);
 778         frame_val |= (val & MI_COM_DATA_MASK);
 779         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 780
 781         tw32_f(MAC_MI_COM, frame_val);
 782
 783         loops = PHY_BUSY_LOOPS;
 784         while (loops != 0) {
 785                 udelay(10);
 786                 frame_val = tr32(MAC_MI_COM);
 787                 if ((frame_val & MI_COM_BUSY) == 0) {
 788                         udelay(5);
 789                         frame_val = tr32(MAC_MI_COM);
 790                         break;
 791                 }
 792                 loops -= 1;
 793         }
 794
 795         ret = -EBUSY;
 796         if (loops != 0)
 797                 ret = 0;
 798
 799         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 800                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 801                 udelay(80);
 802         }
 803
 804         return ret;
 805 }
 806
 807 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
 808 {
 809         u32 phy;
 810
 811         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
 812             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
 813                 return;
 814
 815         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
 816                 u32 ephy;
 817
 818                 if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &ephy)) {
 819                         tg3_writephy(tp, MII_TG3_EPHY_TEST,
 820                                      ephy | MII_TG3_EPHY_SHADOW_EN);
 821                         if (!tg3_readphy(tp, MII_TG3_EPHYTST_MISCCTRL, &phy)) {
 822                                 if (enable)
 823                                         phy |= MII_TG3_EPHYTST_MISCCTRL_MDIX;
 824                                 else
 825                                         phy &= ~MII_TG3_EPHYTST_MISCCTRL_MDIX;
 826                                 tg3_writephy(tp, MII_TG3_EPHYTST_MISCCTRL, phy);
 827                         }
 828                         tg3_writephy(tp, MII_TG3_EPHY_TEST, ephy);
 829                 }
 830         } else {
 831                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
 832                       MII_TG3_AUXCTL_SHDWSEL_MISC;
 833                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
 834                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
 835                         if (enable)
 836                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 837                         else
 838                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 839                         phy |= MII_TG3_AUXCTL_MISC_WREN;
 840                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
 841                 }
 842         }
 843 }
 844
 845 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 846 {
 847         u32 val;
 848
 849         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 850                 return;
 851
 852         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 853             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 854                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 855                              (val | (1 << 15) | (1 << 4)));
 856 }
 857
 858 static int tg3_bmcr_reset(struct tg3 *tp)
 859 {
 860         u32 phy_control;
 861         int limit, err;
 862
 863         /* OK, reset it, and poll the BMCR_RESET bit until it
 864          * clears or we time out.
 865          */
 866         phy_control = BMCR_RESET;
 867         err = tg3_writephy(tp, MII_BMCR, phy_control);
 868         if (err != 0)
 869                 return -EBUSY;
 870
 871         limit = 5000;
 872         while (limit--) {
 873                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 874                 if (err != 0)
 875                         return -EBUSY;
 876
 877                 if ((phy_control & BMCR_RESET) == 0) {
 878                         udelay(40);
 879                         break;
 880                 }
 881                 udelay(10);
 882         }
 883         if (limit <= 0)
 884                 return -EBUSY;
 885
 886         return 0;
 887 }
 888
 889 static int tg3_wait_macro_done(struct tg3 *tp)
 890 {
 891         int limit = 100;
 892
 893         while (limit--) {
 894                 u32 tmp32;
 895
 896                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 897                         if ((tmp32 & 0x1000) == 0)
 898                                 break;
 899                 }
 900         }
 901         if (limit <= 0)
 902                 return -EBUSY;
 903
 904         return 0;
 905 }
 906
 907 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 908 {
 909         static const u32 test_pat[4][6] = {
 910         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 911         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 912         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 913         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 914         };
 915         int chan;
 916
 917         for (chan = 0; chan < 4; chan++) {
 918                 int i;
 919
 920                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 921                              (chan * 0x2000) | 0x0200);
 922                 tg3_writephy(tp, 0x16, 0x0002);
 923
 924                 for (i = 0; i < 6; i++)
 925                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 926                                      test_pat[chan][i]);
 927
 928                 tg3_writephy(tp, 0x16, 0x0202);
 929                 if (tg3_wait_macro_done(tp)) {
 930                         *resetp = 1;
 931                         return -EBUSY;
 932                 }
 933
 934                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 935                              (chan * 0x2000) | 0x0200);
 936                 tg3_writephy(tp, 0x16, 0x0082);
 937                 if (tg3_wait_macro_done(tp)) {
 938                         *resetp = 1;
 939                         return -EBUSY;
 940                 }
 941
 942                 tg3_writephy(tp, 0x16, 0x0802);
 943                 if (tg3_wait_macro_done(tp)) {
 944                         *resetp = 1;
 945                         return -EBUSY;
 946                 }
 947
 948                 for (i = 0; i < 6; i += 2) {
 949                         u32 low, high;
 950
 951                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
 952                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
 953                             tg3_wait_macro_done(tp)) {
 954                                 *resetp = 1;
 955                                 return -EBUSY;
 956                         }
 957                         low &= 0x7fff;
 958                         high &= 0x000f;
 959                         if (low != test_pat[chan][i] ||
 960                             high != test_pat[chan][i+1]) {
 961                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
 962                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
 963                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
 964
 965                                 return -EBUSY;
 966                         }
 967                 }
 968         }
 969
 970         return 0;
 971 }
 972
 973 static int tg3_phy_reset_chanpat(struct tg3 *tp)
 974 {
 975         int chan;
 976
 977         for (chan = 0; chan < 4; chan++) {
 978                 int i;
 979
 980                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 981                              (chan * 0x2000) | 0x0200);
 982                 tg3_writephy(tp, 0x16, 0x0002);
 983                 for (i = 0; i < 6; i++)
 984                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
 985                 tg3_writephy(tp, 0x16, 0x0202);
 986                 if (tg3_wait_macro_done(tp))
 987                         return -EBUSY;
 988         }
 989
 990         return 0;
 991 }
 992
 993 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
 994 {
 995         u32 reg32, phy9_orig;
 996         int retries, do_phy_reset, err;
 997
 998         retries = 10;
 999         do_phy_reset = 1;
1000         do {
1001                 if (do_phy_reset) {
1002                         err = tg3_bmcr_reset(tp);
1003                         if (err)
1004                                 return err;
1005                         do_phy_reset = 0;
1006                 }
1007
1008                 /* Disable transmitter and interrupt.  */
1009                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1010                         continue;
1011
1012                 reg32 |= 0x3000;
1013                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1014
1015                 /* Set full-duplex, 1000 mbps.  */
1016                 tg3_writephy(tp, MII_BMCR,
1017                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1018
1019                 /* Set to master mode.  */
1020                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1021                         continue;
1022
1023                 tg3_writephy(tp, MII_TG3_CTRL,
1024                              (MII_TG3_CTRL_AS_MASTER |
1025                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1026
1027                 /* Enable SM_DSP_CLOCK and 6dB.  */
1028                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1029
1030                 /* Block the PHY control access.  */
1031                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1032                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1033
1034                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1035                 if (!err)
1036                         break;
1037         } while (--retries);
1038
1039         err = tg3_phy_reset_chanpat(tp);
1040         if (err)
1041                 return err;
1042
1043         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1044         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1045
1046         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1047         tg3_writephy(tp, 0x16, 0x0000);
1048
1049         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1050             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1051                 /* Set Extended packet length bit for jumbo frames */
1052                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1053         }
1054         else {
1055                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1056         }
1057
1058         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1059
1060         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1061                 reg32 &= ~0x3000;
1062                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1063         } else if (!err)
1064                 err = -EBUSY;
1065
1066         return err;
1067 }
1068
1069 static void tg3_link_report(struct tg3 *);
1070
1071 /* This will reset the tigon3 PHY if there is no valid
1072  * link unless the FORCE argument is non-zero.
1073  */
1074 static int tg3_phy_reset(struct tg3 *tp)
1075 {
1076         u32 phy_status;
1077         int err;
1078
1079         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1080                 u32 val;
1081
1082                 val = tr32(GRC_MISC_CFG);
1083                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1084                 udelay(40);
1085         }
1086         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1087         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1088         if (err != 0)
1089                 return -EBUSY;
1090
1091         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1092                 netif_carrier_off(tp->dev);
1093                 tg3_link_report(tp);
1094         }
1095
1096         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1097             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1098             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1099                 err = tg3_phy_reset_5703_4_5(tp);
1100                 if (err)
1101                         return err;
1102                 goto out;
1103         }
1104
1105         err = tg3_bmcr_reset(tp);
1106         if (err)
1107                 return err;
1108
1109         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
1110                 u32 val;
1111
1112                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1113                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1114                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1115                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1116                         udelay(40);
1117                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1118                 }
1119
1120                 /* Disable GPHY autopowerdown. */
1121                 tg3_writephy(tp, MII_TG3_MISC_SHDW,
1122                              MII_TG3_MISC_SHDW_WREN |
1123                              MII_TG3_MISC_SHDW_APD_SEL |
1124                              MII_TG3_MISC_SHDW_APD_WKTM_84MS);
1125         }
1126
1127 out:
1128         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1129                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1130                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1131                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1132                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1133                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1134                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1135         }
1136         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1137                 tg3_writephy(tp, 0x1c, 0x8d68);
1138                 tg3_writephy(tp, 0x1c, 0x8d68);
1139         }
1140         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1141                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1142                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1143                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1144                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1145                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1146                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1147                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1148                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1149         }
1150         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1151                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1152                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1153                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1154                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1155                         tg3_writephy(tp, MII_TG3_TEST1,
1156                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1157                 } else
1158                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1159                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1160         }
1161         /* Set Extended packet length bit (bit 14) on all chips that */
1162         /* support jumbo frames */
1163         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1164                 /* Cannot do read-modify-write on 5401 */
1165                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1166         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1167                 u32 phy_reg;
1168
1169                 /* Set bit 14 with read-modify-write to preserve other bits */
1170                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1171                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1172                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1173         }
1174
1175         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1176          * jumbo frames transmission.
1177          */
1178         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1179                 u32 phy_reg;
1180
1181                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1182                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1183                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1184         }
1185
1186         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1187                 /* adjust output voltage */
1188                 tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x12);
1189         }
1190
1191         tg3_phy_toggle_automdix(tp, 1);
1192         tg3_phy_set_wirespeed(tp);
1193         return 0;
1194 }
1195
1196 static void tg3_frob_aux_power(struct tg3 *tp)
1197 {
1198         struct tg3 *tp_peer = tp;
1199
1200         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
1201                 return;
1202
1203         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1204             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1205                 struct net_device *dev_peer;
1206
1207                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1208                 /* remove_one() may have been run on the peer. */
1209                 if (!dev_peer)
1210                         tp_peer = tp;
1211                 else
1212                         tp_peer = netdev_priv(dev_peer);
1213         }
1214
1215         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1216             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1217             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1218             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1219                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1220                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1221                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1222                                     (GRC_LCLCTRL_GPIO_OE0 |
1223                                      GRC_LCLCTRL_GPIO_OE1 |
1224                                      GRC_LCLCTRL_GPIO_OE2 |
1225                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1226                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1227                                     100);
1228                 } else {
1229                         u32 no_gpio2;
1230                         u32 grc_local_ctrl = 0;
1231
1232                         if (tp_peer != tp &&
1233                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1234                                 return;
1235
1236                         /* Workaround to prevent overdrawing Amps. */
1237                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1238                             ASIC_REV_5714) {
1239                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1240                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1241                                             grc_local_ctrl, 100);
1242                         }
1243
1244                         /* On 5753 and variants, GPIO2 cannot be used. */
1245                         no_gpio2 = tp->nic_sram_data_cfg &
1246                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1247
1248                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1249                                          GRC_LCLCTRL_GPIO_OE1 |
1250                                          GRC_LCLCTRL_GPIO_OE2 |
1251                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1252                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1253                         if (no_gpio2) {
1254                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1255                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1256                         }
1257                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1258                                                     grc_local_ctrl, 100);
1259
1260                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1261
1262                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1263                                                     grc_local_ctrl, 100);
1264
1265                         if (!no_gpio2) {
1266                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1267                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1268                                             grc_local_ctrl, 100);
1269                         }
1270                 }
1271         } else {
1272                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1273                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1274                         if (tp_peer != tp &&
1275                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1276                                 return;
1277
1278                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1279                                     (GRC_LCLCTRL_GPIO_OE1 |
1280                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1281
1282                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1283                                     GRC_LCLCTRL_GPIO_OE1, 100);
1284
1285                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1286                                     (GRC_LCLCTRL_GPIO_OE1 |
1287                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1288                 }
1289         }
1290 }
1291
1292 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
1293 {
1294         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
1295                 return 1;
1296         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
1297                 if (speed != SPEED_10)
1298                         return 1;
1299         } else if (speed == SPEED_10)
1300                 return 1;
1301
1302         return 0;
1303 }
1304
1305 static int tg3_setup_phy(struct tg3 *, int);
1306
1307 #define RESET_KIND_SHUTDOWN     0
1308 #define RESET_KIND_INIT         1
1309 #define RESET_KIND_SUSPEND      2
1310
1311 static void tg3_write_sig_post_reset(struct tg3 *, int);
1312 static int tg3_halt_cpu(struct tg3 *, u32);
1313 static int tg3_nvram_lock(struct tg3 *);
1314 static void tg3_nvram_unlock(struct tg3 *);
1315
1316 static void tg3_power_down_phy(struct tg3 *tp)
1317 {
1318         u32 val;
1319
1320         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
1321                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1322                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
1323                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
1324
1325                         sg_dig_ctrl |=
1326                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
1327                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
1328                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
1329                 }
1330                 return;
1331         }
1332
1333         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1334                 tg3_bmcr_reset(tp);
1335                 val = tr32(GRC_MISC_CFG);
1336                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
1337                 udelay(40);
1338                 return;
1339         } else {
1340                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
1341                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1342                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1343         }
1344
1345         /* The PHY should not be powered down on some chips because
1346          * of bugs.
1347          */
1348         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1349             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1350             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1351              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1352                 return;
1353
1354         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
1355                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1356                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1357                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
1358                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1359         }
1360
1361         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1362 }
1363
1364 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1365 {
1366         u32 misc_host_ctrl;
1367         u16 power_control, power_caps;
1368         int pm = tp->pm_cap;
1369
1370         /* Make sure register accesses (indirect or otherwise)
1371          * will function correctly.
1372          */
1373         pci_write_config_dword(tp->pdev,
1374                                TG3PCI_MISC_HOST_CTRL,
1375                                tp->misc_host_ctrl);
1376
1377         pci_read_config_word(tp->pdev,
1378                              pm + PCI_PM_CTRL,
1379                              &power_control);
1380         power_control |= PCI_PM_CTRL_PME_STATUS;
1381         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1382         switch (state) {
1383         case PCI_D0:
1384                 power_control |= 0;
1385                 pci_write_config_word(tp->pdev,
1386                                       pm + PCI_PM_CTRL,
1387                                       power_control);
1388                 udelay(100);    /* Delay after power state change */
1389
1390                 /* Switch out of Vaux if it is a NIC */
1391                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
1392                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1393
1394                 return 0;
1395
1396         case PCI_D1:
1397                 power_control |= 1;
1398                 break;
1399
1400         case PCI_D2:
1401                 power_control |= 2;
1402                 break;
1403
1404         case PCI_D3hot:
1405                 power_control |= 3;
1406                 break;
1407
1408         default:
1409                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1410                        "requested.\n",
1411                        tp->dev->name, state);
1412                 return -EINVAL;
1413         };
1414
1415         power_control |= PCI_PM_CTRL_PME_ENABLE;
1416
1417         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1418         tw32(TG3PCI_MISC_HOST_CTRL,
1419              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1420
1421         if (tp->link_config.phy_is_low_power == 0) {
1422                 tp->link_config.phy_is_low_power = 1;
1423                 tp->link_config.orig_speed = tp->link_config.speed;
1424                 tp->link_config.orig_duplex = tp->link_config.duplex;
1425                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1426         }
1427
1428         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1429                 tp->link_config.speed = SPEED_10;
1430                 tp->link_config.duplex = DUPLEX_HALF;
1431                 tp->link_config.autoneg = AUTONEG_ENABLE;
1432                 tg3_setup_phy(tp, 0);
1433         }
1434
1435         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1436                 u32 val;
1437
1438                 val = tr32(GRC_VCPU_EXT_CTRL);
1439                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
1440         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1441                 int i;
1442                 u32 val;
1443
1444                 for (i = 0; i < 200; i++) {
1445                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1446                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1447                                 break;
1448                         msleep(1);
1449                 }
1450         }
1451         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
1452                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1453                                                      WOL_DRV_STATE_SHUTDOWN |
1454                                                      WOL_DRV_WOL |
1455                                                      WOL_SET_MAGIC_PKT);
1456
1457         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1458
1459         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1460                 u32 mac_mode;
1461
1462                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1463                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1464                         udelay(40);
1465
1466                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
1467                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
1468                         else
1469                                 mac_mode = MAC_MODE_PORT_MODE_MII;
1470
1471                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
1472                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1473                             ASIC_REV_5700) {
1474                                 u32 speed = (tp->tg3_flags &
1475                                              TG3_FLAG_WOL_SPEED_100MB) ?
1476                                              SPEED_100 : SPEED_10;
1477                                 if (tg3_5700_link_polarity(tp, speed))
1478                                         mac_mode |= MAC_MODE_LINK_POLARITY;
1479                                 else
1480                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
1481                         }
1482                 } else {
1483                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1484                 }
1485
1486                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1487                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1488
1489                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1490                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1491                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1492
1493                 tw32_f(MAC_MODE, mac_mode);
1494                 udelay(100);
1495
1496                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1497                 udelay(10);
1498         }
1499
1500         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1501             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1502              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1503                 u32 base_val;
1504
1505                 base_val = tp->pci_clock_ctrl;
1506                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1507                              CLOCK_CTRL_TXCLK_DISABLE);
1508
1509                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1510                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1511         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1512                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
1513                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
1514                 /* do nothing */
1515         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1516                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1517                 u32 newbits1, newbits2;
1518
1519                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1520                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1521                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1522                                     CLOCK_CTRL_TXCLK_DISABLE |
1523                                     CLOCK_CTRL_ALTCLK);
1524                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1525                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1526                         newbits1 = CLOCK_CTRL_625_CORE;
1527                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1528                 } else {
1529                         newbits1 = CLOCK_CTRL_ALTCLK;
1530                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1531                 }
1532
1533                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1534                             40);
1535
1536                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1537                             40);
1538
1539                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1540                         u32 newbits3;
1541
1542                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1543                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1544                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1545                                             CLOCK_CTRL_TXCLK_DISABLE |
1546                                             CLOCK_CTRL_44MHZ_CORE);
1547                         } else {
1548                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1549                         }
1550
1551                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1552                                     tp->pci_clock_ctrl | newbits3, 40);
1553                 }
1554         }
1555
1556         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1557             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
1558             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
1559                 tg3_power_down_phy(tp);
1560
1561         tg3_frob_aux_power(tp);
1562
1563         /* Workaround for unstable PLL clock */
1564         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1565             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1566                 u32 val = tr32(0x7d00);
1567
1568                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1569                 tw32(0x7d00, val);
1570                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1571                         int err;
1572
1573                         err = tg3_nvram_lock(tp);
1574                         tg3_halt_cpu(tp, RX_CPU_BASE);
1575                         if (!err)
1576                                 tg3_nvram_unlock(tp);
1577                 }
1578         }
1579
1580         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1581
1582         /* Finally, set the new power state. */
1583         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1584         udelay(100);    /* Delay after power state change */
1585
1586         return 0;
1587 }
1588
1589 static void tg3_link_report(struct tg3 *tp)
1590 {
1591         if (!netif_carrier_ok(tp->dev)) {
1592                 if (netif_msg_link(tp))
1593                         printk(KERN_INFO PFX "%s: Link is down.\n",
1594                                tp->dev->name);
1595         } else if (netif_msg_link(tp)) {
1596                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1597                        tp->dev->name,
1598                        (tp->link_config.active_speed == SPEED_1000 ?
1599                         1000 :
1600                         (tp->link_config.active_speed == SPEED_100 ?
1601                          100 : 10)),
1602                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1603                         "full" : "half"));
1604
1605                 printk(KERN_INFO PFX
1606                        "%s: Flow control is %s for TX and %s for RX.\n",
1607                        tp->dev->name,
1608                        (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_TX) ?
1609                        "on" : "off",
1610                        (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_RX) ?
1611                        "on" : "off");
1612         }
1613 }
1614
1615 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1616 {
1617         u16 miireg;
1618
1619         if ((flow_ctrl & TG3_FLOW_CTRL_TX) && (flow_ctrl & TG3_FLOW_CTRL_RX))
1620                 miireg = ADVERTISE_PAUSE_CAP;
1621         else if (flow_ctrl & TG3_FLOW_CTRL_TX)
1622                 miireg = ADVERTISE_PAUSE_ASYM;
1623         else if (flow_ctrl & TG3_FLOW_CTRL_RX)
1624                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1625         else
1626                 miireg = 0;
1627
1628         return miireg;
1629 }
1630
1631 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1632 {
1633         u16 miireg;
1634
1635         if ((flow_ctrl & TG3_FLOW_CTRL_TX) && (flow_ctrl & TG3_FLOW_CTRL_RX))
1636                 miireg = ADVERTISE_1000XPAUSE;
1637         else if (flow_ctrl & TG3_FLOW_CTRL_TX)
1638                 miireg = ADVERTISE_1000XPSE_ASYM;
1639         else if (flow_ctrl & TG3_FLOW_CTRL_RX)
1640                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1641         else
1642                 miireg = 0;
1643
1644         return miireg;
1645 }
1646
1647 static u8 tg3_resolve_flowctrl_1000T(u16 lcladv, u16 rmtadv)
1648 {
1649         u8 cap = 0;
1650
1651         if (lcladv & ADVERTISE_PAUSE_CAP) {
1652                 if (lcladv & ADVERTISE_PAUSE_ASYM) {
1653                         if (rmtadv & LPA_PAUSE_CAP)
1654                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1655                         else if (rmtadv & LPA_PAUSE_ASYM)
1656                                 cap = TG3_FLOW_CTRL_RX;
1657                 } else {
1658                         if (rmtadv & LPA_PAUSE_CAP)
1659                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1660                 }
1661         } else if (lcladv & ADVERTISE_PAUSE_ASYM) {
1662                 if ((rmtadv & LPA_PAUSE_CAP) && (rmtadv & LPA_PAUSE_ASYM))
1663                         cap = TG3_FLOW_CTRL_TX;
1664         }
1665
1666         return cap;
1667 }
1668
1669 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1670 {
1671         u8 cap = 0;
1672
1673         if (lcladv & ADVERTISE_1000XPAUSE) {
1674                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1675                         if (rmtadv & LPA_1000XPAUSE)
1676                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1677                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1678                                 cap = TG3_FLOW_CTRL_RX;
1679                 } else {
1680                         if (rmtadv & LPA_1000XPAUSE)
1681                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1682                 }
1683         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1684                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1685                         cap = TG3_FLOW_CTRL_TX;
1686         }
1687
1688         return cap;
1689 }
1690
1691 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1692 {
1693         u8 new_tg3_flags = 0;
1694         u32 old_rx_mode = tp->rx_mode;
1695         u32 old_tx_mode = tp->tx_mode;
1696
1697         if (tp->link_config.autoneg == AUTONEG_ENABLE &&
1698             (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)) {
1699                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
1700                         new_tg3_flags = tg3_resolve_flowctrl_1000X(local_adv,
1701                                                                    remote_adv);
1702                 else
1703                         new_tg3_flags = tg3_resolve_flowctrl_1000T(local_adv,
1704                                                                    remote_adv);
1705         } else {
1706                 new_tg3_flags = tp->link_config.flowctrl;
1707         }
1708
1709         tp->link_config.active_flowctrl = new_tg3_flags;
1710
1711         if (new_tg3_flags & TG3_FLOW_CTRL_RX)
1712                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1713         else
1714                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1715
1716         if (old_rx_mode != tp->rx_mode) {
1717                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1718         }
1719
1720         if (new_tg3_flags & TG3_FLOW_CTRL_TX)
1721                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1722         else
1723                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1724
1725         if (old_tx_mode != tp->tx_mode) {
1726                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1727         }
1728 }
1729
1730 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1731 {
1732         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1733         case MII_TG3_AUX_STAT_10HALF:
1734                 *speed = SPEED_10;
1735                 *duplex = DUPLEX_HALF;
1736                 break;
1737
1738         case MII_TG3_AUX_STAT_10FULL:
1739                 *speed = SPEED_10;
1740                 *duplex = DUPLEX_FULL;
1741                 break;
1742
1743         case MII_TG3_AUX_STAT_100HALF:
1744                 *speed = SPEED_100;
1745                 *duplex = DUPLEX_HALF;
1746                 break;
1747
1748         case MII_TG3_AUX_STAT_100FULL:
1749                 *speed = SPEED_100;
1750                 *duplex = DUPLEX_FULL;
1751                 break;
1752
1753         case MII_TG3_AUX_STAT_1000HALF:
1754                 *speed = SPEED_1000;
1755                 *duplex = DUPLEX_HALF;
1756                 break;
1757
1758         case MII_TG3_AUX_STAT_1000FULL:
1759                 *speed = SPEED_1000;
1760                 *duplex = DUPLEX_FULL;
1761                 break;
1762
1763         default:
1764                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1765                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
1766                                  SPEED_10;
1767                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
1768                                   DUPLEX_HALF;
1769                         break;
1770                 }
1771                 *speed = SPEED_INVALID;
1772                 *duplex = DUPLEX_INVALID;
1773                 break;
1774         };
1775 }
1776
1777 static void tg3_phy_copper_begin(struct tg3 *tp)
1778 {
1779         u32 new_adv;
1780         int i;
1781
1782         if (tp->link_config.phy_is_low_power) {
1783                 /* Entering low power mode.  Disable gigabit and
1784                  * 100baseT advertisements.
1785                  */
1786                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1787
1788                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1789                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1790                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1791                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1792
1793                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1794         } else if (tp->link_config.speed == SPEED_INVALID) {
1795                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1796                         tp->link_config.advertising &=
1797                                 ~(ADVERTISED_1000baseT_Half |
1798                                   ADVERTISED_1000baseT_Full);
1799
1800                 new_adv = ADVERTISE_CSMA;
1801                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1802                         new_adv |= ADVERTISE_10HALF;
1803                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1804                         new_adv |= ADVERTISE_10FULL;
1805                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1806                         new_adv |= ADVERTISE_100HALF;
1807                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1808                         new_adv |= ADVERTISE_100FULL;
1809
1810                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
1811
1812                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1813
1814                 if (tp->link_config.advertising &
1815                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1816                         new_adv = 0;
1817                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1818                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1819                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1820                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1821                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1822                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1823                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1824                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1825                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1826                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1827                 } else {
1828                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1829                 }
1830         } else {
1831                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
1832                 new_adv |= ADVERTISE_CSMA;
1833
1834                 /* Asking for a specific link mode. */
1835                 if (tp->link_config.speed == SPEED_1000) {
1836                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1837
1838                         if (tp->link_config.duplex == DUPLEX_FULL)
1839                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1840                         else
1841                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1842                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1843                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1844                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1845                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1846                 } else {
1847                         if (tp->link_config.speed == SPEED_100) {
1848                                 if (tp->link_config.duplex == DUPLEX_FULL)
1849                                         new_adv |= ADVERTISE_100FULL;
1850                                 else
1851                                         new_adv |= ADVERTISE_100HALF;
1852                         } else {
1853                                 if (tp->link_config.duplex == DUPLEX_FULL)
1854                                         new_adv |= ADVERTISE_10FULL;
1855                                 else
1856                                         new_adv |= ADVERTISE_10HALF;
1857                         }
1858                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1859
1860                         new_adv = 0;
1861                 }
1862
1863                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1864         }
1865
1866         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1867             tp->link_config.speed != SPEED_INVALID) {
1868                 u32 bmcr, orig_bmcr;
1869
1870                 tp->link_config.active_speed = tp->link_config.speed;
1871                 tp->link_config.active_duplex = tp->link_config.duplex;
1872
1873                 bmcr = 0;
1874                 switch (tp->link_config.speed) {
1875                 default:
1876                 case SPEED_10:
1877                         break;
1878
1879                 case SPEED_100:
1880                         bmcr |= BMCR_SPEED100;
1881                         break;
1882
1883                 case SPEED_1000:
1884                         bmcr |= TG3_BMCR_SPEED1000;
1885                         break;
1886                 };
1887
1888                 if (tp->link_config.duplex == DUPLEX_FULL)
1889                         bmcr |= BMCR_FULLDPLX;
1890
1891                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1892                     (bmcr != orig_bmcr)) {
1893                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1894                         for (i = 0; i < 1500; i++) {
1895                                 u32 tmp;
1896
1897                                 udelay(10);
1898                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1899                                     tg3_readphy(tp, MII_BMSR, &tmp))
1900                                         continue;
1901                                 if (!(tmp & BMSR_LSTATUS)) {
1902                                         udelay(40);
1903                                         break;
1904                                 }
1905                         }
1906                         tg3_writephy(tp, MII_BMCR, bmcr);
1907                         udelay(40);
1908                 }
1909         } else {
1910                 tg3_writephy(tp, MII_BMCR,
1911                              BMCR_ANENABLE | BMCR_ANRESTART);
1912         }
1913 }
1914
1915 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1916 {
1917         int err;
1918
1919         /* Turn off tap power management. */
1920         /* Set Extended packet length bit */
1921         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1922
1923         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1924         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1925
1926         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1927         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1928
1929         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1930         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1931
1932         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1933         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1934
1935         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1936         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1937
1938         udelay(40);
1939
1940         return err;
1941 }
1942
1943 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
1944 {
1945         u32 adv_reg, all_mask = 0;
1946
1947         if (mask & ADVERTISED_10baseT_Half)
1948                 all_mask |= ADVERTISE_10HALF;
1949         if (mask & ADVERTISED_10baseT_Full)
1950                 all_mask |= ADVERTISE_10FULL;
1951         if (mask & ADVERTISED_100baseT_Half)
1952                 all_mask |= ADVERTISE_100HALF;
1953         if (mask & ADVERTISED_100baseT_Full)
1954                 all_mask |= ADVERTISE_100FULL;
1955
1956         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1957                 return 0;
1958
1959         if ((adv_reg & all_mask) != all_mask)
1960                 return 0;
1961         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1962                 u32 tg3_ctrl;
1963
1964                 all_mask = 0;
1965                 if (mask & ADVERTISED_1000baseT_Half)
1966                         all_mask |= ADVERTISE_1000HALF;
1967                 if (mask & ADVERTISED_1000baseT_Full)
1968                         all_mask |= ADVERTISE_1000FULL;
1969
1970                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1971                         return 0;
1972
1973                 if ((tg3_ctrl & all_mask) != all_mask)
1974                         return 0;
1975         }
1976         return 1;
1977 }
1978
1979 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
1980 {
1981         u32 curadv, reqadv;
1982
1983         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
1984                 return 1;
1985
1986         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
1987         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
1988
1989         if (tp->link_config.active_duplex == DUPLEX_FULL) {
1990                 if (curadv != reqadv)
1991                         return 0;
1992
1993                 if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)
1994                         tg3_readphy(tp, MII_LPA, rmtadv);
1995         } else {
1996                 /* Reprogram the advertisement register, even if it
1997                  * does not affect the current link.  If the link
1998                  * gets renegotiated in the future, we can save an
1999                  * additional renegotiation cycle by advertising
2000                  * it correctly in the first place.
2001                  */
2002                 if (curadv != reqadv) {
2003                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
2004                                      ADVERTISE_PAUSE_ASYM);
2005                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
2006                 }
2007         }
2008
2009         return 1;
2010 }
2011
2012 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
2013 {
2014         int current_link_up;
2015         u32 bmsr, dummy;
2016         u32 lcl_adv, rmt_adv;
2017         u16 current_speed;
2018         u8 current_duplex;
2019         int i, err;
2020
2021         tw32(MAC_EVENT, 0);
2022
2023         tw32_f(MAC_STATUS,
2024              (MAC_STATUS_SYNC_CHANGED |
2025               MAC_STATUS_CFG_CHANGED |
2026               MAC_STATUS_MI_COMPLETION |
2027               MAC_STATUS_LNKSTATE_CHANGED));
2028         udelay(40);
2029
2030         tp->mi_mode = MAC_MI_MODE_BASE;
2031         tw32_f(MAC_MI_MODE, tp->mi_mode);
2032         udelay(80);
2033
2034         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
2035
2036         /* Some third-party PHYs need to be reset on link going
2037          * down.
2038          */
2039         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2040              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2041              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
2042             netif_carrier_ok(tp->dev)) {
2043                 tg3_readphy(tp, MII_BMSR, &bmsr);
2044                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2045                     !(bmsr & BMSR_LSTATUS))
2046                         force_reset = 1;
2047         }
2048         if (force_reset)
2049                 tg3_phy_reset(tp);
2050
2051         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
2052                 tg3_readphy(tp, MII_BMSR, &bmsr);
2053                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
2054                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
2055                         bmsr = 0;
2056
2057                 if (!(bmsr & BMSR_LSTATUS)) {
2058                         err = tg3_init_5401phy_dsp(tp);
2059                         if (err)
2060                                 return err;
2061
2062                         tg3_readphy(tp, MII_BMSR, &bmsr);
2063                         for (i = 0; i < 1000; i++) {
2064                                 udelay(10);
2065                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2066                                     (bmsr & BMSR_LSTATUS)) {
2067                                         udelay(40);
2068                                         break;
2069                                 }
2070                         }
2071
2072                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
2073                             !(bmsr & BMSR_LSTATUS) &&
2074                             tp->link_config.active_speed == SPEED_1000) {
2075                                 err = tg3_phy_reset(tp);
2076                                 if (!err)
2077                                         err = tg3_init_5401phy_dsp(tp);
2078                                 if (err)
2079                                         return err;
2080                         }
2081                 }
2082         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2083                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
2084                 /* 5701 {A0,B0} CRC bug workaround */
2085                 tg3_writephy(tp, 0x15, 0x0a75);
2086                 tg3_writephy(tp, 0x1c, 0x8c68);
2087                 tg3_writephy(tp, 0x1c, 0x8d68);
2088                 tg3_writephy(tp, 0x1c, 0x8c68);
2089         }
2090
2091         /* Clear pending interrupts... */
2092         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2093         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2094
2095         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
2096                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
2097         else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
2098                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
2099
2100         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2101             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2102                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
2103                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
2104                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
2105                 else
2106                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
2107         }
2108
2109         current_link_up = 0;
2110         current_speed = SPEED_INVALID;
2111         current_duplex = DUPLEX_INVALID;
2112
2113         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
2114                 u32 val;
2115
2116                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
2117                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
2118                 if (!(val & (1 << 10))) {
2119                         val |= (1 << 10);
2120                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
2121                         goto relink;
2122                 }
2123         }
2124
2125         bmsr = 0;
2126         for (i = 0; i < 100; i++) {
2127                 tg3_readphy(tp, MII_BMSR, &bmsr);
2128                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2129                     (bmsr & BMSR_LSTATUS))
2130                         break;
2131                 udelay(40);
2132         }
2133
2134         if (bmsr & BMSR_LSTATUS) {
2135                 u32 aux_stat, bmcr;
2136
2137                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
2138                 for (i = 0; i < 2000; i++) {
2139                         udelay(10);
2140                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
2141                             aux_stat)
2142                                 break;
2143                 }
2144
2145                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
2146                                              &current_speed,
2147                                              &current_duplex);
2148
2149                 bmcr = 0;
2150                 for (i = 0; i < 200; i++) {
2151                         tg3_readphy(tp, MII_BMCR, &bmcr);
2152                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
2153                                 continue;
2154                         if (bmcr && bmcr != 0x7fff)
2155                                 break;
2156                         udelay(10);
2157                 }
2158
2159                 lcl_adv = 0;
2160                 rmt_adv = 0;
2161
2162                 tp->link_config.active_speed = current_speed;
2163                 tp->link_config.active_duplex = current_duplex;
2164
2165                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2166                         if ((bmcr & BMCR_ANENABLE) &&
2167                             tg3_copper_is_advertising_all(tp,
2168                                                 tp->link_config.advertising)) {
2169                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
2170                                                                   &rmt_adv))
2171                                         current_link_up = 1;
2172                         }
2173                 } else {
2174                         if (!(bmcr & BMCR_ANENABLE) &&
2175                             tp->link_config.speed == current_speed &&
2176                             tp->link_config.duplex == current_duplex &&
2177                             tp->link_config.flowctrl ==
2178                             tp->link_config.active_flowctrl) {
2179                                 current_link_up = 1;
2180                         }
2181                 }
2182
2183                 if (current_link_up == 1 &&
2184                     tp->link_config.active_duplex == DUPLEX_FULL)
2185                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
2186         }
2187
2188 relink:
2189         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
2190                 u32 tmp;
2191
2192                 tg3_phy_copper_begin(tp);
2193
2194                 tg3_readphy(tp, MII_BMSR, &tmp);
2195                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
2196                     (tmp & BMSR_LSTATUS))
2197                         current_link_up = 1;
2198         }
2199
2200         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
2201         if (current_link_up == 1) {
2202                 if (tp->link_config.active_speed == SPEED_100 ||
2203                     tp->link_config.active_speed == SPEED_10)
2204                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
2205                 else
2206                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2207         } else
2208                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2209
2210         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2211         if (tp->link_config.active_duplex == DUPLEX_HALF)
2212                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2213
2214         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
2215                 if (current_link_up == 1 &&
2216                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
2217                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
2218                 else
2219                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2220         }
2221
2222         /* ??? Without this setting Netgear GA302T PHY does not
2223          * ??? send/receive packets...
2224          */
2225         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
2226             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
2227                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
2228                 tw32_f(MAC_MI_MODE, tp->mi_mode);
2229                 udelay(80);
2230         }
2231
2232         tw32_f(MAC_MODE, tp->mac_mode);
2233         udelay(40);
2234
2235         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
2236                 /* Polled via timer. */
2237                 tw32_f(MAC_EVENT, 0);
2238         } else {
2239                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2240         }
2241         udelay(40);
2242
2243         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
2244             current_link_up == 1 &&
2245             tp->link_config.active_speed == SPEED_1000 &&
2246             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
2247              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
2248                 udelay(120);
2249                 tw32_f(MAC_STATUS,
2250                      (MAC_STATUS_SYNC_CHANGED |
2251                       MAC_STATUS_CFG_CHANGED));
2252                 udelay(40);
2253                 tg3_write_mem(tp,
2254                               NIC_SRAM_FIRMWARE_MBOX,
2255                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
2256         }
2257
2258         if (current_link_up != netif_carrier_ok(tp->dev)) {
2259                 if (current_link_up)
2260                         netif_carrier_on(tp->dev);
2261                 else
2262                         netif_carrier_off(tp->dev);
2263                 tg3_link_report(tp);
2264         }
2265
2266         return 0;
2267 }
2268
2269 struct tg3_fiber_aneginfo {
2270         int state;
2271 #define ANEG_STATE_UNKNOWN              0
2272 #define ANEG_STATE_AN_ENABLE            1
2273 #define ANEG_STATE_RESTART_INIT         2
2274 #define ANEG_STATE_RESTART              3
2275 #define ANEG_STATE_DISABLE_LINK_OK      4
2276 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2277 #define ANEG_STATE_ABILITY_DETECT       6
2278 #define ANEG_STATE_ACK_DETECT_INIT      7
2279 #define ANEG_STATE_ACK_DETECT           8
2280 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2281 #define ANEG_STATE_COMPLETE_ACK         10
2282 #define ANEG_STATE_IDLE_DETECT_INIT     11
2283 #define ANEG_STATE_IDLE_DETECT          12
2284 #define ANEG_STATE_LINK_OK              13
2285 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2286 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2287
2288         u32 flags;
2289 #define MR_AN_ENABLE            0x00000001
2290 #define MR_RESTART_AN           0x00000002
2291 #define MR_AN_COMPLETE          0x00000004
2292 #define MR_PAGE_RX              0x00000008
2293 #define MR_NP_LOADED            0x00000010
2294 #define MR_TOGGLE_TX            0x00000020
2295 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2296 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2297 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2298 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2299 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2300 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2301 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2302 #define MR_TOGGLE_RX            0x00002000
2303 #define MR_NP_RX                0x00004000
2304
2305 #define MR_LINK_OK              0x80000000
2306
2307         unsigned long link_time, cur_time;
2308
2309         u32 ability_match_cfg;
2310         int ability_match_count;
2311
2312         char ability_match, idle_match, ack_match;
2313
2314         u32 txconfig, rxconfig;
2315 #define ANEG_CFG_NP             0x00000080
2316 #define ANEG_CFG_ACK            0x00000040
2317 #define ANEG_CFG_RF2            0x00000020
2318 #define ANEG_CFG_RF1            0x00000010
2319 #define ANEG_CFG_PS2            0x00000001
2320 #define ANEG_CFG_PS1            0x00008000
2321 #define ANEG_CFG_HD             0x00004000
2322 #define ANEG_CFG_FD             0x00002000
2323 #define ANEG_CFG_INVAL          0x00001f06
2324
2325 };
2326 #define ANEG_OK         0
2327 #define ANEG_DONE       1
2328 #define ANEG_TIMER_ENAB 2
2329 #define ANEG_FAILED     -1
2330
2331 #define ANEG_STATE_SETTLE_TIME  10000
2332
2333 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2334                                    struct tg3_fiber_aneginfo *ap)
2335 {
2336         u16 flowctrl;
2337         unsigned long delta;
2338         u32 rx_cfg_reg;
2339         int ret;
2340
2341         if (ap->state == ANEG_STATE_UNKNOWN) {
2342                 ap->rxconfig = 0;
2343                 ap->link_time = 0;
2344                 ap->cur_time = 0;
2345                 ap->ability_match_cfg = 0;
2346                 ap->ability_match_count = 0;
2347                 ap->ability_match = 0;
2348                 ap->idle_match = 0;
2349                 ap->ack_match = 0;
2350         }
2351         ap->cur_time++;
2352
2353         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2354                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2355
2356                 if (rx_cfg_reg != ap->ability_match_cfg) {
2357                         ap->ability_match_cfg = rx_cfg_reg;
2358                         ap->ability_match = 0;
2359                         ap->ability_match_count = 0;
2360                 } else {
2361                         if (++ap->ability_match_count > 1) {
2362                                 ap->ability_match = 1;
2363                                 ap->ability_match_cfg = rx_cfg_reg;
2364                         }
2365                 }
2366                 if (rx_cfg_reg & ANEG_CFG_ACK)
2367                         ap->ack_match = 1;
2368                 else
2369                         ap->ack_match = 0;
2370
2371                 ap->idle_match = 0;
2372         } else {
2373                 ap->idle_match = 1;
2374                 ap->ability_match_cfg = 0;
2375                 ap->ability_match_count = 0;
2376                 ap->ability_match = 0;
2377                 ap->ack_match = 0;
2378
2379                 rx_cfg_reg = 0;
2380         }
2381
2382         ap->rxconfig = rx_cfg_reg;
2383         ret = ANEG_OK;
2384
2385         switch(ap->state) {
2386         case ANEG_STATE_UNKNOWN:
2387                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2388                         ap->state = ANEG_STATE_AN_ENABLE;
2389
2390                 /* fallthru */
2391         case ANEG_STATE_AN_ENABLE:
2392                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2393                 if (ap->flags & MR_AN_ENABLE) {
2394                         ap->link_time = 0;
2395                         ap->cur_time = 0;
2396                         ap->ability_match_cfg = 0;
2397                         ap->ability_match_count = 0;
2398                         ap->ability_match = 0;
2399                         ap->idle_match = 0;
2400                         ap->ack_match = 0;
2401
2402                         ap->state = ANEG_STATE_RESTART_INIT;
2403                 } else {
2404                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2405                 }
2406                 break;
2407
2408         case ANEG_STATE_RESTART_INIT:
2409                 ap->link_time = ap->cur_time;
2410                 ap->flags &= ~(MR_NP_LOADED);
2411                 ap->txconfig = 0;
2412                 tw32(MAC_TX_AUTO_NEG, 0);
2413                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2414                 tw32_f(MAC_MODE, tp->mac_mode);
2415                 udelay(40);
2416
2417                 ret = ANEG_TIMER_ENAB;
2418                 ap->state = ANEG_STATE_RESTART;
2419
2420                 /* fallthru */
2421         case ANEG_STATE_RESTART:
2422                 delta = ap->cur_time - ap->link_time;
2423                 if (delta > ANEG_STATE_SETTLE_TIME) {
2424                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2425                 } else {
2426                         ret = ANEG_TIMER_ENAB;
2427                 }
2428                 break;
2429
2430         case ANEG_STATE_DISABLE_LINK_OK:
2431                 ret = ANEG_DONE;
2432                 break;
2433
2434         case ANEG_STATE_ABILITY_DETECT_INIT:
2435                 ap->flags &= ~(MR_TOGGLE_TX);
2436                 ap->txconfig = ANEG_CFG_FD;
2437                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
2438                 if (flowctrl & ADVERTISE_1000XPAUSE)
2439                         ap->txconfig |= ANEG_CFG_PS1;
2440                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
2441                         ap->txconfig |= ANEG_CFG_PS2;
2442                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2443                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2444                 tw32_f(MAC_MODE, tp->mac_mode);
2445                 udelay(40);
2446
2447                 ap->state = ANEG_STATE_ABILITY_DETECT;
2448                 break;
2449
2450         case ANEG_STATE_ABILITY_DETECT:
2451                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2452                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2453                 }
2454                 break;
2455
2456         case ANEG_STATE_ACK_DETECT_INIT:
2457                 ap->txconfig |= ANEG_CFG_ACK;
2458                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2459                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2460                 tw32_f(MAC_MODE, tp->mac_mode);
2461                 udelay(40);
2462
2463                 ap->state = ANEG_STATE_ACK_DETECT;
2464
2465                 /* fallthru */
2466         case ANEG_STATE_ACK_DETECT:
2467                 if (ap->ack_match != 0) {
2468                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2469                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2470                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2471                         } else {
2472                                 ap->state = ANEG_STATE_AN_ENABLE;
2473                         }
2474                 } else if (ap->ability_match != 0 &&
2475                            ap->rxconfig == 0) {
2476                         ap->state = ANEG_STATE_AN_ENABLE;
2477                 }
2478                 break;
2479
2480         case ANEG_STATE_COMPLETE_ACK_INIT:
2481                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2482                         ret = ANEG_FAILED;
2483                         break;
2484                 }
2485                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2486                                MR_LP_ADV_HALF_DUPLEX |
2487                                MR_LP_ADV_SYM_PAUSE |
2488                                MR_LP_ADV_ASYM_PAUSE |
2489                                MR_LP_ADV_REMOTE_FAULT1 |
2490                                MR_LP_ADV_REMOTE_FAULT2 |
2491                                MR_LP_ADV_NEXT_PAGE |
2492                                MR_TOGGLE_RX |
2493                                MR_NP_RX);
2494                 if (ap->rxconfig & ANEG_CFG_FD)
2495                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2496                 if (ap->rxconfig & ANEG_CFG_HD)
2497                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2498                 if (ap->rxconfig & ANEG_CFG_PS1)
2499                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2500                 if (ap->rxconfig & ANEG_CFG_PS2)
2501                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2502                 if (ap->rxconfig & ANEG_CFG_RF1)
2503                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2504                 if (ap->rxconfig & ANEG_CFG_RF2)
2505                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2506                 if (ap->rxconfig & ANEG_CFG_NP)
2507                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2508
2509                 ap->link_time = ap->cur_time;
2510
2511                 ap->flags ^= (MR_TOGGLE_TX);
2512                 if (ap->rxconfig & 0x0008)
2513                         ap->flags |= MR_TOGGLE_RX;
2514                 if (ap->rxconfig & ANEG_CFG_NP)
2515                         ap->flags |= MR_NP_RX;
2516                 ap->flags |= MR_PAGE_RX;
2517
2518                 ap->state = ANEG_STATE_COMPLETE_ACK;
2519                 ret = ANEG_TIMER_ENAB;
2520                 break;
2521
2522         case ANEG_STATE_COMPLETE_ACK:
2523                 if (ap->ability_match != 0 &&
2524                     ap->rxconfig == 0) {
2525                         ap->state = ANEG_STATE_AN_ENABLE;
2526                         break;
2527                 }
2528                 delta = ap->cur_time - ap->link_time;
2529                 if (delta > ANEG_STATE_SETTLE_TIME) {
2530                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2531                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2532                         } else {
2533                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2534                                     !(ap->flags & MR_NP_RX)) {
2535                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2536                                 } else {
2537                                         ret = ANEG_FAILED;
2538                                 }
2539                         }
2540                 }
2541                 break;
2542
2543         case ANEG_STATE_IDLE_DETECT_INIT:
2544                 ap->link_time = ap->cur_time;
2545                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2546                 tw32_f(MAC_MODE, tp->mac_mode);
2547                 udelay(40);
2548
2549                 ap->state = ANEG_STATE_IDLE_DETECT;
2550                 ret = ANEG_TIMER_ENAB;
2551                 break;
2552
2553         case ANEG_STATE_IDLE_DETECT:
2554                 if (ap->ability_match != 0 &&
2555                     ap->rxconfig == 0) {
2556                         ap->state = ANEG_STATE_AN_ENABLE;
2557                         break;
2558                 }
2559                 delta = ap->cur_time - ap->link_time;
2560                 if (delta > ANEG_STATE_SETTLE_TIME) {
2561                         /* XXX another gem from the Broadcom driver :( */
2562                         ap->state = ANEG_STATE_LINK_OK;
2563                 }
2564                 break;
2565
2566         case ANEG_STATE_LINK_OK:
2567                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2568                 ret = ANEG_DONE;
2569                 break;
2570
2571         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2572                 /* ??? unimplemented */
2573                 break;
2574
2575         case ANEG_STATE_NEXT_PAGE_WAIT:
2576                 /* ??? unimplemented */
2577                 break;
2578
2579         default:
2580                 ret = ANEG_FAILED;
2581                 break;
2582         };
2583
2584         return ret;
2585 }
2586
2587 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
2588 {
2589         int res = 0;
2590         struct tg3_fiber_aneginfo aninfo;
2591         int status = ANEG_FAILED;
2592         unsigned int tick;
2593         u32 tmp;
2594
2595         tw32_f(MAC_TX_AUTO_NEG, 0);
2596
2597         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2598         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2599         udelay(40);
2600
2601         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2602         udelay(40);
2603
2604         memset(&aninfo, 0, sizeof(aninfo));
2605         aninfo.flags |= MR_AN_ENABLE;
2606         aninfo.state = ANEG_STATE_UNKNOWN;
2607         aninfo.cur_time = 0;
2608         tick = 0;
2609         while (++tick < 195000) {
2610                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2611                 if (status == ANEG_DONE || status == ANEG_FAILED)
2612                         break;
2613
2614                 udelay(1);
2615         }
2616
2617         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2618         tw32_f(MAC_MODE, tp->mac_mode);
2619         udelay(40);
2620
2621         *txflags = aninfo.txconfig;
2622         *rxflags = aninfo.flags;
2623
2624         if (status == ANEG_DONE &&
2625             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2626                              MR_LP_ADV_FULL_DUPLEX)))
2627                 res = 1;
2628
2629         return res;
2630 }
2631
2632 static void tg3_init_bcm8002(struct tg3 *tp)
2633 {
2634         u32 mac_status = tr32(MAC_STATUS);
2635         int i;
2636
2637         /* Reset when initting first time or we have a link. */
2638         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2639             !(mac_status & MAC_STATUS_PCS_SYNCED))
2640                 return;
2641
2642         /* Set PLL lock range. */
2643         tg3_writephy(tp, 0x16, 0x8007);
2644
2645         /* SW reset */
2646         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2647
2648         /* Wait for reset to complete. */
2649         /* XXX schedule_timeout() ... */
2650         for (i = 0; i < 500; i++)
2651                 udelay(10);
2652
2653         /* Config mode; select PMA/Ch 1 regs. */
2654         tg3_writephy(tp, 0x10, 0x8411);
2655
2656         /* Enable auto-lock and comdet, select txclk for tx. */
2657         tg3_writephy(tp, 0x11, 0x0a10);
2658
2659         tg3_writephy(tp, 0x18, 0x00a0);
2660         tg3_writephy(tp, 0x16, 0x41ff);
2661
2662         /* Assert and deassert POR. */
2663         tg3_writephy(tp, 0x13, 0x0400);
2664         udelay(40);
2665         tg3_writephy(tp, 0x13, 0x0000);
2666
2667         tg3_writephy(tp, 0x11, 0x0a50);
2668         udelay(40);
2669         tg3_writephy(tp, 0x11, 0x0a10);
2670
2671         /* Wait for signal to stabilize */
2672         /* XXX schedule_timeout() ... */
2673         for (i = 0; i < 15000; i++)
2674                 udelay(10);
2675
2676         /* Deselect the channel register so we can read the PHYID
2677          * later.
2678          */
2679         tg3_writephy(tp, 0x10, 0x8011);
2680 }
2681
2682 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2683 {
2684         u16 flowctrl;
2685         u32 sg_dig_ctrl, sg_dig_status;
2686         u32 serdes_cfg, expected_sg_dig_ctrl;
2687         int workaround, port_a;
2688         int current_link_up;
2689
2690         serdes_cfg = 0;
2691         expected_sg_dig_ctrl = 0;
2692         workaround = 0;
2693         port_a = 1;
2694         current_link_up = 0;
2695
2696         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2697             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2698                 workaround = 1;
2699                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2700                         port_a = 0;
2701
2702                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2703                 /* preserve bits 20-23 for voltage regulator */
2704                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2705         }
2706
2707         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2708
2709         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2710                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
2711                         if (workaround) {
2712                                 u32 val = serdes_cfg;
2713
2714                                 if (port_a)
2715                                         val |= 0xc010000;
2716                                 else
2717                                         val |= 0x4010000;
2718                                 tw32_f(MAC_SERDES_CFG, val);
2719                         }
2720
2721                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
2722                 }
2723                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2724                         tg3_setup_flow_control(tp, 0, 0);
2725                         current_link_up = 1;
2726                 }
2727                 goto out;
2728         }
2729
2730         /* Want auto-negotiation.  */
2731         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
2732
2733         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
2734         if (flowctrl & ADVERTISE_1000XPAUSE)
2735                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
2736         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
2737                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
2738
2739         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2740                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
2741                     tp->serdes_counter &&
2742                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
2743                                     MAC_STATUS_RCVD_CFG)) ==
2744                      MAC_STATUS_PCS_SYNCED)) {
2745                         tp->serdes_counter--;
2746                         current_link_up = 1;
2747                         goto out;
2748                 }
2749 restart_autoneg:
2750                 if (workaround)
2751                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2752                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
2753                 udelay(5);
2754                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2755
2756                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2757                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2758         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2759                                  MAC_STATUS_SIGNAL_DET)) {
2760                 sg_dig_status = tr32(SG_DIG_STATUS);
2761                 mac_status = tr32(MAC_STATUS);
2762
2763                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
2764                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2765                         u32 local_adv = 0, remote_adv = 0;
2766
2767                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
2768                                 local_adv |= ADVERTISE_1000XPAUSE;
2769                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
2770                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
2771
2772                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
2773                                 remote_adv |= LPA_1000XPAUSE;
2774                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
2775                                 remote_adv |= LPA_1000XPAUSE_ASYM;
2776
2777                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2778                         current_link_up = 1;
2779                         tp->serdes_counter = 0;
2780                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2781                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
2782                         if (tp->serdes_counter)
2783                                 tp->serdes_counter--;
2784                         else {
2785                                 if (workaround) {
2786                                         u32 val = serdes_cfg;
2787
2788                                         if (port_a)
2789                                                 val |= 0xc010000;
2790                                         else
2791                                                 val |= 0x4010000;
2792
2793                                         tw32_f(MAC_SERDES_CFG, val);
2794                                 }
2795
2796                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
2797                                 udelay(40);
2798
2799                                 /* Link parallel detection - link is up */
2800                                 /* only if we have PCS_SYNC and not */
2801                                 /* receiving config code words */
2802                                 mac_status = tr32(MAC_STATUS);
2803                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2804                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2805                                         tg3_setup_flow_control(tp, 0, 0);
2806                                         current_link_up = 1;
2807                                         tp->tg3_flags2 |=
2808                                                 TG3_FLG2_PARALLEL_DETECT;
2809                                         tp->serdes_counter =
2810                                                 SERDES_PARALLEL_DET_TIMEOUT;
2811                                 } else
2812                                         goto restart_autoneg;
2813                         }
2814                 }
2815         } else {
2816                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2817                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2818         }
2819
2820 out:
2821         return current_link_up;
2822 }
2823
2824 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2825 {
2826         int current_link_up = 0;
2827
2828         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
2829                 goto out;
2830
2831         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2832                 u32 txflags, rxflags;
2833                 int i;
2834
2835                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
2836                         u32 local_adv = 0, remote_adv = 0;
2837
2838                         if (txflags & ANEG_CFG_PS1)
2839                                 local_adv |= ADVERTISE_1000XPAUSE;
2840                         if (txflags & ANEG_CFG_PS2)
2841                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
2842
2843                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
2844                                 remote_adv |= LPA_1000XPAUSE;
2845                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
2846                                 remote_adv |= LPA_1000XPAUSE_ASYM;
2847
2848                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2849
2850                         current_link_up = 1;
2851                 }
2852                 for (i = 0; i < 30; i++) {
2853                         udelay(20);
2854                         tw32_f(MAC_STATUS,
2855                                (MAC_STATUS_SYNC_CHANGED |
2856                                 MAC_STATUS_CFG_CHANGED));
2857                         udelay(40);
2858                         if ((tr32(MAC_STATUS) &
2859                              (MAC_STATUS_SYNC_CHANGED |
2860                               MAC_STATUS_CFG_CHANGED)) == 0)
2861                                 break;
2862                 }
2863
2864                 mac_status = tr32(MAC_STATUS);
2865                 if (current_link_up == 0 &&
2866                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2867                     !(mac_status & MAC_STATUS_RCVD_CFG))
2868                         current_link_up = 1;
2869         } else {
2870                 tg3_setup_flow_control(tp, 0, 0);
2871
2872                 /* Forcing 1000FD link up. */
2873                 current_link_up = 1;
2874
2875                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2876                 udelay(40);
2877
2878                 tw32_f(MAC_MODE, tp->mac_mode);
2879                 udelay(40);
2880         }
2881
2882 out:
2883         return current_link_up;
2884 }
2885
2886 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2887 {
2888         u32 orig_pause_cfg;
2889         u16 orig_active_speed;
2890         u8 orig_active_duplex;
2891         u32 mac_status;
2892         int current_link_up;
2893         int i;
2894
2895         orig_pause_cfg = tp->link_config.active_flowctrl;
2896         orig_active_speed = tp->link_config.active_speed;
2897         orig_active_duplex = tp->link_config.active_duplex;
2898
2899         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2900             netif_carrier_ok(tp->dev) &&
2901             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2902                 mac_status = tr32(MAC_STATUS);
2903                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2904                                MAC_STATUS_SIGNAL_DET |
2905                                MAC_STATUS_CFG_CHANGED |
2906                                MAC_STATUS_RCVD_CFG);
2907                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2908                                    MAC_STATUS_SIGNAL_DET)) {
2909                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2910                                             MAC_STATUS_CFG_CHANGED));
2911                         return 0;
2912                 }
2913         }
2914
2915         tw32_f(MAC_TX_AUTO_NEG, 0);
2916
2917         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2918         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2919         tw32_f(MAC_MODE, tp->mac_mode);
2920         udelay(40);
2921
2922         if (tp->phy_id == PHY_ID_BCM8002)
2923                 tg3_init_bcm8002(tp);
2924
2925         /* Enable link change event even when serdes polling.  */
2926         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2927         udelay(40);
2928
2929         current_link_up = 0;
2930         mac_status = tr32(MAC_STATUS);
2931
2932         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2933                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2934         else
2935                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2936
2937         tp->hw_status->status =
2938                 (SD_STATUS_UPDATED |
2939                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2940
2941         for (i = 0; i < 100; i++) {
2942                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2943                                     MAC_STATUS_CFG_CHANGED));
2944                 udelay(5);
2945                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2946                                          MAC_STATUS_CFG_CHANGED |
2947                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
2948                         break;
2949         }
2950
2951         mac_status = tr32(MAC_STATUS);
2952         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2953                 current_link_up = 0;
2954                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2955                     tp->serdes_counter == 0) {
2956                         tw32_f(MAC_MODE, (tp->mac_mode |
2957                                           MAC_MODE_SEND_CONFIGS));
2958                         udelay(1);
2959                         tw32_f(MAC_MODE, tp->mac_mode);
2960                 }
2961         }
2962
2963         if (current_link_up == 1) {
2964                 tp->link_config.active_speed = SPEED_1000;
2965                 tp->link_config.active_duplex = DUPLEX_FULL;
2966                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2967                                     LED_CTRL_LNKLED_OVERRIDE |
2968                                     LED_CTRL_1000MBPS_ON));
2969         } else {
2970                 tp->link_config.active_speed = SPEED_INVALID;
2971                 tp->link_config.active_duplex = DUPLEX_INVALID;
2972                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2973                                     LED_CTRL_LNKLED_OVERRIDE |
2974                                     LED_CTRL_TRAFFIC_OVERRIDE));
2975         }
2976
2977         if (current_link_up != netif_carrier_ok(tp->dev)) {
2978                 if (current_link_up)
2979                         netif_carrier_on(tp->dev);
2980                 else
2981                         netif_carrier_off(tp->dev);
2982                 tg3_link_report(tp);
2983         } else {
2984                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
2985                 if (orig_pause_cfg != now_pause_cfg ||
2986                     orig_active_speed != tp->link_config.active_speed ||
2987                     orig_active_duplex != tp->link_config.active_duplex)
2988                         tg3_link_report(tp);
2989         }
2990
2991         return 0;
2992 }
2993
2994 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2995 {
2996         int current_link_up, err = 0;
2997         u32 bmsr, bmcr;
2998         u16 current_speed;
2999         u8 current_duplex;
3000         u32 local_adv, remote_adv;
3001
3002         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3003         tw32_f(MAC_MODE, tp->mac_mode);
3004         udelay(40);
3005
3006         tw32(MAC_EVENT, 0);
3007
3008         tw32_f(MAC_STATUS,
3009              (MAC_STATUS_SYNC_CHANGED |
3010               MAC_STATUS_CFG_CHANGED |
3011               MAC_STATUS_MI_COMPLETION |
3012               MAC_STATUS_LNKSTATE_CHANGED));
3013         udelay(40);
3014
3015         if (force_reset)
3016                 tg3_phy_reset(tp);
3017
3018         current_link_up = 0;
3019         current_speed = SPEED_INVALID;
3020         current_duplex = DUPLEX_INVALID;
3021
3022         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3023         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3024         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
3025                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3026                         bmsr |= BMSR_LSTATUS;
3027                 else
3028                         bmsr &= ~BMSR_LSTATUS;
3029         }
3030
3031         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
3032
3033         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
3034             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
3035              tp->link_config.flowctrl == tp->link_config.active_flowctrl) {
3036                 /* do nothing, just check for link up at the end */
3037         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3038                 u32 adv, new_adv;
3039
3040                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3041                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
3042                                   ADVERTISE_1000XPAUSE |
3043                                   ADVERTISE_1000XPSE_ASYM |
3044                                   ADVERTISE_SLCT);
3045
3046                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3047
3048                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
3049                         new_adv |= ADVERTISE_1000XHALF;
3050                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
3051                         new_adv |= ADVERTISE_1000XFULL;
3052
3053                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
3054                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
3055                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
3056                         tg3_writephy(tp, MII_BMCR, bmcr);
3057
3058                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3059                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
3060                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3061
3062                         return err;
3063                 }
3064         } else {
3065                 u32 new_bmcr;
3066
3067                 bmcr &= ~BMCR_SPEED1000;
3068                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
3069
3070                 if (tp->link_config.duplex == DUPLEX_FULL)
3071                         new_bmcr |= BMCR_FULLDPLX;
3072
3073                 if (new_bmcr != bmcr) {
3074                         /* BMCR_SPEED1000 is a reserved bit that needs
3075                          * to be set on write.
3076                          */
3077                         new_bmcr |= BMCR_SPEED1000;
3078
3079                         /* Force a linkdown */
3080                         if (netif_carrier_ok(tp->dev)) {
3081                                 u32 adv;
3082
3083                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3084                                 adv &= ~(ADVERTISE_1000XFULL |
3085                                          ADVERTISE_1000XHALF |
3086                                          ADVERTISE_SLCT);
3087                                 tg3_writephy(tp, MII_ADVERTISE, adv);
3088                                 tg3_writephy(tp, MII_BMCR, bmcr |
3089                                                            BMCR_ANRESTART |
3090                                                            BMCR_ANENABLE);
3091                                 udelay(10);
3092                                 netif_carrier_off(tp->dev);
3093                         }
3094                         tg3_writephy(tp, MII_BMCR, new_bmcr);
3095                         bmcr = new_bmcr;
3096                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3097                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3098                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
3099                             ASIC_REV_5714) {
3100                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3101                                         bmsr |= BMSR_LSTATUS;
3102                                 else
3103                                         bmsr &= ~BMSR_LSTATUS;
3104                         }
3105                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3106                 }
3107         }
3108
3109         if (bmsr & BMSR_LSTATUS) {
3110                 current_speed = SPEED_1000;
3111                 current_link_up = 1;
3112                 if (bmcr & BMCR_FULLDPLX)
3113                         current_duplex = DUPLEX_FULL;
3114                 else
3115                         current_duplex = DUPLEX_HALF;
3116
3117                 local_adv = 0;
3118                 remote_adv = 0;
3119
3120                 if (bmcr & BMCR_ANENABLE) {
3121                         u32 common;
3122
3123                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
3124                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
3125                         common = local_adv & remote_adv;
3126                         if (common & (ADVERTISE_1000XHALF |
3127                                       ADVERTISE_1000XFULL)) {
3128                                 if (common & ADVERTISE_1000XFULL)
3129                                         current_duplex = DUPLEX_FULL;
3130                                 else
3131                                         current_duplex = DUPLEX_HALF;
3132                         }
3133                         else
3134                                 current_link_up = 0;
3135                 }
3136         }
3137
3138         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
3139                 tg3_setup_flow_control(tp, local_adv, remote_adv);
3140
3141         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3142         if (tp->link_config.active_duplex == DUPLEX_HALF)
3143                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3144
3145         tw32_f(MAC_MODE, tp->mac_mode);
3146         udelay(40);
3147
3148         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3149
3150         tp->link_config.active_speed = current_speed;
3151         tp->link_config.active_duplex = current_duplex;
3152
3153         if (current_link_up != netif_carrier_ok(tp->dev)) {
3154                 if (current_link_up)
3155                         netif_carrier_on(tp->dev);
3156                 else {
3157                         netif_carrier_off(tp->dev);
3158                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3159                 }
3160                 tg3_link_report(tp);
3161         }
3162         return err;
3163 }
3164
3165 static void tg3_serdes_parallel_detect(struct tg3 *tp)
3166 {
3167         if (tp->serdes_counter) {
3168                 /* Give autoneg time to complete. */
3169                 tp->serdes_counter--;
3170                 return;
3171         }
3172         if (!netif_carrier_ok(tp->dev) &&
3173             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
3174                 u32 bmcr;
3175
3176                 tg3_readphy(tp, MII_BMCR, &bmcr);
3177                 if (bmcr & BMCR_ANENABLE) {
3178                         u32 phy1, phy2;
3179
3180                         /* Select shadow register 0x1f */
3181                         tg3_writephy(tp, 0x1c, 0x7c00);
3182                         tg3_readphy(tp, 0x1c, &phy1);
3183
3184                         /* Select expansion interrupt status register */
3185                         tg3_writephy(tp, 0x17, 0x0f01);
3186                         tg3_readphy(tp, 0x15, &phy2);
3187                         tg3_readphy(tp, 0x15, &phy2);
3188
3189                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
3190                                 /* We have signal detect and not receiving
3191                                  * config code words, link is up by parallel
3192                                  * detection.
3193                                  */
3194
3195                                 bmcr &= ~BMCR_ANENABLE;
3196                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
3197                                 tg3_writephy(tp, MII_BMCR, bmcr);
3198                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
3199                         }
3200                 }
3201         }
3202         else if (netif_carrier_ok(tp->dev) &&
3203                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
3204                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
3205                 u32 phy2;
3206
3207                 /* Select expansion interrupt status register */
3208                 tg3_writephy(tp, 0x17, 0x0f01);
3209                 tg3_readphy(tp, 0x15, &phy2);
3210                 if (phy2 & 0x20) {
3211                         u32 bmcr;
3212
3213                         /* Config code words received, turn on autoneg. */
3214                         tg3_readphy(tp, MII_BMCR, &bmcr);
3215                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
3216
3217                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3218
3219                 }
3220         }
3221 }
3222
3223 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
3224 {
3225         int err;
3226
3227         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
3228                 err = tg3_setup_fiber_phy(tp, force_reset);
3229         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
3230                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
3231         } else {
3232                 err = tg3_setup_copper_phy(tp, force_reset);
3233         }
3234
3235         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
3236             tp->pci_chip_rev_id == CHIPREV_ID_5784_A1) {
3237                 u32 val, scale;
3238
3239                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
3240                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
3241                         scale = 65;
3242                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
3243                         scale = 6;
3244                 else
3245                         scale = 12;
3246
3247                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
3248                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
3249                 tw32(GRC_MISC_CFG, val);
3250         }
3251
3252         if (tp->link_config.active_speed == SPEED_1000 &&
3253             tp->link_config.active_duplex == DUPLEX_HALF)
3254                 tw32(MAC_TX_LENGTHS,
3255                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3256                       (6 << TX_LENGTHS_IPG_SHIFT) |
3257                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
3258         else
3259                 tw32(MAC_TX_LENGTHS,
3260                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3261                       (6 << TX_LENGTHS_IPG_SHIFT) |
3262                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
3263
3264         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
3265                 if (netif_carrier_ok(tp->dev)) {
3266                         tw32(HOSTCC_STAT_COAL_TICKS,
3267                              tp->coal.stats_block_coalesce_usecs);
3268                 } else {
3269                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
3270                 }
3271         }
3272
3273         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
3274                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
3275                 if (!netif_carrier_ok(tp->dev))
3276                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
3277                               tp->pwrmgmt_thresh;
3278                 else
3279                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
3280                 tw32(PCIE_PWR_MGMT_THRESH, val);
3281         }
3282
3283         return err;
3284 }
3285
3286 /* This is called whenever we suspect that the system chipset is re-
3287  * ordering the sequence of MMIO to the tx send mailbox. The symptom
3288  * is bogus tx completions. We try to recover by setting the
3289  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
3290  * in the workqueue.
3291  */
3292 static void tg3_tx_recover(struct tg3 *tp)
3293 {
3294         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
3295                tp->write32_tx_mbox == tg3_write_indirect_mbox);
3296
3297         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
3298                "mapped I/O cycles to the network device, attempting to "
3299                "recover. Please report the problem to the driver maintainer "
3300                "and include system chipset information.\n", tp->dev->name);
3301
3302         spin_lock(&tp->lock);
3303         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
3304         spin_unlock(&tp->lock);
3305 }
3306
3307 static inline u32 tg3_tx_avail(struct tg3 *tp)
3308 {
3309         smp_mb();
3310         return (tp->tx_pending -
3311                 ((tp->tx_prod - tp->tx_cons) & (TG3_TX_RING_SIZE - 1)));
3312 }
3313
3314 /* Tigon3 never reports partial packet sends.  So we do not
3315  * need special logic to handle SKBs that have not had all
3316  * of their frags sent yet, like SunGEM does.
3317  */
3318 static void tg3_tx(struct tg3 *tp)
3319 {
3320         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
3321         u32 sw_idx = tp->tx_cons;
3322
3323         while (sw_idx != hw_idx) {
3324                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3325                 struct sk_buff *skb = ri->skb;
3326                 int i, tx_bug = 0;
3327
3328                 if (unlikely(skb == NULL)) {
3329                         tg3_tx_recover(tp);
3330                         return;
3331                 }
3332
3333                 pci_unmap_single(tp->pdev,
3334                                  pci_unmap_addr(ri, mapping),
3335                                  skb_headlen(skb),
3336                                  PCI_DMA_TODEVICE);
3337
3338                 ri->skb = NULL;
3339
3340                 sw_idx = NEXT_TX(sw_idx);
3341
3342                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3343                         ri = &tp->tx_buffers[sw_idx];
3344                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3345                                 tx_bug = 1;
3346
3347                         pci_unmap_page(tp->pdev,
3348                                        pci_unmap_addr(ri, mapping),
3349                                        skb_shinfo(skb)->frags[i].size,
3350                                        PCI_DMA_TODEVICE);
3351
3352                         sw_idx = NEXT_TX(sw_idx);
3353                 }
3354
3355                 dev_kfree_skb(skb);
3356
3357                 if (unlikely(tx_bug)) {
3358                         tg3_tx_recover(tp);
3359                         return;
3360                 }
3361         }
3362
3363         tp->tx_cons = sw_idx;
3364
3365         /* Need to make the tx_cons update visible to tg3_start_xmit()
3366          * before checking for netif_queue_stopped().  Without the
3367          * memory barrier, there is a small possibility that tg3_start_xmit()
3368          * will miss it and cause the queue to be stopped forever.
3369          */
3370         smp_mb();
3371
3372         if (unlikely(netif_queue_stopped(tp->dev) &&
3373                      (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))) {
3374                 netif_tx_lock(tp->dev);
3375                 if (netif_queue_stopped(tp->dev) &&
3376                     (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))
3377                         netif_wake_queue(tp->dev);
3378                 netif_tx_unlock(tp->dev);
3379         }
3380 }
3381
3382 /* Returns size of skb allocated or < 0 on error.
3383  *
3384  * We only need to fill in the address because the other members
3385  * of the RX descriptor are invariant, see tg3_init_rings.
3386  *
3387  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3388  * posting buffers we only dirty the first cache line of the RX
3389  * descriptor (containing the address).  Whereas for the RX status
3390  * buffers the cpu only reads the last cacheline of the RX descriptor
3391  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3392  */
3393 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3394                             int src_idx, u32 dest_idx_unmasked)
3395 {
3396         struct tg3_rx_buffer_desc *desc;
3397         struct ring_info *map, *src_map;
3398         struct sk_buff *skb;
3399         dma_addr_t mapping;
3400         int skb_size, dest_idx;
3401
3402         src_map = NULL;
3403         switch (opaque_key) {
3404         case RXD_OPAQUE_RING_STD:
3405                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3406                 desc = &tp->rx_std[dest_idx];
3407                 map = &tp->rx_std_buffers[dest_idx];
3408                 if (src_idx >= 0)
3409                         src_map = &tp->rx_std_buffers[src_idx];
3410                 skb_size = tp->rx_pkt_buf_sz;
3411                 break;
3412
3413         case RXD_OPAQUE_RING_JUMBO:
3414                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3415                 desc = &tp->rx_jumbo[dest_idx];
3416                 map = &tp->rx_jumbo_buffers[dest_idx];
3417                 if (src_idx >= 0)
3418                         src_map = &tp->rx_jumbo_buffers[src_idx];
3419                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3420                 break;
3421
3422         default:
3423                 return -EINVAL;
3424         };
3425
3426         /* Do not overwrite any of the map or rp information
3427          * until we are sure we can commit to a new buffer.
3428          *
3429          * Callers depend upon this behavior and assume that
3430          * we leave everything unchanged if we fail.
3431          */
3432         skb = netdev_alloc_skb(tp->dev, skb_size);
3433         if (skb == NULL)
3434                 return -ENOMEM;
3435
3436         skb_reserve(skb, tp->rx_offset);
3437
3438         mapping = pci_map_single(tp->pdev, skb->data,
3439                                  skb_size - tp->rx_offset,
3440                                  PCI_DMA_FROMDEVICE);
3441
3442         map->skb = skb;
3443         pci_unmap_addr_set(map, mapping, mapping);
3444
3445         if (src_map != NULL)
3446                 src_map->skb = NULL;
3447
3448         desc->addr_hi = ((u64)mapping >> 32);
3449         desc->addr_lo = ((u64)mapping & 0xffffffff);
3450
3451         return skb_size;
3452 }
3453
3454 /* We only need to move over in the address because the other
3455  * members of the RX descriptor are invariant.  See notes above
3456  * tg3_alloc_rx_skb for full details.
3457  */
3458 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3459                            int src_idx, u32 dest_idx_unmasked)
3460 {
3461         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3462         struct ring_info *src_map, *dest_map;
3463         int dest_idx;
3464
3465         switch (opaque_key) {
3466         case RXD_OPAQUE_RING_STD:
3467                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3468                 dest_desc = &tp->rx_std[dest_idx];
3469                 dest_map = &tp->rx_std_buffers[dest_idx];
3470                 src_desc = &tp->rx_std[src_idx];
3471                 src_map = &tp->rx_std_buffers[src_idx];
3472                 break;
3473
3474         case RXD_OPAQUE_RING_JUMBO:
3475                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3476                 dest_desc = &tp->rx_jumbo[dest_idx];
3477                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3478                 src_desc = &tp->rx_jumbo[src_idx];
3479                 src_map = &tp->rx_jumbo_buffers[src_idx];
3480                 break;
3481
3482         default:
3483                 return;
3484         };
3485
3486         dest_map->skb = src_map->skb;
3487         pci_unmap_addr_set(dest_map, mapping,
3488                            pci_unmap_addr(src_map, mapping));
3489         dest_desc->addr_hi = src_desc->addr_hi;
3490         dest_desc->addr_lo = src_desc->addr_lo;
3491
3492         src_map->skb = NULL;
3493 }
3494
3495 #if TG3_VLAN_TAG_USED
3496 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3497 {
3498         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3499 }
3500 #endif
3501
3502 /* The RX ring scheme is composed of multiple rings which post fresh
3503  * buffers to the chip, and one special ring the chip uses to report
3504  * status back to the host.
3505  *
3506  * The special ring reports the status of received packets to the
3507  * host.  The chip does not write into the original descriptor the
3508  * RX buffer was obtained from.  The chip simply takes the original
3509  * descriptor as provided by the host, updates the status and length
3510  * field, then writes this into the next status ring entry.
3511  *
3512  * Each ring the host uses to post buffers to the chip is described
3513  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3514  * it is first placed into the on-chip ram.  When the packet's length
3515  * is known, it walks down the TG3_BDINFO entries to select the ring.
3516  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3517  * which is within the range of the new packet's length is chosen.
3518  *
3519  * The "separate ring for rx status" scheme may sound queer, but it makes
3520  * sense from a cache coherency perspective.  If only the host writes
3521  * to the buffer post rings, and only the chip writes to the rx status
3522  * rings, then cache lines never move beyond shared-modified state.
3523  * If both the host and chip were to write into the same ring, cache line
3524  * eviction could occur since both entities want it in an exclusive state.
3525  */
3526 static int tg3_rx(struct tg3 *tp, int budget)
3527 {
3528         u32 work_mask, rx_std_posted = 0;
3529         u32 sw_idx = tp->rx_rcb_ptr;
3530         u16 hw_idx;
3531         int received;
3532
3533         hw_idx = tp->hw_status->idx[0].rx_producer;
3534         /*
3535          * We need to order the read of hw_idx and the read of
3536          * the opaque cookie.
3537          */
3538         rmb();
3539         work_mask = 0;
3540         received = 0;
3541         while (sw_idx != hw_idx && budget > 0) {
3542                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3543                 unsigned int len;
3544                 struct sk_buff *skb;
3545                 dma_addr_t dma_addr;
3546                 u32 opaque_key, desc_idx, *post_ptr;
3547
3548                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3549                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3550                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3551                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3552                                                   mapping);
3553                         skb = tp->rx_std_buffers[desc_idx].skb;
3554                         post_ptr = &tp->rx_std_ptr;
3555                         rx_std_posted++;
3556                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3557                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3558                                                   mapping);
3559                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3560                         post_ptr = &tp->rx_jumbo_ptr;
3561                 }
3562                 else {
3563                         goto next_pkt_nopost;
3564                 }
3565
3566                 work_mask |= opaque_key;
3567
3568                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3569                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3570                 drop_it:
3571                         tg3_recycle_rx(tp, opaque_key,
3572                                        desc_idx, *post_ptr);
3573                 drop_it_no_recycle:
3574                         /* Other statistics kept track of by card. */
3575                         tp->net_stats.rx_dropped++;
3576                         goto next_pkt;
3577                 }
3578
3579                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3580
3581                 if (len > RX_COPY_THRESHOLD
3582                         && tp->rx_offset == 2
3583                         /* rx_offset != 2 iff this is a 5701 card running
3584                          * in PCI-X mode [see tg3_get_invariants()] */
3585                 ) {
3586                         int skb_size;
3587
3588                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3589                                                     desc_idx, *post_ptr);
3590                         if (skb_size < 0)
3591                                 goto drop_it;
3592
3593                         pci_unmap_single(tp->pdev, dma_addr,
3594                                          skb_size - tp->rx_offset,
3595                                          PCI_DMA_FROMDEVICE);
3596
3597                         skb_put(skb, len);
3598                 } else {
3599                         struct sk_buff *copy_skb;
3600
3601                         tg3_recycle_rx(tp, opaque_key,
3602                                        desc_idx, *post_ptr);
3603
3604                         copy_skb = netdev_alloc_skb(tp->dev, len + 2);
3605                         if (copy_skb == NULL)
3606                                 goto drop_it_no_recycle;
3607
3608                         skb_reserve(copy_skb, 2);
3609                         skb_put(copy_skb, len);
3610                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3611                         skb_copy_from_linear_data(skb, copy_skb->data, len);
3612                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3613
3614                         /* We'll reuse the original ring buffer. */
3615                         skb = copy_skb;
3616                 }
3617
3618                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3619                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3620                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3621                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3622                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3623                 else
3624                         skb->ip_summed = CHECKSUM_NONE;
3625
3626                 skb->protocol = eth_type_trans(skb, tp->dev);
3627 #if TG3_VLAN_TAG_USED
3628                 if (tp->vlgrp != NULL &&
3629                     desc->type_flags & RXD_FLAG_VLAN) {
3630                         tg3_vlan_rx(tp, skb,
3631                                     desc->err_vlan & RXD_VLAN_MASK);
3632                 } else
3633 #endif
3634                         netif_receive_skb(skb);
3635
3636                 tp->dev->last_rx = jiffies;
3637                 received++;
3638                 budget--;
3639
3640 next_pkt:
3641                 (*post_ptr)++;
3642
3643                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
3644                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
3645
3646                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
3647                                      TG3_64BIT_REG_LOW, idx);
3648                         work_mask &= ~RXD_OPAQUE_RING_STD;
3649                         rx_std_posted = 0;
3650                 }
3651 next_pkt_nopost:
3652                 sw_idx++;
3653                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
3654
3655                 /* Refresh hw_idx to see if there is new work */
3656                 if (sw_idx == hw_idx) {
3657                         hw_idx = tp->hw_status->idx[0].rx_producer;
3658                         rmb();
3659                 }
3660         }
3661
3662         /* ACK the status ring. */
3663         tp->rx_rcb_ptr = sw_idx;
3664         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3665
3666         /* Refill RX ring(s). */
3667         if (work_mask & RXD_OPAQUE_RING_STD) {
3668                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3669                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3670                              sw_idx);
3671         }
3672         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3673                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3674                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3675                              sw_idx);
3676         }
3677         mmiowb();
3678
3679         return received;
3680 }
3681
3682 static int tg3_poll_work(struct tg3 *tp, int work_done, int budget)
3683 {
3684         struct tg3_hw_status *sblk = tp->hw_status;
3685
3686         /* handle link change and other phy events */
3687         if (!(tp->tg3_flags &
3688               (TG3_FLAG_USE_LINKCHG_REG |
3689                TG3_FLAG_POLL_SERDES))) {
3690                 if (sblk->status & SD_STATUS_LINK_CHG) {
3691                         sblk->status = SD_STATUS_UPDATED |
3692                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3693                         spin_lock(&tp->lock);
3694                         tg3_setup_phy(tp, 0);
3695                         spin_unlock(&tp->lock);
3696                 }
3697         }
3698
3699         /* run TX completion thread */
3700         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3701                 tg3_tx(tp);
3702                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3703                         return work_done;
3704         }
3705
3706         /* run RX thread, within the bounds set by NAPI.
3707          * All RX "locking" is done by ensuring outside
3708          * code synchronizes with tg3->napi.poll()
3709          */
3710         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
3711                 work_done += tg3_rx(tp, budget - work_done);
3712
3713         return work_done;
3714 }
3715
3716 static int tg3_poll(struct napi_struct *napi, int budget)
3717 {
3718         struct tg3 *tp = container_of(napi, struct tg3, napi);
3719         int work_done = 0;
3720         struct tg3_hw_status *sblk = tp->hw_status;
3721
3722         while (1) {
3723                 work_done = tg3_poll_work(tp, work_done, budget);
3724
3725                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3726                         goto tx_recovery;
3727
3728                 if (unlikely(work_done >= budget))
3729                         break;
3730
3731                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3732                         /* tp->last_tag is used in tg3_restart_ints() below
3733                          * to tell the hw how much work has been processed,
3734                          * so we must read it before checking for more work.
3735                          */
3736                         tp->last_tag = sblk->status_tag;
3737                         rmb();
3738                 } else
3739                         sblk->status &= ~SD_STATUS_UPDATED;
3740
3741                 if (likely(!tg3_has_work(tp))) {
3742                         netif_rx_complete(tp->dev, napi);
3743                         tg3_restart_ints(tp);
3744                         break;
3745                 }
3746         }
3747
3748         return work_done;
3749
3750 tx_recovery:
3751         /* work_done is guaranteed to be less than budget. */
3752         netif_rx_complete(tp->dev, napi);
3753         schedule_work(&tp->reset_task);
3754         return work_done;
3755 }
3756
3757 static void tg3_irq_quiesce(struct tg3 *tp)
3758 {
3759         BUG_ON(tp->irq_sync);
3760
3761         tp->irq_sync = 1;
3762         smp_mb();
3763
3764         synchronize_irq(tp->pdev->irq);
3765 }
3766
3767 static inline int tg3_irq_sync(struct tg3 *tp)
3768 {
3769         return tp->irq_sync;
3770 }
3771
3772 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3773  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3774  * with as well.  Most of the time, this is not necessary except when
3775  * shutting down the device.
3776  */
3777 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3778 {
3779         spin_lock_bh(&tp->lock);
3780         if (irq_sync)
3781                 tg3_irq_quiesce(tp);
3782 }
3783
3784 static inline void tg3_full_unlock(struct tg3 *tp)
3785 {
3786         spin_unlock_bh(&tp->lock);
3787 }
3788
3789 /* One-shot MSI handler - Chip automatically disables interrupt
3790  * after sending MSI so driver doesn't have to do it.
3791  */
3792 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
3793 {
3794         struct net_device *dev = dev_id;
3795         struct tg3 *tp = netdev_priv(dev);
3796
3797         prefetch(tp->hw_status);
3798         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3799
3800         if (likely(!tg3_irq_sync(tp)))
3801                 netif_rx_schedule(dev, &tp->napi);
3802
3803         return IRQ_HANDLED;
3804 }
3805
3806 /* MSI ISR - No need to check for interrupt sharing and no need to
3807  * flush status block and interrupt mailbox. PCI ordering rules
3808  * guarantee that MSI will arrive after the status block.
3809  */
3810 static irqreturn_t tg3_msi(int irq, void *dev_id)
3811 {
3812         struct net_device *dev = dev_id;
3813         struct tg3 *tp = netdev_priv(dev);
3814
3815         prefetch(tp->hw_status);
3816         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3817         /*
3818          * Writing any value to intr-mbox-0 clears PCI INTA# and
3819          * chip-internal interrupt pending events.
3820          * Writing non-zero to intr-mbox-0 additional tells the
3821          * NIC to stop sending us irqs, engaging "in-intr-handler"
3822          * event coalescing.
3823          */
3824         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3825         if (likely(!tg3_irq_sync(tp)))
3826                 netif_rx_schedule(dev, &tp->napi);
3827
3828         return IRQ_RETVAL(1);
3829 }
3830
3831 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
3832 {
3833         struct net_device *dev = dev_id;
3834         struct tg3 *tp = netdev_priv(dev);
3835         struct tg3_hw_status *sblk = tp->hw_status;
3836         unsigned int handled = 1;
3837
3838         /* In INTx mode, it is possible for the interrupt to arrive at
3839          * the CPU before the status block posted prior to the interrupt.
3840          * Reading the PCI State register will confirm whether the
3841          * interrupt is ours and will flush the status block.
3842          */
3843         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
3844                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3845                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3846                         handled = 0;
3847                         goto out;
3848                 }
3849         }
3850
3851         /*
3852          * Writing any value to intr-mbox-0 clears PCI INTA# and
3853          * chip-internal interrupt pending events.
3854          * Writing non-zero to intr-mbox-0 additional tells the
3855          * NIC to stop sending us irqs, engaging "in-intr-handler"
3856          * event coalescing.
3857          *
3858          * Flush the mailbox to de-assert the IRQ immediately to prevent
3859          * spurious interrupts.  The flush impacts performance but
3860          * excessive spurious interrupts can be worse in some cases.
3861          */
3862         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3863         if (tg3_irq_sync(tp))
3864                 goto out;
3865         sblk->status &= ~SD_STATUS_UPDATED;
3866         if (likely(tg3_has_work(tp))) {
3867                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3868                 netif_rx_schedule(dev, &tp->napi);
3869         } else {
3870                 /* No work, shared interrupt perhaps?  re-enable
3871                  * interrupts, and flush that PCI write
3872                  */
3873                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3874                                0x00000000);
3875         }
3876 out:
3877         return IRQ_RETVAL(handled);
3878 }
3879
3880 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
3881 {
3882         struct net_device *dev = dev_id;
3883         struct tg3 *tp = netdev_priv(dev);
3884         struct tg3_hw_status *sblk = tp->hw_status;
3885         unsigned int handled = 1;
3886
3887         /* In INTx mode, it is possible for the interrupt to arrive at
3888          * the CPU before the status block posted prior to the interrupt.
3889          * Reading the PCI State register will confirm whether the
3890          * interrupt is ours and will flush the status block.
3891          */
3892         if (unlikely(sblk->status_tag == tp->last_tag)) {
3893                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3894                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3895                         handled = 0;
3896                         goto out;
3897                 }
3898         }
3899
3900         /*
3901          * writing any value to intr-mbox-0 clears PCI INTA# and
3902          * chip-internal interrupt pending events.
3903          * writing non-zero to intr-mbox-0 additional tells the
3904          * NIC to stop sending us irqs, engaging "in-intr-handler"
3905          * event coalescing.
3906          *
3907          * Flush the mailbox to de-assert the IRQ immediately to prevent
3908          * spurious interrupts.  The flush impacts performance but
3909          * excessive spurious interrupts can be worse in some cases.
3910          */
3911         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3912         if (tg3_irq_sync(tp))
3913                 goto out;
3914         if (netif_rx_schedule_prep(dev, &tp->napi)) {
3915                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3916                 /* Update last_tag to mark that this status has been
3917                  * seen. Because interrupt may be shared, we may be
3918                  * racing with tg3_poll(), so only update last_tag
3919                  * if tg3_poll() is not scheduled.
3920                  */
3921                 tp->last_tag = sblk->status_tag;
3922                 __netif_rx_schedule(dev, &tp->napi);
3923         }
3924 out:
3925         return IRQ_RETVAL(handled);
3926 }
3927
3928 /* ISR for interrupt test */
3929 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
3930 {
3931         struct net_device *dev = dev_id;
3932         struct tg3 *tp = netdev_priv(dev);
3933         struct tg3_hw_status *sblk = tp->hw_status;
3934
3935         if ((sblk->status & SD_STATUS_UPDATED) ||
3936             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3937                 tg3_disable_ints(tp);
3938                 return IRQ_RETVAL(1);
3939         }
3940         return IRQ_RETVAL(0);
3941 }
3942
3943 static int tg3_init_hw(struct tg3 *, int);
3944 static int tg3_halt(struct tg3 *, int, int);
3945
3946 /* Restart hardware after configuration changes, self-test, etc.
3947  * Invoked with tp->lock held.
3948  */
3949 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
3950 {
3951         int err;
3952
3953         err = tg3_init_hw(tp, reset_phy);
3954         if (err) {
3955                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
3956                        "aborting.\n", tp->dev->name);
3957                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
3958                 tg3_full_unlock(tp);
3959                 del_timer_sync(&tp->timer);
3960                 tp->irq_sync = 0;
3961                 napi_enable(&tp->napi);
3962                 dev_close(tp->dev);
3963                 tg3_full_lock(tp, 0);
3964         }
3965         return err;
3966 }
3967
3968 #ifdef CONFIG_NET_POLL_CONTROLLER
3969 static void tg3_poll_controller(struct net_device *dev)
3970 {
3971         struct tg3 *tp = netdev_priv(dev);
3972
3973         tg3_interrupt(tp->pdev->irq, dev);
3974 }
3975 #endif
3976
3977 static void tg3_reset_task(struct work_struct *work)
3978 {
3979         struct tg3 *tp = container_of(work, struct tg3, reset_task);
3980         unsigned int restart_timer;
3981
3982         tg3_full_lock(tp, 0);
3983
3984         if (!netif_running(tp->dev)) {
3985                 tg3_full_unlock(tp);
3986                 return;
3987         }
3988
3989         tg3_full_unlock(tp);
3990
3991         tg3_netif_stop(tp);
3992
3993         tg3_full_lock(tp, 1);
3994
3995         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3996         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3997
3998         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3999                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
4000                 tp->write32_rx_mbox = tg3_write_flush_reg32;
4001                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
4002                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
4003         }
4004
4005         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
4006         if (tg3_init_hw(tp, 1))
4007                 goto out;
4008
4009         tg3_netif_start(tp);
4010
4011         if (restart_timer)
4012                 mod_timer(&tp->timer, jiffies + 1);
4013
4014 out:
4015         tg3_full_unlock(tp);
4016 }
4017
4018 static void tg3_dump_short_state(struct tg3 *tp)
4019 {
4020         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
4021                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
4022         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
4023                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
4024 }
4025
4026 static void tg3_tx_timeout(struct net_device *dev)
4027 {
4028         struct tg3 *tp = netdev_priv(dev);
4029
4030         if (netif_msg_tx_err(tp)) {
4031                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
4032                        dev->name);
4033                 tg3_dump_short_state(tp);
4034         }
4035
4036         schedule_work(&tp->reset_task);
4037 }
4038
4039 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
4040 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
4041 {
4042         u32 base = (u32) mapping & 0xffffffff;
4043
4044         return ((base > 0xffffdcc0) &&
4045                 (base + len + 8 < base));
4046 }
4047
4048 /* Test for DMA addresses > 40-bit */
4049 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
4050                                           int len)
4051 {
4052 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
4053         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
4054                 return (((u64) mapping + len) > DMA_40BIT_MASK);
4055         return 0;
4056 #else
4057         return 0;
4058 #endif
4059 }
4060
4061 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
4062
4063 /* Workaround 4GB and 40-bit hardware DMA bugs. */
4064 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
4065                                        u32 last_plus_one, u32 *start,
4066                                        u32 base_flags, u32 mss)
4067 {
4068         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
4069         dma_addr_t new_addr = 0;
4070         u32 entry = *start;
4071         int i, ret = 0;
4072
4073         if (!new_skb) {
4074                 ret = -1;
4075         } else {
4076                 /* New SKB is guaranteed to be linear. */
4077                 entry = *start;
4078                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
4079                                           PCI_DMA_TODEVICE);
4080                 /* Make sure new skb does not cross any 4G boundaries.
4081                  * Drop the packet if it does.
4082                  */
4083                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
4084                         ret = -1;
4085                         dev_kfree_skb(new_skb);
4086                         new_skb = NULL;
4087                 } else {
4088                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
4089                                     base_flags, 1 | (mss << 1));
4090                         *start = NEXT_TX(entry);
4091                 }
4092         }
4093
4094         /* Now clean up the sw ring entries. */
4095         i = 0;
4096         while (entry != last_plus_one) {
4097                 int len;
4098
4099                 if (i == 0)
4100                         len = skb_headlen(skb);
4101                 else
4102                         len = skb_shinfo(skb)->frags[i-1].size;
4103                 pci_unmap_single(tp->pdev,
4104                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
4105                                  len, PCI_DMA_TODEVICE);
4106                 if (i == 0) {
4107                         tp->tx_buffers[entry].skb = new_skb;
4108                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
4109                 } else {
4110                         tp->tx_buffers[entry].skb = NULL;
4111                 }
4112                 entry = NEXT_TX(entry);
4113                 i++;
4114         }
4115
4116         dev_kfree_skb(skb);
4117
4118         return ret;
4119 }
4120
4121 static void tg3_set_txd(struct tg3 *tp, int entry,
4122                         dma_addr_t mapping, int len, u32 flags,
4123                         u32 mss_and_is_end)
4124 {
4125         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
4126         int is_end = (mss_and_is_end & 0x1);
4127         u32 mss = (mss_and_is_end >> 1);
4128         u32 vlan_tag = 0;
4129
4130         if (is_end)
4131                 flags |= TXD_FLAG_END;
4132         if (flags & TXD_FLAG_VLAN) {
4133                 vlan_tag = flags >> 16;
4134                 flags &= 0xffff;
4135         }
4136         vlan_tag |= (mss << TXD_MSS_SHIFT);
4137
4138         txd->addr_hi = ((u64) mapping >> 32);
4139         txd->addr_lo = ((u64) mapping & 0xffffffff);
4140         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
4141         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
4142 }
4143
4144 /* hard_start_xmit for devices that don't have any bugs and
4145  * support TG3_FLG2_HW_TSO_2 only.
4146  */
4147 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
4148 {
4149         struct tg3 *tp = netdev_priv(dev);
4150         dma_addr_t mapping;
4151         u32 len, entry, base_flags, mss;
4152
4153         len = skb_headlen(skb);
4154
4155         /* We are running in BH disabled context with netif_tx_lock
4156          * and TX reclaim runs via tp->napi.poll inside of a software
4157          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4158          * no IRQ context deadlocks to worry about either.  Rejoice!
4159          */
4160         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4161                 if (!netif_queue_stopped(dev)) {
4162                         netif_stop_queue(dev);
4163
4164                         /* This is a hard error, log it. */
4165                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4166                                "queue awake!\n", dev->name);
4167                 }
4168                 return NETDEV_TX_BUSY;
4169         }
4170
4171         entry = tp->tx_prod;
4172         base_flags = 0;
4173         mss = 0;
4174         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4175                 int tcp_opt_len, ip_tcp_len;
4176
4177                 if (skb_header_cloned(skb) &&
4178                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4179                         dev_kfree_skb(skb);
4180                         goto out_unlock;
4181                 }
4182
4183                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
4184                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
4185                 else {
4186                         struct iphdr *iph = ip_hdr(skb);
4187
4188                         tcp_opt_len = tcp_optlen(skb);
4189                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4190
4191                         iph->check = 0;
4192                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
4193                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
4194                 }
4195
4196                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4197                                TXD_FLAG_CPU_POST_DMA);
4198
4199                 tcp_hdr(skb)->check = 0;
4200
4201         }
4202         else if (skb->ip_summed == CHECKSUM_PARTIAL)
4203                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4204 #if TG3_VLAN_TAG_USED
4205         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4206                 base_flags |= (TXD_FLAG_VLAN |
4207                                (vlan_tx_tag_get(skb) << 16));
4208 #endif
4209
4210         /* Queue skb data, a.k.a. the main skb fragment. */
4211         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4212
4213         tp->tx_buffers[entry].skb = skb;
4214         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4215
4216         tg3_set_txd(tp, entry, mapping, len, base_flags,
4217                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4218
4219         entry = NEXT_TX(entry);
4220
4221         /* Now loop through additional data fragments, and queue them. */
4222         if (skb_shinfo(skb)->nr_frags > 0) {
4223                 unsigned int i, last;
4224
4225                 last = skb_shinfo(skb)->nr_frags - 1;
4226                 for (i = 0; i <= last; i++) {
4227                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4228
4229                         len = frag->size;
4230                         mapping = pci_map_page(tp->pdev,
4231                                                frag->page,
4232                                                frag->page_offset,
4233                                                len, PCI_DMA_TODEVICE);
4234
4235                         tp->tx_buffers[entry].skb = NULL;
4236                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4237
4238                         tg3_set_txd(tp, entry, mapping, len,
4239                                     base_flags, (i == last) | (mss << 1));
4240
4241                         entry = NEXT_TX(entry);
4242                 }
4243         }
4244
4245         /* Packets are ready, update Tx producer idx local and on card. */
4246         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4247
4248         tp->tx_prod = entry;
4249         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4250                 netif_stop_queue(dev);
4251                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4252                         netif_wake_queue(tp->dev);
4253         }
4254
4255 out_unlock:
4256         mmiowb();
4257
4258         dev->trans_start = jiffies;
4259
4260         return NETDEV_TX_OK;
4261 }
4262
4263 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
4264
4265 /* Use GSO to workaround a rare TSO bug that may be triggered when the
4266  * TSO header is greater than 80 bytes.
4267  */
4268 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
4269 {
4270         struct sk_buff *segs, *nskb;
4271
4272         /* Estimate the number of fragments in the worst case */
4273         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
4274                 netif_stop_queue(tp->dev);
4275                 if (tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))
4276                         return NETDEV_TX_BUSY;
4277
4278                 netif_wake_queue(tp->dev);
4279         }
4280
4281         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
4282         if (unlikely(IS_ERR(segs)))
4283                 goto tg3_tso_bug_end;
4284
4285         do {
4286                 nskb = segs;
4287                 segs = segs->next;
4288                 nskb->next = NULL;
4289                 tg3_start_xmit_dma_bug(nskb, tp->dev);
4290         } while (segs);
4291
4292 tg3_tso_bug_end:
4293         dev_kfree_skb(skb);
4294
4295         return NETDEV_TX_OK;
4296 }
4297
4298 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
4299  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
4300  */
4301 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
4302 {
4303         struct tg3 *tp = netdev_priv(dev);
4304         dma_addr_t mapping;
4305         u32 len, entry, base_flags, mss;
4306         int would_hit_hwbug;
4307
4308         len = skb_headlen(skb);
4309
4310         /* We are running in BH disabled context with netif_tx_lock
4311          * and TX reclaim runs via tp->napi.poll inside of a software
4312          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4313          * no IRQ context deadlocks to worry about either.  Rejoice!
4314          */
4315         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4316                 if (!netif_queue_stopped(dev)) {
4317                         netif_stop_queue(dev);
4318
4319                         /* This is a hard error, log it. */
4320                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4321                                "queue awake!\n", dev->name);
4322                 }
4323                 return NETDEV_TX_BUSY;
4324         }
4325
4326         entry = tp->tx_prod;
4327         base_flags = 0;
4328         if (skb->ip_summed == CHECKSUM_PARTIAL)
4329                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4330         mss = 0;
4331         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4332                 struct iphdr *iph;
4333                 int tcp_opt_len, ip_tcp_len, hdr_len;
4334
4335                 if (skb_header_cloned(skb) &&
4336                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4337                         dev_kfree_skb(skb);
4338                         goto out_unlock;
4339                 }
4340
4341                 tcp_opt_len = tcp_optlen(skb);
4342                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4343
4344                 hdr_len = ip_tcp_len + tcp_opt_len;
4345                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
4346                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
4347                         return (tg3_tso_bug(tp, skb));
4348
4349                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4350                                TXD_FLAG_CPU_POST_DMA);
4351
4352                 iph = ip_hdr(skb);
4353                 iph->check = 0;
4354                 iph->tot_len = htons(mss + hdr_len);
4355                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
4356                         tcp_hdr(skb)->check = 0;
4357                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
4358                 } else
4359                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4360                                                                  iph->daddr, 0,
4361                                                                  IPPROTO_TCP,
4362                                                                  0);
4363
4364                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
4365                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
4366                         if (tcp_opt_len || iph->ihl > 5) {
4367                                 int tsflags;
4368
4369                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4370                                 mss |= (tsflags << 11);
4371                         }
4372                 } else {
4373                         if (tcp_opt_len || iph->ihl > 5) {
4374                                 int tsflags;
4375
4376                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4377                                 base_flags |= tsflags << 12;
4378                         }
4379                 }
4380         }
4381 #if TG3_VLAN_TAG_USED
4382         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4383                 base_flags |= (TXD_FLAG_VLAN |
4384                                (vlan_tx_tag_get(skb) << 16));
4385 #endif
4386
4387         /* Queue skb data, a.k.a. the main skb fragment. */
4388         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4389
4390         tp->tx_buffers[entry].skb = skb;
4391         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4392
4393         would_hit_hwbug = 0;
4394
4395         if (tg3_4g_overflow_test(mapping, len))
4396                 would_hit_hwbug = 1;
4397
4398         tg3_set_txd(tp, entry, mapping, len, base_flags,
4399                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4400
4401         entry = NEXT_TX(entry);
4402
4403         /* Now loop through additional data fragments, and queue them. */
4404         if (skb_shinfo(skb)->nr_frags > 0) {
4405                 unsigned int i, last;
4406
4407                 last = skb_shinfo(skb)->nr_frags - 1;
4408                 for (i = 0; i <= last; i++) {
4409                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4410
4411                         len = frag->size;
4412                         mapping = pci_map_page(tp->pdev,
4413                                                frag->page,
4414                                                frag->page_offset,
4415                                                len, PCI_DMA_TODEVICE);
4416
4417                         tp->tx_buffers[entry].skb = NULL;
4418                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4419
4420                         if (tg3_4g_overflow_test(mapping, len))
4421                                 would_hit_hwbug = 1;
4422
4423                         if (tg3_40bit_overflow_test(tp, mapping, len))
4424                                 would_hit_hwbug = 1;
4425
4426                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4427                                 tg3_set_txd(tp, entry, mapping, len,
4428                                             base_flags, (i == last)|(mss << 1));
4429                         else
4430                                 tg3_set_txd(tp, entry, mapping, len,
4431                                             base_flags, (i == last));
4432
4433                         entry = NEXT_TX(entry);
4434                 }
4435         }
4436
4437         if (would_hit_hwbug) {
4438                 u32 last_plus_one = entry;
4439                 u32 start;
4440
4441                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4442                 start &= (TG3_TX_RING_SIZE - 1);
4443
4444                 /* If the workaround fails due to memory/mapping
4445                  * failure, silently drop this packet.
4446                  */
4447                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4448                                                 &start, base_flags, mss))
4449                         goto out_unlock;
4450
4451                 entry = start;
4452         }
4453
4454         /* Packets are ready, update Tx producer idx local and on card. */
4455         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4456
4457         tp->tx_prod = entry;
4458         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4459                 netif_stop_queue(dev);
4460                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4461                         netif_wake_queue(tp->dev);
4462         }
4463
4464 out_unlock:
4465         mmiowb();
4466
4467         dev->trans_start = jiffies;
4468
4469         return NETDEV_TX_OK;
4470 }
4471
4472 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4473                                int new_mtu)
4474 {
4475         dev->mtu = new_mtu;
4476
4477         if (new_mtu > ETH_DATA_LEN) {
4478                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4479                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4480                         ethtool_op_set_tso(dev, 0);
4481                 }
4482                 else
4483                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4484         } else {
4485                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4486                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4487                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4488         }
4489 }
4490
4491 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4492 {
4493         struct tg3 *tp = netdev_priv(dev);
4494         int err;
4495
4496         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4497                 return -EINVAL;
4498
4499         if (!netif_running(dev)) {
4500                 /* We'll just catch it later when the
4501                  * device is up'd.
4502                  */
4503                 tg3_set_mtu(dev, tp, new_mtu);
4504                 return 0;
4505         }
4506
4507         tg3_netif_stop(tp);
4508
4509         tg3_full_lock(tp, 1);
4510
4511         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4512
4513         tg3_set_mtu(dev, tp, new_mtu);
4514
4515         err = tg3_restart_hw(tp, 0);
4516
4517         if (!err)
4518                 tg3_netif_start(tp);
4519
4520         tg3_full_unlock(tp);
4521
4522         return err;
4523 }
4524
4525 /* Free up pending packets in all rx/tx rings.
4526  *
4527  * The chip has been shut down and the driver detached from
4528  * the networking, so no interrupts or new tx packets will
4529  * end up in the driver.  tp->{tx,}lock is not held and we are not
4530  * in an interrupt context and thus may sleep.
4531  */
4532 static void tg3_free_rings(struct tg3 *tp)
4533 {
4534         struct ring_info *rxp;
4535         int i;
4536
4537         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4538                 rxp = &tp->rx_std_buffers[i];
4539
4540                 if (rxp->skb == NULL)
4541                         continue;
4542                 pci_unmap_single(tp->pdev,
4543                                  pci_unmap_addr(rxp, mapping),
4544                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4545                                  PCI_DMA_FROMDEVICE);
4546                 dev_kfree_skb_any(rxp->skb);
4547                 rxp->skb = NULL;
4548         }
4549
4550         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4551                 rxp = &tp->rx_jumbo_buffers[i];
4552
4553                 if (rxp->skb == NULL)
4554                         continue;
4555                 pci_unmap_single(tp->pdev,
4556                                  pci_unmap_addr(rxp, mapping),
4557                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4558                                  PCI_DMA_FROMDEVICE);
4559                 dev_kfree_skb_any(rxp->skb);
4560                 rxp->skb = NULL;
4561         }
4562
4563         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4564                 struct tx_ring_info *txp;
4565                 struct sk_buff *skb;
4566                 int j;
4567
4568                 txp = &tp->tx_buffers[i];
4569                 skb = txp->skb;
4570
4571                 if (skb == NULL) {
4572                         i++;
4573                         continue;
4574                 }
4575
4576                 pci_unmap_single(tp->pdev,
4577                                  pci_unmap_addr(txp, mapping),
4578                                  skb_headlen(skb),
4579                                  PCI_DMA_TODEVICE);
4580                 txp->skb = NULL;
4581
4582                 i++;
4583
4584                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4585                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4586                         pci_unmap_page(tp->pdev,
4587                                        pci_unmap_addr(txp, mapping),
4588                                        skb_shinfo(skb)->frags[j].size,
4589                                        PCI_DMA_TODEVICE);
4590                         i++;
4591                 }
4592
4593                 dev_kfree_skb_any(skb);
4594         }
4595 }
4596
4597 /* Initialize tx/rx rings for packet processing.
4598  *
4599  * The chip has been shut down and the driver detached from
4600  * the networking, so no interrupts or new tx packets will
4601  * end up in the driver.  tp->{tx,}lock are held and thus
4602  * we may not sleep.
4603  */
4604 static int tg3_init_rings(struct tg3 *tp)
4605 {
4606         u32 i;
4607
4608         /* Free up all the SKBs. */
4609         tg3_free_rings(tp);
4610
4611         /* Zero out all descriptors. */
4612         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4613         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4614         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4615         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4616
4617         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4618         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4619             (tp->dev->mtu > ETH_DATA_LEN))
4620                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4621
4622         /* Initialize invariants of the rings, we only set this
4623          * stuff once.  This works because the card does not
4624          * write into the rx buffer posting rings.
4625          */
4626         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4627                 struct tg3_rx_buffer_desc *rxd;
4628
4629                 rxd = &tp->rx_std[i];
4630                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4631                         << RXD_LEN_SHIFT;
4632                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4633                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4634                                (i << RXD_OPAQUE_INDEX_SHIFT));
4635         }
4636
4637         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4638                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4639                         struct tg3_rx_buffer_desc *rxd;
4640
4641                         rxd = &tp->rx_jumbo[i];
4642                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4643                                 << RXD_LEN_SHIFT;
4644                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4645                                 RXD_FLAG_JUMBO;
4646                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4647                                (i << RXD_OPAQUE_INDEX_SHIFT));
4648                 }
4649         }
4650
4651         /* Now allocate fresh SKBs for each rx ring. */
4652         for (i = 0; i < tp->rx_pending; i++) {
4653                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD, -1, i) < 0) {
4654                         printk(KERN_WARNING PFX
4655                                "%s: Using a smaller RX standard ring, "
4656                                "only %d out of %d buffers were allocated "
4657                                "successfully.\n",
4658                                tp->dev->name, i, tp->rx_pending);
4659                         if (i == 0)
4660                                 return -ENOMEM;
4661                         tp->rx_pending = i;
4662                         break;
4663                 }
4664         }
4665
4666         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4667                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4668                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4669                                              -1, i) < 0) {
4670                                 printk(KERN_WARNING PFX
4671                                        "%s: Using a smaller RX jumbo ring, "
4672                                        "only %d out of %d buffers were "
4673                                        "allocated successfully.\n",
4674                                        tp->dev->name, i, tp->rx_jumbo_pending);
4675                                 if (i == 0) {
4676                                         tg3_free_rings(tp);
4677                                         return -ENOMEM;
4678                                 }
4679                                 tp->rx_jumbo_pending = i;
4680                                 break;
4681                         }
4682                 }
4683         }
4684         return 0;
4685 }
4686
4687 /*
4688  * Must not be invoked with interrupt sources disabled and
4689  * the hardware shutdown down.
4690  */
4691 static void tg3_free_consistent(struct tg3 *tp)
4692 {
4693         kfree(tp->rx_std_buffers);
4694         tp->rx_std_buffers = NULL;
4695         if (tp->rx_std) {
4696                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4697                                     tp->rx_std, tp->rx_std_mapping);
4698                 tp->rx_std = NULL;
4699         }
4700         if (tp->rx_jumbo) {
4701                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4702                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4703                 tp->rx_jumbo = NULL;
4704         }
4705         if (tp->rx_rcb) {
4706                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4707                                     tp->rx_rcb, tp->rx_rcb_mapping);
4708                 tp->rx_rcb = NULL;
4709         }
4710         if (tp->tx_ring) {
4711                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4712                         tp->tx_ring, tp->tx_desc_mapping);
4713                 tp->tx_ring = NULL;
4714         }
4715         if (tp->hw_status) {
4716                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4717                                     tp->hw_status, tp->status_mapping);
4718                 tp->hw_status = NULL;
4719         }
4720         if (tp->hw_stats) {
4721                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4722                                     tp->hw_stats, tp->stats_mapping);
4723                 tp->hw_stats = NULL;
4724         }
4725 }
4726
4727 /*
4728  * Must not be invoked with interrupt sources disabled and
4729  * the hardware shutdown down.  Can sleep.
4730  */
4731 static int tg3_alloc_consistent(struct tg3 *tp)
4732 {
4733         tp->rx_std_buffers = kzalloc((sizeof(struct ring_info) *
4734                                       (TG3_RX_RING_SIZE +
4735                                        TG3_RX_JUMBO_RING_SIZE)) +
4736                                      (sizeof(struct tx_ring_info) *
4737                                       TG3_TX_RING_SIZE),
4738                                      GFP_KERNEL);
4739         if (!tp->rx_std_buffers)
4740                 return -ENOMEM;
4741
4742         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4743         tp->tx_buffers = (struct tx_ring_info *)
4744                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4745
4746         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4747                                           &tp->rx_std_mapping);
4748         if (!tp->rx_std)
4749                 goto err_out;
4750
4751         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4752                                             &tp->rx_jumbo_mapping);
4753
4754         if (!tp->rx_jumbo)
4755                 goto err_out;
4756
4757         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4758                                           &tp->rx_rcb_mapping);
4759         if (!tp->rx_rcb)
4760                 goto err_out;
4761
4762         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4763                                            &tp->tx_desc_mapping);
4764         if (!tp->tx_ring)
4765                 goto err_out;
4766
4767         tp->hw_status = pci_alloc_consistent(tp->pdev,
4768                                              TG3_HW_STATUS_SIZE,
4769                                              &tp->status_mapping);
4770         if (!tp->hw_status)
4771                 goto err_out;
4772
4773         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4774                                             sizeof(struct tg3_hw_stats),
4775                                             &tp->stats_mapping);
4776         if (!tp->hw_stats)
4777                 goto err_out;
4778
4779         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4780         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4781
4782         return 0;
4783
4784 err_out:
4785         tg3_free_consistent(tp);
4786         return -ENOMEM;
4787 }
4788
4789 #define MAX_WAIT_CNT 1000
4790
4791 /* To stop a block, clear the enable bit and poll till it
4792  * clears.  tp->lock is held.
4793  */
4794 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4795 {
4796         unsigned int i;
4797         u32 val;
4798
4799         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4800                 switch (ofs) {
4801                 case RCVLSC_MODE:
4802                 case DMAC_MODE:
4803                 case MBFREE_MODE:
4804                 case BUFMGR_MODE:
4805                 case MEMARB_MODE:
4806                         /* We can't enable/disable these bits of the
4807                          * 5705/5750, just say success.
4808                          */
4809                         return 0;
4810
4811                 default:
4812                         break;
4813                 };
4814         }
4815
4816         val = tr32(ofs);
4817         val &= ~enable_bit;
4818         tw32_f(ofs, val);
4819
4820         for (i = 0; i < MAX_WAIT_CNT; i++) {
4821                 udelay(100);
4822                 val = tr32(ofs);
4823                 if ((val & enable_bit) == 0)
4824                         break;
4825         }
4826
4827         if (i == MAX_WAIT_CNT && !silent) {
4828                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4829                        "ofs=%lx enable_bit=%x\n",
4830                        ofs, enable_bit);
4831                 return -ENODEV;
4832         }
4833
4834         return 0;
4835 }
4836
4837 /* tp->lock is held. */
4838 static int tg3_abort_hw(struct tg3 *tp, int silent)
4839 {
4840         int i, err;
4841
4842         tg3_disable_ints(tp);
4843
4844         tp->rx_mode &= ~RX_MODE_ENABLE;
4845         tw32_f(MAC_RX_MODE, tp->rx_mode);
4846         udelay(10);
4847
4848         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4849         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4850         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4851         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4852         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4853         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4854
4855         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4856         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4857         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4858         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4859         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4860         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4861         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4862
4863         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4864         tw32_f(MAC_MODE, tp->mac_mode);
4865         udelay(40);
4866
4867         tp->tx_mode &= ~TX_MODE_ENABLE;
4868         tw32_f(MAC_TX_MODE, tp->tx_mode);
4869
4870         for (i = 0; i < MAX_WAIT_CNT; i++) {
4871                 udelay(100);
4872                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4873                         break;
4874         }
4875         if (i >= MAX_WAIT_CNT) {
4876                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4877                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4878                        tp->dev->name, tr32(MAC_TX_MODE));
4879                 err |= -ENODEV;
4880         }
4881
4882         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4883         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4884         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4885
4886         tw32(FTQ_RESET, 0xffffffff);
4887         tw32(FTQ_RESET, 0x00000000);
4888
4889         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4890         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4891
4892         if (tp->hw_status)
4893                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4894         if (tp->hw_stats)
4895                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4896
4897         return err;
4898 }
4899
4900 /* tp->lock is held. */
4901 static int tg3_nvram_lock(struct tg3 *tp)
4902 {
4903         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4904                 int i;
4905
4906                 if (tp->nvram_lock_cnt == 0) {
4907                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4908                         for (i = 0; i < 8000; i++) {
4909                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4910                                         break;
4911                                 udelay(20);
4912                         }
4913                         if (i == 8000) {
4914                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4915                                 return -ENODEV;
4916                         }
4917                 }
4918                 tp->nvram_lock_cnt++;
4919         }
4920         return 0;
4921 }
4922
4923 /* tp->lock is held. */
4924 static void tg3_nvram_unlock(struct tg3 *tp)
4925 {
4926         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4927                 if (tp->nvram_lock_cnt > 0)
4928                         tp->nvram_lock_cnt--;
4929                 if (tp->nvram_lock_cnt == 0)
4930                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4931         }
4932 }
4933
4934 /* tp->lock is held. */
4935 static void tg3_enable_nvram_access(struct tg3 *tp)
4936 {
4937         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4938             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4939                 u32 nvaccess = tr32(NVRAM_ACCESS);
4940
4941                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4942         }
4943 }
4944
4945 /* tp->lock is held. */
4946 static void tg3_disable_nvram_access(struct tg3 *tp)
4947 {
4948         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4949             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4950                 u32 nvaccess = tr32(NVRAM_ACCESS);
4951
4952                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4953         }
4954 }
4955
4956 static void tg3_ape_send_event(struct tg3 *tp, u32 event)
4957 {
4958         int i;
4959         u32 apedata;
4960
4961         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
4962         if (apedata != APE_SEG_SIG_MAGIC)
4963                 return;
4964
4965         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
4966         if (apedata != APE_FW_STATUS_READY)
4967                 return;
4968
4969         /* Wait for up to 1 millisecond for APE to service previous event. */
4970         for (i = 0; i < 10; i++) {
4971                 if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
4972                         return;
4973
4974                 apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
4975
4976                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4977                         tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
4978                                         event | APE_EVENT_STATUS_EVENT_PENDING);
4979
4980                 tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
4981
4982                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4983                         break;
4984
4985                 udelay(100);
4986         }
4987
4988         if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
4989                 tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
4990 }
4991
4992 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
4993 {
4994         u32 event;
4995         u32 apedata;
4996
4997         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
4998                 return;
4999
5000         switch (kind) {
5001                 case RESET_KIND_INIT:
5002                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
5003                                         APE_HOST_SEG_SIG_MAGIC);
5004                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
5005                                         APE_HOST_SEG_LEN_MAGIC);
5006                         apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
5007                         tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
5008                         tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
5009                                         APE_HOST_DRIVER_ID_MAGIC);
5010                         tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
5011                                         APE_HOST_BEHAV_NO_PHYLOCK);
5012
5013                         event = APE_EVENT_STATUS_STATE_START;
5014                         break;
5015                 case RESET_KIND_SHUTDOWN:
5016                         event = APE_EVENT_STATUS_STATE_UNLOAD;
5017                         break;
5018                 case RESET_KIND_SUSPEND:
5019                         event = APE_EVENT_STATUS_STATE_SUSPEND;
5020                         break;
5021                 default:
5022                         return;
5023         }
5024
5025         event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
5026
5027         tg3_ape_send_event(tp, event);
5028 }
5029
5030 /* tp->lock is held. */
5031 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
5032 {
5033         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
5034                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
5035
5036         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
5037                 switch (kind) {
5038                 case RESET_KIND_INIT:
5039                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5040                                       DRV_STATE_START);
5041                         break;
5042
5043                 case RESET_KIND_SHUTDOWN:
5044                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5045                                       DRV_STATE_UNLOAD);
5046                         break;
5047
5048                 case RESET_KIND_SUSPEND:
5049                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5050                                       DRV_STATE_SUSPEND);
5051                         break;
5052
5053                 default:
5054                         break;
5055                 };
5056         }
5057
5058         if (kind == RESET_KIND_INIT ||
5059             kind == RESET_KIND_SUSPEND)
5060                 tg3_ape_driver_state_change(tp, kind);
5061 }
5062
5063 /* tp->lock is held. */
5064 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
5065 {
5066         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
5067                 switch (kind) {
5068                 case RESET_KIND_INIT:
5069                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5070                                       DRV_STATE_START_DONE);
5071                         break;
5072
5073                 case RESET_KIND_SHUTDOWN:
5074                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5075                                       DRV_STATE_UNLOAD_DONE);
5076                         break;
5077
5078                 default:
5079                         break;
5080                 };
5081         }
5082
5083         if (kind == RESET_KIND_SHUTDOWN)
5084                 tg3_ape_driver_state_change(tp, kind);
5085 }
5086
5087 /* tp->lock is held. */
5088 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
5089 {
5090         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
5091                 switch (kind) {
5092                 case RESET_KIND_INIT:
5093                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5094                                       DRV_STATE_START);
5095                         break;
5096
5097                 case RESET_KIND_SHUTDOWN:
5098                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5099                                       DRV_STATE_UNLOAD);
5100                         break;
5101
5102                 case RESET_KIND_SUSPEND:
5103                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5104                                       DRV_STATE_SUSPEND);
5105                         break;
5106
5107                 default:
5108                         break;
5109                 };
5110         }
5111 }
5112
5113 static int tg3_poll_fw(struct tg3 *tp)
5114 {
5115         int i;
5116         u32 val;
5117
5118         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5119                 /* Wait up to 20ms for init done. */
5120                 for (i = 0; i < 200; i++) {
5121                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
5122                                 return 0;
5123                         udelay(100);
5124                 }
5125                 return -ENODEV;
5126         }
5127
5128         /* Wait for firmware initialization to complete. */
5129         for (i = 0; i < 100000; i++) {
5130                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
5131                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
5132                         break;
5133                 udelay(10);
5134         }
5135
5136         /* Chip might not be fitted with firmware.  Some Sun onboard
5137          * parts are configured like that.  So don't signal the timeout
5138          * of the above loop as an error, but do report the lack of
5139          * running firmware once.
5140          */
5141         if (i >= 100000 &&
5142             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
5143                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
5144
5145                 printk(KERN_INFO PFX "%s: No firmware running.\n",
5146                        tp->dev->name);
5147         }
5148
5149         return 0;
5150 }
5151
5152 /* Save PCI command register before chip reset */
5153 static void tg3_save_pci_state(struct tg3 *tp)
5154 {
5155         pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
5156 }
5157
5158 /* Restore PCI state after chip reset */
5159 static void tg3_restore_pci_state(struct tg3 *tp)
5160 {
5161         u32 val;
5162
5163         /* Re-enable indirect register accesses. */
5164         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
5165                                tp->misc_host_ctrl);
5166
5167         /* Set MAX PCI retry to zero. */
5168         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
5169         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
5170             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
5171                 val |= PCISTATE_RETRY_SAME_DMA;
5172         /* Allow reads and writes to the APE register and memory space. */
5173         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
5174                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
5175                        PCISTATE_ALLOW_APE_SHMEM_WR;
5176         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
5177
5178         pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
5179
5180         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
5181                 pcie_set_readrq(tp->pdev, 4096);
5182         else {
5183                 pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
5184                                       tp->pci_cacheline_sz);
5185                 pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
5186                                       tp->pci_lat_timer);
5187         }
5188
5189         /* Make sure PCI-X relaxed ordering bit is clear. */
5190         if (tp->pcix_cap) {
5191                 u16 pcix_cmd;
5192
5193                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5194                                      &pcix_cmd);
5195                 pcix_cmd &= ~PCI_X_CMD_ERO;
5196                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5197                                       pcix_cmd);
5198         }
5199
5200         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
5201
5202                 /* Chip reset on 5780 will reset MSI enable bit,
5203                  * so need to restore it.
5204                  */
5205                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
5206                         u16 ctrl;
5207
5208                         pci_read_config_word(tp->pdev,
5209                                              tp->msi_cap + PCI_MSI_FLAGS,
5210                                              &ctrl);
5211                         pci_write_config_word(tp->pdev,
5212                                               tp->msi_cap + PCI_MSI_FLAGS,
5213                                               ctrl | PCI_MSI_FLAGS_ENABLE);
5214                         val = tr32(MSGINT_MODE);
5215                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
5216                 }
5217         }
5218 }
5219
5220 static void tg3_stop_fw(struct tg3 *);
5221
5222 /* tp->lock is held. */
5223 static int tg3_chip_reset(struct tg3 *tp)
5224 {
5225         u32 val;
5226         void (*write_op)(struct tg3 *, u32, u32);
5227         int err;
5228
5229         tg3_nvram_lock(tp);
5230
5231         /* No matching tg3_nvram_unlock() after this because
5232          * chip reset below will undo the nvram lock.
5233          */
5234         tp->nvram_lock_cnt = 0;
5235
5236         /* GRC_MISC_CFG core clock reset will clear the memory
5237          * enable bit in PCI register 4 and the MSI enable bit
5238          * on some chips, so we save relevant registers here.
5239          */
5240         tg3_save_pci_state(tp);
5241
5242         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
5243             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
5244             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
5245             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
5246             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
5247                 tw32(GRC_FASTBOOT_PC, 0);
5248
5249         /*
5250          * We must avoid the readl() that normally takes place.
5251          * It locks machines, causes machine checks, and other
5252          * fun things.  So, temporarily disable the 5701
5253          * hardware workaround, while we do the reset.
5254          */
5255         write_op = tp->write32;
5256         if (write_op == tg3_write_flush_reg32)
5257                 tp->write32 = tg3_write32;
5258
5259         /* Prevent the irq handler from reading or writing PCI registers
5260          * during chip reset when the memory enable bit in the PCI command
5261          * register may be cleared.  The chip does not generate interrupt
5262          * at this time, but the irq handler may still be called due to irq
5263          * sharing or irqpoll.
5264          */
5265         tp->tg3_flags |= TG3_FLAG_CHIP_RESETTING;
5266         if (tp->hw_status) {
5267                 tp->hw_status->status = 0;
5268                 tp->hw_status->status_tag = 0;
5269         }
5270         tp->last_tag = 0;
5271         smp_mb();
5272         synchronize_irq(tp->pdev->irq);
5273
5274         /* do the reset */
5275         val = GRC_MISC_CFG_CORECLK_RESET;
5276
5277         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5278                 if (tr32(0x7e2c) == 0x60) {
5279                         tw32(0x7e2c, 0x20);
5280                 }
5281                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5282                         tw32(GRC_MISC_CFG, (1 << 29));
5283                         val |= (1 << 29);
5284                 }
5285         }
5286
5287         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5288                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
5289                 tw32(GRC_VCPU_EXT_CTRL,
5290                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
5291         }
5292
5293         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5294                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
5295         tw32(GRC_MISC_CFG, val);
5296
5297         /* restore 5701 hardware bug workaround write method */
5298         tp->write32 = write_op;
5299
5300         /* Unfortunately, we have to delay before the PCI read back.
5301          * Some 575X chips even will not respond to a PCI cfg access
5302          * when the reset command is given to the chip.
5303          *
5304          * How do these hardware designers expect things to work
5305          * properly if the PCI write is posted for a long period
5306          * of time?  It is always necessary to have some method by
5307          * which a register read back can occur to push the write
5308          * out which does the reset.
5309          *
5310          * For most tg3 variants the trick below was working.
5311          * Ho hum...
5312          */
5313         udelay(120);
5314
5315         /* Flush PCI posted writes.  The normal MMIO registers
5316          * are inaccessible at this time so this is the only
5317          * way to make this reliably (actually, this is no longer
5318          * the case, see above).  I tried to use indirect
5319          * register read/write but this upset some 5701 variants.
5320          */
5321         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
5322
5323         udelay(120);
5324
5325         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5326                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
5327                         int i;
5328                         u32 cfg_val;
5329
5330                         /* Wait for link training to complete.  */
5331                         for (i = 0; i < 5000; i++)
5332                                 udelay(100);
5333
5334                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
5335                         pci_write_config_dword(tp->pdev, 0xc4,
5336                                                cfg_val | (1 << 15));
5337                 }
5338                 /* Set PCIE max payload size and clear error status.  */
5339                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
5340         }
5341
5342         tg3_restore_pci_state(tp);
5343
5344         tp->tg3_flags &= ~TG3_FLAG_CHIP_RESETTING;
5345
5346         val = 0;
5347         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
5348                 val = tr32(MEMARB_MODE);
5349         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
5350
5351         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
5352                 tg3_stop_fw(tp);
5353                 tw32(0x5000, 0x400);
5354         }
5355
5356         tw32(GRC_MODE, tp->grc_mode);
5357
5358         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
5359                 val = tr32(0xc4);
5360
5361                 tw32(0xc4, val | (1 << 15));
5362         }
5363
5364         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
5365             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5366                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
5367                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
5368                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
5369                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
5370         }
5371
5372         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
5373                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
5374                 tw32_f(MAC_MODE, tp->mac_mode);
5375         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
5376                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
5377                 tw32_f(MAC_MODE, tp->mac_mode);
5378         } else
5379                 tw32_f(MAC_MODE, 0);
5380         udelay(40);
5381
5382         err = tg3_poll_fw(tp);
5383         if (err)
5384                 return err;
5385
5386         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
5387             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5388                 val = tr32(0x7c00);
5389
5390                 tw32(0x7c00, val | (1 << 25));
5391         }
5392
5393         /* Reprobe ASF enable state.  */
5394         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
5395         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
5396         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
5397         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
5398                 u32 nic_cfg;
5399
5400                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
5401                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
5402                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
5403                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
5404                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
5405                 }
5406         }
5407
5408         return 0;
5409 }
5410
5411 /* tp->lock is held. */
5412 static void tg3_stop_fw(struct tg3 *tp)
5413 {
5414         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
5415            !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
5416                 u32 val;
5417                 int i;
5418
5419                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
5420                 val = tr32(GRC_RX_CPU_EVENT);
5421                 val |= (1 << 14);
5422                 tw32(GRC_RX_CPU_EVENT, val);
5423
5424                 /* Wait for RX cpu to ACK the event.  */
5425                 for (i = 0; i < 100; i++) {
5426                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
5427                                 break;
5428                         udelay(1);
5429                 }
5430         }
5431 }
5432
5433 /* tp->lock is held. */
5434 static int tg3_halt(struct tg3 *tp, int kind, int silent)
5435 {
5436         int err;
5437
5438         tg3_stop_fw(tp);
5439
5440         tg3_write_sig_pre_reset(tp, kind);
5441
5442         tg3_abort_hw(tp, silent);
5443         err = tg3_chip_reset(tp);
5444
5445         tg3_write_sig_legacy(tp, kind);
5446         tg3_write_sig_post_reset(tp, kind);
5447
5448         if (err)
5449                 return err;
5450
5451         return 0;
5452 }
5453
5454 #define TG3_FW_RELEASE_MAJOR    0x0
5455 #define TG3_FW_RELASE_MINOR     0x0
5456 #define TG3_FW_RELEASE_FIX      0x0
5457 #define TG3_FW_START_ADDR       0x08000000
5458 #define TG3_FW_TEXT_ADDR        0x08000000
5459 #define TG3_FW_TEXT_LEN         0x9c0
5460 #define TG3_FW_RODATA_ADDR      0x080009c0
5461 #define TG3_FW_RODATA_LEN       0x60
5462 #define TG3_FW_DATA_ADDR        0x08000a40
5463 #define TG3_FW_DATA_LEN         0x20
5464 #define TG3_FW_SBSS_ADDR        0x08000a60
5465 #define TG3_FW_SBSS_LEN         0xc
5466 #define TG3_FW_BSS_ADDR         0x08000a70
5467 #define TG3_FW_BSS_LEN          0x10
5468
5469 static const u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
5470         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
5471         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
5472         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
5473         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
5474         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
5475         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
5476         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
5477         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
5478         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
5479         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
5480         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
5481         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
5482         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
5483         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
5484         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
5485         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5486         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
5487         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
5488         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
5489         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5490         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
5491         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
5492         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5493         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5494         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5495         0, 0, 0, 0, 0, 0,
5496         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
5497         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5498         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5499         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5500         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
5501         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
5502         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
5503         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
5504         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5505         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5506         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
5507         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5508         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5509         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5510         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
5511         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
5512         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
5513         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
5514         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
5515         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
5516         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
5517         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
5518         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
5519         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
5520         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
5521         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
5522         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
5523         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
5524         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
5525         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
5526         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
5527         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
5528         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
5529         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
5530         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
5531         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
5532         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
5533         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
5534         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
5535         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
5536         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
5537         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
5538         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
5539         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
5540         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
5541         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
5542         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
5543         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
5544         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
5545         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
5546         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
5547         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
5548         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
5549         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
5550         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
5551         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
5552         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
5553         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
5554         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
5555         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
5556         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
5557         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
5558         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
5559         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
5560         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
5561 };
5562
5563 static const u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
5564         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
5565         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
5566         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5567         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
5568         0x00000000
5569 };
5570
5571 #if 0 /* All zeros, don't eat up space with it. */
5572 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
5573         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5574         0x00000000, 0x00000000, 0x00000000, 0x00000000
5575 };
5576 #endif
5577
5578 #define RX_CPU_SCRATCH_BASE     0x30000
5579 #define RX_CPU_SCRATCH_SIZE     0x04000
5580 #define TX_CPU_SCRATCH_BASE     0x34000
5581 #define TX_CPU_SCRATCH_SIZE     0x04000
5582
5583 /* tp->lock is held. */
5584 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
5585 {
5586         int i;
5587
5588         BUG_ON(offset == TX_CPU_BASE &&
5589             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
5590
5591         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5592                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
5593
5594                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
5595                 return 0;
5596         }
5597         if (offset == RX_CPU_BASE) {
5598                 for (i = 0; i < 10000; i++) {
5599                         tw32(offset + CPU_STATE, 0xffffffff);
5600                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5601                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5602                                 break;
5603                 }
5604
5605                 tw32(offset + CPU_STATE, 0xffffffff);
5606                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
5607                 udelay(10);
5608         } else {
5609                 for (i = 0; i < 10000; i++) {
5610                         tw32(offset + CPU_STATE, 0xffffffff);
5611                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5612                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5613                                 break;
5614                 }
5615         }
5616
5617         if (i >= 10000) {
5618                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5619                        "and %s CPU\n",
5620                        tp->dev->name,
5621                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5622                 return -ENODEV;
5623         }
5624
5625         /* Clear firmware's nvram arbitration. */
5626         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5627                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5628         return 0;
5629 }
5630
5631 struct fw_info {
5632         unsigned int text_base;
5633         unsigned int text_len;
5634         const u32 *text_data;
5635         unsigned int rodata_base;
5636         unsigned int rodata_len;
5637         const u32 *rodata_data;
5638         unsigned int data_base;
5639         unsigned int data_len;
5640         const u32 *data_data;
5641 };
5642
5643 /* tp->lock is held. */
5644 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5645                                  int cpu_scratch_size, struct fw_info *info)
5646 {
5647         int err, lock_err, i;
5648         void (*write_op)(struct tg3 *, u32, u32);
5649
5650         if (cpu_base == TX_CPU_BASE &&
5651             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5652                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5653                        "TX cpu firmware on %s which is 5705.\n",
5654                        tp->dev->name);
5655                 return -EINVAL;
5656         }
5657
5658         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5659                 write_op = tg3_write_mem;
5660         else
5661                 write_op = tg3_write_indirect_reg32;
5662
5663         /* It is possible that bootcode is still loading at this point.
5664          * Get the nvram lock first before halting the cpu.
5665          */
5666         lock_err = tg3_nvram_lock(tp);
5667         err = tg3_halt_cpu(tp, cpu_base);
5668         if (!lock_err)
5669                 tg3_nvram_unlock(tp);
5670         if (err)
5671                 goto out;
5672
5673         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5674                 write_op(tp, cpu_scratch_base + i, 0);
5675         tw32(cpu_base + CPU_STATE, 0xffffffff);
5676         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5677         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5678                 write_op(tp, (cpu_scratch_base +
5679                               (info->text_base & 0xffff) +
5680                               (i * sizeof(u32))),
5681                          (info->text_data ?
5682                           info->text_data[i] : 0));
5683         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5684                 write_op(tp, (cpu_scratch_base +
5685                               (info->rodata_base & 0xffff) +
5686                               (i * sizeof(u32))),
5687                          (info->rodata_data ?
5688                           info->rodata_data[i] : 0));
5689         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5690                 write_op(tp, (cpu_scratch_base +
5691                               (info->data_base & 0xffff) +
5692                               (i * sizeof(u32))),
5693                          (info->data_data ?
5694                           info->data_data[i] : 0));
5695
5696         err = 0;
5697
5698 out:
5699         return err;
5700 }
5701
5702 /* tp->lock is held. */
5703 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5704 {
5705         struct fw_info info;
5706         int err, i;
5707
5708         info.text_base = TG3_FW_TEXT_ADDR;
5709         info.text_len = TG3_FW_TEXT_LEN;
5710         info.text_data = &tg3FwText[0];
5711         info.rodata_base = TG3_FW_RODATA_ADDR;
5712         info.rodata_len = TG3_FW_RODATA_LEN;
5713         info.rodata_data = &tg3FwRodata[0];
5714         info.data_base = TG3_FW_DATA_ADDR;
5715         info.data_len = TG3_FW_DATA_LEN;
5716         info.data_data = NULL;
5717
5718         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5719                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5720                                     &info);
5721         if (err)
5722                 return err;
5723
5724         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5725                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5726                                     &info);
5727         if (err)
5728                 return err;
5729
5730         /* Now startup only the RX cpu. */
5731         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5732         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5733
5734         for (i = 0; i < 5; i++) {
5735                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5736                         break;
5737                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5738                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5739                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5740                 udelay(1000);
5741         }
5742         if (i >= 5) {
5743                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5744                        "to set RX CPU PC, is %08x should be %08x\n",
5745                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5746                        TG3_FW_TEXT_ADDR);
5747                 return -ENODEV;
5748         }
5749         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5750         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5751
5752         return 0;
5753 }
5754
5755
5756 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5757 #define TG3_TSO_FW_RELASE_MINOR         0x6
5758 #define TG3_TSO_FW_RELEASE_FIX          0x0
5759 #define TG3_TSO_FW_START_ADDR           0x08000000
5760 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5761 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5762 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5763 #define TG3_TSO_FW_RODATA_LEN           0x60
5764 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5765 #define TG3_TSO_FW_DATA_LEN             0x30
5766 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5767 #define TG3_TSO_FW_SBSS_LEN             0x2c
5768 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5769 #define TG3_TSO_FW_BSS_LEN              0x894
5770
5771 static const u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5772         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5773         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5774         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5775         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5776         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5777         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5778         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5779         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5780         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5781         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5782         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5783         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5784         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5785         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5786         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5787         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5788         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5789         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5790         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5791         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5792         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5793         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5794         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5795         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5796         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5797         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5798         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5799         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5800         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5801         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5802         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5803         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5804         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5805         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5806         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5807         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5808         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5809         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5810         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5811         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5812         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5813         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5814         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5815         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5816         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5817         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5818         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5819         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5820         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5821         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5822         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5823         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5824         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5825         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5826         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5827         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5828         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5829         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5830         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5831         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5832         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5833         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5834         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5835         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5836         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5837         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5838         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5839         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5840         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5841         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5842         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5843         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5844         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5845         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5846         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5847         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5848         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5849         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5850         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5851         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5852         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5853         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5854         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5855         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5856         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5857         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5858         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5859         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5860         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5861         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5862         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5863         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5864         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5865         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5866         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5867         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5868         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5869         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5870         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5871         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5872         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5873         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5874         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5875         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5876         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5877         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5878         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5879         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5880         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5881         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5882         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5883         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5884         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5885         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5886         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5887         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5888         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5889         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5890         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5891         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5892         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5893         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5894         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5895         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5896         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5897         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5898         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5899         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5900         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5901         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5902         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5903         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5904         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5905         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5906         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5907         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5908         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5909         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5910         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5911         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5912         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5913         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5914         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5915         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5916         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5917         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5918         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5919         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5920         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5921         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5922         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5923         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5924         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5925         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5926         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5927         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5928         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5929         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5930         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5931         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5932         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5933         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5934         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5935         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5936         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5937         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5938         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5939         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5940         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5941         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5942         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5943         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5944         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5945         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5946         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5947         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5948         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5949         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5950         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5951         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5952         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5953         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5954         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5955         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5956         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5957         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5958         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5959         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5960         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5961         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5962         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5963         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5964         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5965         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5966         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5967         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5968         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5969         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5970         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5971         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5972         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5973         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5974         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5975         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5976         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5977         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5978         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5979         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5980         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5981         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5982         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5983         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5984         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5985         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5986         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5987         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5988         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5989         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5990         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5991         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5992         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5993         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5994         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5995         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5996         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5997         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5998         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5999         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
6000         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
6001         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
6002         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
6003         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
6004         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
6005         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
6006         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
6007         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
6008         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
6009         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
6010         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
6011         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
6012         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
6013         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
6014         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
6015         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
6016         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
6017         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
6018         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
6019         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
6020         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
6021         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
6022         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
6023         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
6024         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
6025         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
6026         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
6027         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
6028         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
6029         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
6030         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
6031         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
6032         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
6033         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
6034         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
6035         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
6036         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
6037         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
6038         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
6039         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
6040         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
6041         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
6042         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
6043         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
6044         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
6045         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
6046         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
6047         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
6048         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
6049         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
6050         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
6051         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
6052         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
6053         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
6054         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
6055         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
6056 };
6057
6058 static const u32 tg3TsoFwRodata[] = {
6059         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
6060         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
6061         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
6062         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
6063         0x00000000,
6064 };
6065
6066 static const u32 tg3TsoFwData[] = {
6067         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
6068         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
6069         0x00000000,
6070 };
6071
6072 /* 5705 needs a special version of the TSO firmware.  */
6073 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
6074 #define TG3_TSO5_FW_RELASE_MINOR        0x2
6075 #define TG3_TSO5_FW_RELEASE_FIX         0x0
6076 #define TG3_TSO5_FW_START_ADDR          0x00010000
6077 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
6078 #define TG3_TSO5_FW_TEXT_LEN            0xe90
6079 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
6080 #define TG3_TSO5_FW_RODATA_LEN          0x50
6081 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
6082 #define TG3_TSO5_FW_DATA_LEN            0x20
6083 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
6084 #define TG3_TSO5_FW_SBSS_LEN            0x28
6085 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
6086 #define TG3_TSO5_FW_BSS_LEN             0x88
6087
6088 static const u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
6089         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
6090         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
6091         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
6092         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
6093         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
6094         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
6095         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
6096         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
6097         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
6098         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
6099         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
6100         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
6101         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
6102         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
6103         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
6104         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
6105         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
6106         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
6107         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
6108         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
6109         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
6110         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
6111         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
6112         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
6113         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
6114         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
6115         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
6116         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
6117         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
6118         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
6119         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6120         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
6121         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
6122         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
6123         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
6124         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
6125         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
6126         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
6127         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
6128         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
6129         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
6130         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
6131         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
6132         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
6133         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
6134         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
6135         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
6136         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
6137         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
6138         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
6139         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
6140         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
6141         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
6142         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
6143         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
6144         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
6145         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
6146         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
6147         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
6148         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
6149         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
6150         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
6151         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
6152         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
6153         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
6154         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
6155         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6156         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
6157         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
6158         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
6159         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
6160         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
6161         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
6162         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
6163         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
6164         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
6165         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
6166         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
6167         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
6168         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
6169         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
6170         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
6171         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
6172         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
6173         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
6174         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
6175         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
6176         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
6177         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
6178         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
6179         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
6180         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
6181         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
6182         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
6183         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
6184         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
6185         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
6186         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
6187         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
6188         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
6189         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
6190         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
6191         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
6192         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
6193         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
6194         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
6195         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6196         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6197         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
6198         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
6199         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
6200         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
6201         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
6202         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
6203         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
6204         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
6205         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
6206         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6207         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6208         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
6209         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
6210         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
6211         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
6212         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
6213         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
6214         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
6215         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
6216         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
6217         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
6218         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
6219         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
6220         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
6221         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
6222         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
6223         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
6224         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
6225         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
6226         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
6227         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
6228         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
6229         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
6230         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
6231         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
6232         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
6233         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
6234         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
6235         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
6236         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
6237         0x00804821, 0x8faa0010, 0x3c020001, 0x8c420f3c, 0x3c040001, 0x8c840f44,
6238         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010001, 0xac230f3c, 0x14400003,
6239         0x00004021, 0x3c010001, 0xac200f3c, 0x3c020001, 0x8c420f3c, 0x3c030001,
6240         0x8c630f40, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
6241         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020001, 0x8c420f3c,
6242         0x3c030001, 0x8c630f40, 0x8f64680c, 0x00021140, 0x00431021, 0xac440008,
6243         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
6244         0x00000000, 0x00000000, 0x00000000,
6245 };
6246
6247 static const u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
6248         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
6249         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
6250         0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
6251         0x00000000, 0x00000000, 0x00000000,
6252 };
6253
6254 static const u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
6255         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
6256         0x00000000, 0x00000000, 0x00000000,
6257 };
6258
6259 /* tp->lock is held. */
6260 static int tg3_load_tso_firmware(struct tg3 *tp)
6261 {
6262         struct fw_info info;
6263         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
6264         int err, i;
6265
6266         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6267                 return 0;
6268
6269         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6270                 info.text_base = TG3_TSO5_FW_TEXT_ADDR;
6271                 info.text_len = TG3_TSO5_FW_TEXT_LEN;
6272                 info.text_data = &tg3Tso5FwText[0];
6273                 info.rodata_base = TG3_TSO5_FW_RODATA_ADDR;
6274                 info.rodata_len = TG3_TSO5_FW_RODATA_LEN;
6275                 info.rodata_data = &tg3Tso5FwRodata[0];
6276                 info.data_base = TG3_TSO5_FW_DATA_ADDR;
6277                 info.data_len = TG3_TSO5_FW_DATA_LEN;
6278                 info.data_data = &tg3Tso5FwData[0];
6279                 cpu_base = RX_CPU_BASE;
6280                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
6281                 cpu_scratch_size = (info.text_len +
6282                                     info.rodata_len +
6283                                     info.data_len +
6284                                     TG3_TSO5_FW_SBSS_LEN +
6285                                     TG3_TSO5_FW_BSS_LEN);
6286         } else {
6287                 info.text_base = TG3_TSO_FW_TEXT_ADDR;
6288                 info.text_len = TG3_TSO_FW_TEXT_LEN;
6289                 info.text_data = &tg3TsoFwText[0];
6290                 info.rodata_base = TG3_TSO_FW_RODATA_ADDR;
6291                 info.rodata_len = TG3_TSO_FW_RODATA_LEN;
6292                 info.rodata_data = &tg3TsoFwRodata[0];
6293                 info.data_base = TG3_TSO_FW_DATA_ADDR;
6294                 info.data_len = TG3_TSO_FW_DATA_LEN;
6295                 info.data_data = &tg3TsoFwData[0];
6296                 cpu_base = TX_CPU_BASE;
6297                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
6298                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
6299         }
6300
6301         err = tg3_load_firmware_cpu(tp, cpu_base,
6302                                     cpu_scratch_base, cpu_scratch_size,
6303                                     &info);
6304         if (err)
6305                 return err;
6306
6307         /* Now startup the cpu. */
6308         tw32(cpu_base + CPU_STATE, 0xffffffff);
6309         tw32_f(cpu_base + CPU_PC,    info.text_base);
6310
6311         for (i = 0; i < 5; i++) {
6312                 if (tr32(cpu_base + CPU_PC) == info.text_base)
6313                         break;
6314                 tw32(cpu_base + CPU_STATE, 0xffffffff);
6315                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
6316                 tw32_f(cpu_base + CPU_PC,    info.text_base);
6317                 udelay(1000);
6318         }
6319         if (i >= 5) {
6320                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
6321                        "to set CPU PC, is %08x should be %08x\n",
6322                        tp->dev->name, tr32(cpu_base + CPU_PC),
6323                        info.text_base);
6324                 return -ENODEV;
6325         }
6326         tw32(cpu_base + CPU_STATE, 0xffffffff);
6327         tw32_f(cpu_base + CPU_MODE,  0x00000000);
6328         return 0;
6329 }
6330
6331
6332 /* tp->lock is held. */
6333 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
6334 {
6335         u32 addr_high, addr_low;
6336         int i;
6337
6338         addr_high = ((tp->dev->dev_addr[0] << 8) |
6339                      tp->dev->dev_addr[1]);
6340         addr_low = ((tp->dev->dev_addr[2] << 24) |
6341                     (tp->dev->dev_addr[3] << 16) |
6342                     (tp->dev->dev_addr[4] <<  8) |
6343                     (tp->dev->dev_addr[5] <<  0));
6344         for (i = 0; i < 4; i++) {
6345                 if (i == 1 && skip_mac_1)
6346                         continue;
6347                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
6348                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
6349         }
6350
6351         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
6352             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6353                 for (i = 0; i < 12; i++) {
6354                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
6355                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
6356                 }
6357         }
6358
6359         addr_high = (tp->dev->dev_addr[0] +
6360                      tp->dev->dev_addr[1] +
6361                      tp->dev->dev_addr[2] +
6362                      tp->dev->dev_addr[3] +
6363                      tp->dev->dev_addr[4] +
6364                      tp->dev->dev_addr[5]) &
6365                 TX_BACKOFF_SEED_MASK;
6366         tw32(MAC_TX_BACKOFF_SEED, addr_high);
6367 }
6368
6369 static int tg3_set_mac_addr(struct net_device *dev, void *p)
6370 {
6371         struct tg3 *tp = netdev_priv(dev);
6372         struct sockaddr *addr = p;
6373         int err = 0, skip_mac_1 = 0;
6374
6375         if (!is_valid_ether_addr(addr->sa_data))
6376                 return -EINVAL;
6377
6378         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
6379
6380         if (!netif_running(dev))
6381                 return 0;
6382
6383         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6384                 u32 addr0_high, addr0_low, addr1_high, addr1_low;
6385
6386                 addr0_high = tr32(MAC_ADDR_0_HIGH);
6387                 addr0_low = tr32(MAC_ADDR_0_LOW);
6388                 addr1_high = tr32(MAC_ADDR_1_HIGH);
6389                 addr1_low = tr32(MAC_ADDR_1_LOW);
6390
6391                 /* Skip MAC addr 1 if ASF is using it. */
6392                 if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
6393                     !(addr1_high == 0 && addr1_low == 0))
6394                         skip_mac_1 = 1;
6395         }
6396         spin_lock_bh(&tp->lock);
6397         __tg3_set_mac_addr(tp, skip_mac_1);
6398         spin_unlock_bh(&tp->lock);
6399
6400         return err;
6401 }
6402
6403 /* tp->lock is held. */
6404 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
6405                            dma_addr_t mapping, u32 maxlen_flags,
6406                            u32 nic_addr)
6407 {
6408         tg3_write_mem(tp,
6409                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
6410                       ((u64) mapping >> 32));
6411         tg3_write_mem(tp,
6412                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
6413                       ((u64) mapping & 0xffffffff));
6414         tg3_write_mem(tp,
6415                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
6416                        maxlen_flags);
6417
6418         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6419                 tg3_write_mem(tp,
6420                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
6421                               nic_addr);
6422 }
6423
6424 static void __tg3_set_rx_mode(struct net_device *);
6425 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
6426 {
6427         tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
6428         tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
6429         tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
6430         tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
6431         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6432                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
6433                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
6434         }
6435         tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
6436         tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
6437         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6438                 u32 val = ec->stats_block_coalesce_usecs;
6439
6440                 if (!netif_carrier_ok(tp->dev))
6441                         val = 0;
6442
6443                 tw32(HOSTCC_STAT_COAL_TICKS, val);
6444         }
6445 }
6446
6447 /* tp->lock is held. */
6448 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
6449 {
6450         u32 val, rdmac_mode;
6451         int i, err, limit;
6452
6453         tg3_disable_ints(tp);
6454
6455         tg3_stop_fw(tp);
6456
6457         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
6458
6459         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
6460                 tg3_abort_hw(tp, 1);
6461         }
6462
6463         if (reset_phy)
6464                 tg3_phy_reset(tp);
6465
6466         err = tg3_chip_reset(tp);
6467         if (err)
6468                 return err;
6469
6470         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
6471
6472         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
6473             tp->pci_chip_rev_id == CHIPREV_ID_5784_A1) {
6474                 val = tr32(TG3_CPMU_CTRL);
6475                 val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
6476                 tw32(TG3_CPMU_CTRL, val);
6477
6478                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
6479                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
6480                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
6481                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
6482
6483                 val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
6484                 val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
6485                 val |= CPMU_LNK_AWARE_MACCLK_6_25;
6486                 tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
6487
6488                 val = tr32(TG3_CPMU_HST_ACC);
6489                 val &= ~CPMU_HST_ACC_MACCLK_MASK;
6490                 val |= CPMU_HST_ACC_MACCLK_6_25;
6491                 tw32(TG3_CPMU_HST_ACC, val);
6492         }
6493
6494         /* This works around an issue with Athlon chipsets on
6495          * B3 tigon3 silicon.  This bit has no effect on any
6496          * other revision.  But do not set this on PCI Express
6497          * chips and don't even touch the clocks if the CPMU is present.
6498          */
6499         if (!(tp->tg3_flags & TG3_FLAG_CPMU_PRESENT)) {
6500                 if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
6501                         tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
6502                 tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
6503         }
6504
6505         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
6506             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
6507                 val = tr32(TG3PCI_PCISTATE);
6508                 val |= PCISTATE_RETRY_SAME_DMA;
6509                 tw32(TG3PCI_PCISTATE, val);
6510         }
6511
6512         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
6513                 /* Allow reads and writes to the
6514                  * APE register and memory space.
6515                  */
6516                 val = tr32(TG3PCI_PCISTATE);
6517                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
6518                        PCISTATE_ALLOW_APE_SHMEM_WR;
6519                 tw32(TG3PCI_PCISTATE, val);
6520         }
6521
6522         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
6523                 /* Enable some hw fixes.  */
6524                 val = tr32(TG3PCI_MSI_DATA);
6525                 val |= (1 << 26) | (1 << 28) | (1 << 29);
6526                 tw32(TG3PCI_MSI_DATA, val);
6527         }
6528
6529         /* Descriptor ring init may make accesses to the
6530          * NIC SRAM area to setup the TX descriptors, so we
6531          * can only do this after the hardware has been
6532          * successfully reset.
6533          */
6534         err = tg3_init_rings(tp);
6535         if (err)
6536                 return err;
6537
6538         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
6539             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) {
6540                 /* This value is determined during the probe time DMA
6541                  * engine test, tg3_test_dma.
6542                  */
6543                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
6544         }
6545
6546         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
6547                           GRC_MODE_4X_NIC_SEND_RINGS |
6548                           GRC_MODE_NO_TX_PHDR_CSUM |
6549                           GRC_MODE_NO_RX_PHDR_CSUM);
6550         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
6551
6552         /* Pseudo-header checksum is done by hardware logic and not
6553          * the offload processers, so make the chip do the pseudo-
6554          * header checksums on receive.  For transmit it is more
6555          * convenient to do the pseudo-header checksum in software
6556          * as Linux does that on transmit for us in all cases.
6557          */
6558         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
6559
6560         tw32(GRC_MODE,
6561              tp->grc_mode |
6562              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
6563
6564         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
6565         val = tr32(GRC_MISC_CFG);
6566         val &= ~0xff;
6567         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
6568         tw32(GRC_MISC_CFG, val);
6569
6570         /* Initialize MBUF/DESC pool. */
6571         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6572                 /* Do nothing.  */
6573         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
6574                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
6575                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
6576                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
6577                 else
6578                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
6579                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
6580                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
6581         }
6582         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6583                 int fw_len;
6584
6585                 fw_len = (TG3_TSO5_FW_TEXT_LEN +
6586                           TG3_TSO5_FW_RODATA_LEN +
6587                           TG3_TSO5_FW_DATA_LEN +
6588                           TG3_TSO5_FW_SBSS_LEN +
6589                           TG3_TSO5_FW_BSS_LEN);
6590                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
6591                 tw32(BUFMGR_MB_POOL_ADDR,
6592                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
6593                 tw32(BUFMGR_MB_POOL_SIZE,
6594                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
6595         }
6596
6597         if (tp->dev->mtu <= ETH_DATA_LEN) {
6598                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6599                      tp->bufmgr_config.mbuf_read_dma_low_water);
6600                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6601                      tp->bufmgr_config.mbuf_mac_rx_low_water);
6602                 tw32(BUFMGR_MB_HIGH_WATER,
6603                      tp->bufmgr_config.mbuf_high_water);
6604         } else {
6605                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6606                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
6607                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6608                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
6609                 tw32(BUFMGR_MB_HIGH_WATER,
6610                      tp->bufmgr_config.mbuf_high_water_jumbo);
6611         }
6612         tw32(BUFMGR_DMA_LOW_WATER,
6613              tp->bufmgr_config.dma_low_water);
6614         tw32(BUFMGR_DMA_HIGH_WATER,
6615              tp->bufmgr_config.dma_high_water);
6616
6617         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
6618         for (i = 0; i < 2000; i++) {
6619                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
6620                         break;
6621                 udelay(10);
6622         }
6623         if (i >= 2000) {
6624                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
6625                        tp->dev->name);
6626                 return -ENODEV;
6627         }
6628
6629         /* Setup replenish threshold. */
6630         val = tp->rx_pending / 8;
6631         if (val == 0)
6632                 val = 1;
6633         else if (val > tp->rx_std_max_post)
6634                 val = tp->rx_std_max_post;
6635         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6636                 if (tp->pci_chip_rev_id == CHIPREV_ID_5906_A1)
6637                         tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
6638
6639                 if (val > (TG3_RX_INTERNAL_RING_SZ_5906 / 2))
6640                         val = TG3_RX_INTERNAL_RING_SZ_5906 / 2;
6641         }
6642
6643         tw32(RCVBDI_STD_THRESH, val);
6644
6645         /* Initialize TG3_BDINFO's at:
6646          *  RCVDBDI_STD_BD:     standard eth size rx ring
6647          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
6648          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
6649          *
6650          * like so:
6651          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
6652          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
6653          *                              ring attribute flags
6654          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
6655          *
6656          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
6657          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
6658          *
6659          * The size of each ring is fixed in the firmware, but the location is
6660          * configurable.
6661          */
6662         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6663              ((u64) tp->rx_std_mapping >> 32));
6664         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6665              ((u64) tp->rx_std_mapping & 0xffffffff));
6666         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
6667              NIC_SRAM_RX_BUFFER_DESC);
6668
6669         /* Don't even try to program the JUMBO/MINI buffer descriptor
6670          * configs on 5705.
6671          */
6672         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6673                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6674                      RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT);
6675         } else {
6676                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6677                      RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6678
6679                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
6680                      BDINFO_FLAGS_DISABLED);
6681
6682                 /* Setup replenish threshold. */
6683                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
6684
6685                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
6686                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6687                              ((u64) tp->rx_jumbo_mapping >> 32));
6688                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6689                              ((u64) tp->rx_jumbo_mapping & 0xffffffff));
6690                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6691                              RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6692                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
6693                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
6694                 } else {
6695                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6696                              BDINFO_FLAGS_DISABLED);
6697                 }
6698
6699         }
6700
6701         /* There is only one send ring on 5705/5750, no need to explicitly
6702          * disable the others.
6703          */
6704         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6705                 /* Clear out send RCB ring in SRAM. */
6706                 for (i = NIC_SRAM_SEND_RCB; i < NIC_SRAM_RCV_RET_RCB; i += TG3_BDINFO_SIZE)
6707                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6708                                       BDINFO_FLAGS_DISABLED);
6709         }
6710
6711         tp->tx_prod = 0;
6712         tp->tx_cons = 0;
6713         tw32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6714         tw32_tx_mbox(MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6715
6716         tg3_set_bdinfo(tp, NIC_SRAM_SEND_RCB,
6717                        tp->tx_desc_mapping,
6718                        (TG3_TX_RING_SIZE <<
6719                         BDINFO_FLAGS_MAXLEN_SHIFT),
6720                        NIC_SRAM_TX_BUFFER_DESC);
6721
6722         /* There is only one receive return ring on 5705/5750, no need
6723          * to explicitly disable the others.
6724          */
6725         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6726                 for (i = NIC_SRAM_RCV_RET_RCB; i < NIC_SRAM_STATS_BLK;
6727                      i += TG3_BDINFO_SIZE) {
6728                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6729                                       BDINFO_FLAGS_DISABLED);
6730                 }
6731         }
6732
6733         tp->rx_rcb_ptr = 0;
6734         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, 0);
6735
6736         tg3_set_bdinfo(tp, NIC_SRAM_RCV_RET_RCB,
6737                        tp->rx_rcb_mapping,
6738                        (TG3_RX_RCB_RING_SIZE(tp) <<
6739                         BDINFO_FLAGS_MAXLEN_SHIFT),
6740                        0);
6741
6742         tp->rx_std_ptr = tp->rx_pending;
6743         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
6744                      tp->rx_std_ptr);
6745
6746         tp->rx_jumbo_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
6747                                                 tp->rx_jumbo_pending : 0;
6748         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
6749                      tp->rx_jumbo_ptr);
6750
6751         /* Initialize MAC address and backoff seed. */
6752         __tg3_set_mac_addr(tp, 0);
6753
6754         /* MTU + ethernet header + FCS + optional VLAN tag */
6755         tw32(MAC_RX_MTU_SIZE, tp->dev->mtu + ETH_HLEN + 8);
6756
6757         /* The slot time is changed by tg3_setup_phy if we
6758          * run at gigabit with half duplex.
6759          */
6760         tw32(MAC_TX_LENGTHS,
6761              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6762              (6 << TX_LENGTHS_IPG_SHIFT) |
6763              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6764
6765         /* Receive rules. */
6766         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
6767         tw32(RCVLPC_CONFIG, 0x0181);
6768
6769         /* Calculate RDMAC_MODE setting early, we need it to determine
6770          * the RCVLPC_STATE_ENABLE mask.
6771          */
6772         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
6773                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
6774                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
6775                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
6776                       RDMAC_MODE_LNGREAD_ENAB);
6777
6778         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
6779                 rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
6780                               RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
6781                               RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
6782
6783         /* If statement applies to 5705 and 5750 PCI devices only */
6784         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6785              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6786             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
6787                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
6788                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6789                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
6790                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6791                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
6792                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6793                 }
6794         }
6795
6796         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6797                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6798
6799         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6800                 rdmac_mode |= (1 << 27);
6801
6802         /* Receive/send statistics. */
6803         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6804                 val = tr32(RCVLPC_STATS_ENABLE);
6805                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
6806                 tw32(RCVLPC_STATS_ENABLE, val);
6807         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
6808                    (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
6809                 val = tr32(RCVLPC_STATS_ENABLE);
6810                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
6811                 tw32(RCVLPC_STATS_ENABLE, val);
6812         } else {
6813                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
6814         }
6815         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
6816         tw32(SNDDATAI_STATSENAB, 0xffffff);
6817         tw32(SNDDATAI_STATSCTRL,
6818              (SNDDATAI_SCTRL_ENABLE |
6819               SNDDATAI_SCTRL_FASTUPD));
6820
6821         /* Setup host coalescing engine. */
6822         tw32(HOSTCC_MODE, 0);
6823         for (i = 0; i < 2000; i++) {
6824                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
6825                         break;
6826                 udelay(10);
6827         }
6828
6829         __tg3_set_coalesce(tp, &tp->coal);
6830
6831         /* set status block DMA address */
6832         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6833              ((u64) tp->status_mapping >> 32));
6834         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6835              ((u64) tp->status_mapping & 0xffffffff));
6836
6837         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6838                 /* Status/statistics block address.  See tg3_timer,
6839                  * the tg3_periodic_fetch_stats call there, and
6840                  * tg3_get_stats to see how this works for 5705/5750 chips.
6841                  */
6842                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6843                      ((u64) tp->stats_mapping >> 32));
6844                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6845                      ((u64) tp->stats_mapping & 0xffffffff));
6846                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
6847                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
6848         }
6849
6850         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
6851
6852         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
6853         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
6854         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6855                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
6856
6857         /* Clear statistics/status block in chip, and status block in ram. */
6858         for (i = NIC_SRAM_STATS_BLK;
6859              i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
6860              i += sizeof(u32)) {
6861                 tg3_write_mem(tp, i, 0);
6862                 udelay(40);
6863         }
6864         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
6865
6866         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6867                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
6868                 /* reset to prevent losing 1st rx packet intermittently */
6869                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6870                 udelay(10);
6871         }
6872
6873         tp->mac_mode = MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
6874                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
6875         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6876             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
6877             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700)
6878                 tp->mac_mode |= MAC_MODE_LINK_POLARITY;
6879         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
6880         udelay(40);
6881
6882         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
6883          * If TG3_FLG2_IS_NIC is zero, we should read the
6884          * register to preserve the GPIO settings for LOMs. The GPIOs,
6885          * whether used as inputs or outputs, are set by boot code after
6886          * reset.
6887          */
6888         if (!(tp->tg3_flags2 & TG3_FLG2_IS_NIC)) {
6889                 u32 gpio_mask;
6890
6891                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
6892                             GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
6893                             GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
6894
6895                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
6896                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
6897                                      GRC_LCLCTRL_GPIO_OUTPUT3;
6898
6899                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6900                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
6901
6902                 tp->grc_local_ctrl &= ~gpio_mask;
6903                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
6904
6905                 /* GPIO1 must be driven high for eeprom write protect */
6906                 if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT)
6907                         tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
6908                                                GRC_LCLCTRL_GPIO_OUTPUT1);
6909         }
6910         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6911         udelay(100);
6912
6913         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0);
6914         tp->last_tag = 0;
6915
6916         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6917                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
6918                 udelay(40);
6919         }
6920
6921         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
6922                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
6923                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
6924                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
6925                WDMAC_MODE_LNGREAD_ENAB);
6926
6927         /* If statement applies to 5705 and 5750 PCI devices only */
6928         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6929              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6930             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
6931                 if ((tp->tg3_flags & TG3_FLG2_TSO_CAPABLE) &&
6932                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6933                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6934                         /* nothing */
6935                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6936                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
6937                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
6938                         val |= WDMAC_MODE_RX_ACCEL;
6939                 }
6940         }
6941
6942         /* Enable host coalescing bug fix */
6943         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755) ||
6944             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787) ||
6945             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784) ||
6946             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761))
6947                 val |= (1 << 29);
6948
6949         tw32_f(WDMAC_MODE, val);
6950         udelay(40);
6951
6952         if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
6953                 u16 pcix_cmd;
6954
6955                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6956                                      &pcix_cmd);
6957                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
6958                         pcix_cmd &= ~PCI_X_CMD_MAX_READ;
6959                         pcix_cmd |= PCI_X_CMD_READ_2K;
6960                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6961                         pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
6962                         pcix_cmd |= PCI_X_CMD_READ_2K;
6963                 }
6964                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6965                                       pcix_cmd);
6966         }
6967
6968         tw32_f(RDMAC_MODE, rdmac_mode);
6969         udelay(40);
6970
6971         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
6972         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6973                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
6974
6975         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
6976                 tw32(SNDDATAC_MODE,
6977                      SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
6978         else
6979                 tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
6980
6981         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
6982         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
6983         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
6984         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
6985         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6986                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
6987         tw32(SNDBDI_MODE, SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE);
6988         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
6989
6990         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
6991                 err = tg3_load_5701_a0_firmware_fix(tp);
6992                 if (err)
6993                         return err;
6994         }
6995
6996         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6997                 err = tg3_load_tso_firmware(tp);
6998                 if (err)
6999                         return err;
7000         }
7001
7002         tp->tx_mode = TX_MODE_ENABLE;
7003         tw32_f(MAC_TX_MODE, tp->tx_mode);
7004         udelay(100);
7005
7006         tp->rx_mode = RX_MODE_ENABLE;
7007         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
7008             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
7009                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
7010
7011         tw32_f(MAC_RX_MODE, tp->rx_mode);
7012         udelay(10);
7013
7014         if (tp->link_config.phy_is_low_power) {
7015                 tp->link_config.phy_is_low_power = 0;
7016                 tp->link_config.speed = tp->link_config.orig_speed;
7017                 tp->link_config.duplex = tp->link_config.orig_duplex;
7018                 tp->link_config.autoneg = tp->link_config.orig_autoneg;
7019         }
7020
7021         tp->mi_mode = MAC_MI_MODE_BASE;
7022         tw32_f(MAC_MI_MODE, tp->mi_mode);
7023         udelay(80);
7024
7025         tw32(MAC_LED_CTRL, tp->led_ctrl);
7026
7027         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
7028         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7029                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
7030                 udelay(10);
7031         }
7032         tw32_f(MAC_RX_MODE, tp->rx_mode);
7033         udelay(10);
7034
7035         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7036                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
7037                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
7038                         /* Set drive transmission level to 1.2V  */
7039                         /* only if the signal pre-emphasis bit is not set  */
7040                         val = tr32(MAC_SERDES_CFG);
7041                         val &= 0xfffff000;
7042                         val |= 0x880;
7043                         tw32(MAC_SERDES_CFG, val);
7044                 }
7045                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
7046                         tw32(MAC_SERDES_CFG, 0x616000);
7047         }
7048
7049         /* Prevent chip from dropping frames when flow control
7050          * is enabled.
7051          */
7052         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
7053
7054         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
7055             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
7056                 /* Use hardware link auto-negotiation */
7057                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
7058         }
7059
7060         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
7061             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
7062                 u32 tmp;
7063
7064                 tmp = tr32(SERDES_RX_CTRL);
7065                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
7066                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
7067                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
7068                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
7069         }
7070
7071         err = tg3_setup_phy(tp, 0);
7072         if (err)
7073                 return err;
7074
7075         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7076             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906) {
7077                 u32 tmp;
7078
7079                 /* Clear CRC stats. */
7080                 if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
7081                         tg3_writephy(tp, MII_TG3_TEST1,
7082                                      tmp | MII_TG3_TEST1_CRC_EN);
7083                         tg3_readphy(tp, 0x14, &tmp);
7084                 }
7085         }
7086
7087         __tg3_set_rx_mode(tp->dev);
7088
7089         /* Initialize receive rules. */
7090         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
7091         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
7092         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
7093         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
7094
7095         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
7096             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
7097                 limit = 8;
7098         else
7099                 limit = 16;
7100         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
7101                 limit -= 4;
7102         switch (limit) {
7103         case 16:
7104                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
7105         case 15:
7106                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
7107         case 14:
7108                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
7109         case 13:
7110                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
7111         case 12:
7112                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
7113         case 11:
7114                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
7115         case 10:
7116                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
7117         case 9:
7118                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
7119         case 8:
7120                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
7121         case 7:
7122                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
7123         case 6:
7124                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
7125         case 5:
7126                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
7127         case 4:
7128                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
7129         case 3:
7130                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
7131         case 2:
7132         case 1:
7133
7134         default:
7135                 break;
7136         };
7137
7138         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
7139                 /* Write our heartbeat update interval to APE. */
7140                 tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
7141                                 APE_HOST_HEARTBEAT_INT_DISABLE);
7142
7143         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
7144
7145         return 0;
7146 }
7147
7148 /* Called at device open time to get the chip ready for
7149  * packet processing.  Invoked with tp->lock held.
7150  */
7151 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
7152 {
7153         int err;
7154
7155         /* Force the chip into D0. */
7156         err = tg3_set_power_state(tp, PCI_D0);
7157         if (err)
7158                 goto out;
7159
7160         tg3_switch_clocks(tp);
7161
7162         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
7163
7164         err = tg3_reset_hw(tp, reset_phy);
7165
7166 out:
7167         return err;
7168 }
7169
7170 #define TG3_STAT_ADD32(PSTAT, REG) \
7171 do {    u32 __val = tr32(REG); \
7172         (PSTAT)->low += __val; \
7173         if ((PSTAT)->low < __val) \
7174                 (PSTAT)->high += 1; \
7175 } while (0)
7176
7177 static void tg3_periodic_fetch_stats(struct tg3 *tp)
7178 {
7179         struct tg3_hw_stats *sp = tp->hw_stats;
7180
7181         if (!netif_carrier_ok(tp->dev))
7182                 return;
7183
7184         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
7185         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
7186         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
7187         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
7188         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
7189         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
7190         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
7191         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
7192         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
7193         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
7194         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
7195         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
7196         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
7197
7198         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
7199         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
7200         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
7201         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
7202         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
7203         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
7204         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
7205         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
7206         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
7207         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
7208         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
7209         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
7210         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
7211         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
7212
7213         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
7214         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
7215         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
7216 }
7217
7218 static void tg3_timer(unsigned long __opaque)
7219 {
7220         struct tg3 *tp = (struct tg3 *) __opaque;
7221
7222         if (tp->irq_sync)
7223                 goto restart_timer;
7224
7225         spin_lock(&tp->lock);
7226
7227         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7228                 /* All of this garbage is because when using non-tagged
7229                  * IRQ status the mailbox/status_block protocol the chip
7230                  * uses with the cpu is race prone.
7231                  */
7232                 if (tp->hw_status->status & SD_STATUS_UPDATED) {
7233                         tw32(GRC_LOCAL_CTRL,
7234                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
7235                 } else {
7236                         tw32(HOSTCC_MODE, tp->coalesce_mode |
7237                              (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
7238                 }
7239
7240                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
7241                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
7242                         spin_unlock(&tp->lock);
7243                         schedule_work(&tp->reset_task);
7244                         return;
7245                 }
7246         }
7247
7248         /* This part only runs once per second. */
7249         if (!--tp->timer_counter) {
7250                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
7251                         tg3_periodic_fetch_stats(tp);
7252
7253                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
7254                         u32 mac_stat;
7255                         int phy_event;
7256
7257                         mac_stat = tr32(MAC_STATUS);
7258
7259                         phy_event = 0;
7260                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
7261                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
7262                                         phy_event = 1;
7263                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
7264                                 phy_event = 1;
7265
7266                         if (phy_event)
7267                                 tg3_setup_phy(tp, 0);
7268                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
7269                         u32 mac_stat = tr32(MAC_STATUS);
7270                         int need_setup = 0;
7271
7272                         if (netif_carrier_ok(tp->dev) &&
7273                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
7274                                 need_setup = 1;
7275                         }
7276                         if (! netif_carrier_ok(tp->dev) &&
7277                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
7278                                          MAC_STATUS_SIGNAL_DET))) {
7279                                 need_setup = 1;
7280                         }
7281                         if (need_setup) {
7282                                 if (!tp->serdes_counter) {
7283                                         tw32_f(MAC_MODE,
7284                                              (tp->mac_mode &
7285                                               ~MAC_MODE_PORT_MODE_MASK));
7286                                         udelay(40);
7287                                         tw32_f(MAC_MODE, tp->mac_mode);
7288                                         udelay(40);
7289                                 }
7290                                 tg3_setup_phy(tp, 0);
7291                         }
7292                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
7293                         tg3_serdes_parallel_detect(tp);
7294
7295                 tp->timer_counter = tp->timer_multiplier;
7296         }
7297
7298         /* Heartbeat is only sent once every 2 seconds.
7299          *
7300          * The heartbeat is to tell the ASF firmware that the host
7301          * driver is still alive.  In the event that the OS crashes,
7302          * ASF needs to reset the hardware to free up the FIFO space
7303          * that may be filled with rx packets destined for the host.
7304          * If the FIFO is full, ASF will no longer function properly.
7305          *
7306          * Unintended resets have been reported on real time kernels
7307          * where the timer doesn't run on time.  Netpoll will also have
7308          * same problem.
7309          *
7310          * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
7311          * to check the ring condition when the heartbeat is expiring
7312          * before doing the reset.  This will prevent most unintended
7313          * resets.
7314          */
7315         if (!--tp->asf_counter) {
7316                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
7317                         u32 val;
7318
7319                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
7320                                       FWCMD_NICDRV_ALIVE3);
7321                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
7322                         /* 5 seconds timeout */
7323                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
7324                         val = tr32(GRC_RX_CPU_EVENT);
7325                         val |= (1 << 14);
7326                         tw32(GRC_RX_CPU_EVENT, val);
7327                 }
7328                 tp->asf_counter = tp->asf_multiplier;
7329         }
7330
7331         spin_unlock(&tp->lock);
7332
7333 restart_timer:
7334         tp->timer.expires = jiffies + tp->timer_offset;
7335         add_timer(&tp->timer);
7336 }
7337
7338 static int tg3_request_irq(struct tg3 *tp)
7339 {
7340         irq_handler_t fn;
7341         unsigned long flags;
7342         struct net_device *dev = tp->dev;
7343
7344         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7345                 fn = tg3_msi;
7346                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
7347                         fn = tg3_msi_1shot;
7348                 flags = IRQF_SAMPLE_RANDOM;
7349         } else {
7350                 fn = tg3_interrupt;
7351                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7352                         fn = tg3_interrupt_tagged;
7353                 flags = IRQF_SHARED | IRQF_SAMPLE_RANDOM;
7354         }
7355         return (request_irq(tp->pdev->irq, fn, flags, dev->name, dev));
7356 }
7357
7358 static int tg3_test_interrupt(struct tg3 *tp)
7359 {
7360         struct net_device *dev = tp->dev;
7361         int err, i, intr_ok = 0;
7362
7363         if (!netif_running(dev))
7364                 return -ENODEV;
7365
7366         tg3_disable_ints(tp);
7367
7368         free_irq(tp->pdev->irq, dev);
7369
7370         err = request_irq(tp->pdev->irq, tg3_test_isr,
7371                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, dev);
7372         if (err)
7373                 return err;
7374
7375         tp->hw_status->status &= ~SD_STATUS_UPDATED;
7376         tg3_enable_ints(tp);
7377
7378         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
7379                HOSTCC_MODE_NOW);
7380
7381         for (i = 0; i < 5; i++) {
7382                 u32 int_mbox, misc_host_ctrl;
7383
7384                 int_mbox = tr32_mailbox(MAILBOX_INTERRUPT_0 +
7385                                         TG3_64BIT_REG_LOW);
7386                 misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
7387
7388                 if ((int_mbox != 0) ||
7389                     (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
7390                         intr_ok = 1;
7391                         break;
7392                 }
7393
7394                 msleep(10);
7395         }
7396
7397         tg3_disable_ints(tp);
7398
7399         free_irq(tp->pdev->irq, dev);
7400
7401         err = tg3_request_irq(tp);
7402
7403         if (err)
7404                 return err;
7405
7406         if (intr_ok)
7407                 return 0;
7408
7409         return -EIO;
7410 }
7411
7412 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
7413  * successfully restored
7414  */
7415 static int tg3_test_msi(struct tg3 *tp)
7416 {
7417         struct net_device *dev = tp->dev;
7418         int err;
7419         u16 pci_cmd;
7420
7421         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
7422                 return 0;
7423
7424         /* Turn off SERR reporting in case MSI terminates with Master
7425          * Abort.
7426          */
7427         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
7428         pci_write_config_word(tp->pdev, PCI_COMMAND,
7429                               pci_cmd & ~PCI_COMMAND_SERR);
7430
7431         err = tg3_test_interrupt(tp);
7432
7433         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
7434
7435         if (!err)
7436                 return 0;
7437
7438         /* other failures */
7439         if (err != -EIO)
7440                 return err;
7441
7442         /* MSI test failed, go back to INTx mode */
7443         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
7444                "switching to INTx mode. Please report this failure to "
7445                "the PCI maintainer and include system chipset information.\n",
7446                        tp->dev->name);
7447
7448         free_irq(tp->pdev->irq, dev);
7449         pci_disable_msi(tp->pdev);
7450
7451         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7452
7453         err = tg3_request_irq(tp);
7454         if (err)
7455                 return err;
7456
7457         /* Need to reset the chip because the MSI cycle may have terminated
7458          * with Master Abort.
7459          */
7460         tg3_full_lock(tp, 1);
7461
7462         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7463         err = tg3_init_hw(tp, 1);
7464
7465         tg3_full_unlock(tp);
7466
7467         if (err)
7468                 free_irq(tp->pdev->irq, dev);
7469
7470         return err;
7471 }
7472
7473 static int tg3_open(struct net_device *dev)
7474 {
7475         struct tg3 *tp = netdev_priv(dev);
7476         int err;
7477
7478         netif_carrier_off(tp->dev);
7479
7480         tg3_full_lock(tp, 0);
7481
7482         err = tg3_set_power_state(tp, PCI_D0);
7483         if (err) {
7484                 tg3_full_unlock(tp);
7485                 return err;
7486         }
7487
7488         tg3_disable_ints(tp);
7489         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7490
7491         tg3_full_unlock(tp);
7492
7493         /* The placement of this call is tied
7494          * to the setup and use of Host TX descriptors.
7495          */
7496         err = tg3_alloc_consistent(tp);
7497         if (err)
7498                 return err;
7499
7500         if (tp->tg3_flags & TG3_FLAG_SUPPORT_MSI) {
7501                 /* All MSI supporting chips should support tagged
7502                  * status.  Assert that this is the case.
7503                  */
7504                 if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7505                         printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
7506                                "Not using MSI.\n", tp->dev->name);
7507                 } else if (pci_enable_msi(tp->pdev) == 0) {
7508                         u32 msi_mode;
7509
7510                         msi_mode = tr32(MSGINT_MODE);
7511                         tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
7512                         tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
7513                 }
7514         }
7515         err = tg3_request_irq(tp);
7516
7517         if (err) {
7518                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7519                         pci_disable_msi(tp->pdev);
7520                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7521                 }
7522                 tg3_free_consistent(tp);
7523                 return err;
7524         }
7525
7526         napi_enable(&tp->napi);
7527
7528         tg3_full_lock(tp, 0);
7529
7530         err = tg3_init_hw(tp, 1);
7531         if (err) {
7532                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7533                 tg3_free_rings(tp);
7534         } else {
7535                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7536                         tp->timer_offset = HZ;
7537                 else
7538                         tp->timer_offset = HZ / 10;
7539
7540                 BUG_ON(tp->timer_offset > HZ);
7541                 tp->timer_counter = tp->timer_multiplier =
7542                         (HZ / tp->timer_offset);
7543                 tp->asf_counter = tp->asf_multiplier =
7544                         ((HZ / tp->timer_offset) * 2);
7545
7546                 init_timer(&tp->timer);
7547                 tp->timer.expires = jiffies + tp->timer_offset;
7548                 tp->timer.data = (unsigned long) tp;
7549                 tp->timer.function = tg3_timer;
7550         }
7551
7552         tg3_full_unlock(tp);
7553
7554         if (err) {
7555                 napi_disable(&tp->napi);
7556                 free_irq(tp->pdev->irq, dev);
7557                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7558                         pci_disable_msi(tp->pdev);
7559                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7560                 }
7561                 tg3_free_consistent(tp);
7562                 return err;
7563         }
7564
7565         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7566                 err = tg3_test_msi(tp);
7567
7568                 if (err) {
7569                         tg3_full_lock(tp, 0);
7570
7571                         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7572                                 pci_disable_msi(tp->pdev);
7573                                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7574                         }
7575                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7576                         tg3_free_rings(tp);
7577                         tg3_free_consistent(tp);
7578
7579                         tg3_full_unlock(tp);
7580
7581                         napi_disable(&tp->napi);
7582
7583                         return err;
7584                 }
7585
7586                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7587                         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI) {
7588                                 u32 val = tr32(PCIE_TRANSACTION_CFG);
7589
7590                                 tw32(PCIE_TRANSACTION_CFG,
7591                                      val | PCIE_TRANS_CFG_1SHOT_MSI);
7592                         }
7593                 }
7594         }
7595
7596         tg3_full_lock(tp, 0);
7597
7598         add_timer(&tp->timer);
7599         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
7600         tg3_enable_ints(tp);
7601
7602         tg3_full_unlock(tp);
7603
7604         netif_start_queue(dev);
7605
7606         return 0;
7607 }
7608
7609 #if 0
7610 /*static*/ void tg3_dump_state(struct tg3 *tp)
7611 {
7612         u32 val32, val32_2, val32_3, val32_4, val32_5;
7613         u16 val16;
7614         int i;
7615
7616         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
7617         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
7618         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
7619                val16, val32);
7620
7621         /* MAC block */
7622         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
7623                tr32(MAC_MODE), tr32(MAC_STATUS));
7624         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
7625                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
7626         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
7627                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
7628         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
7629                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
7630
7631         /* Send data initiator control block */
7632         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
7633                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
7634         printk("       SNDDATAI_STATSCTRL[%08x]\n",
7635                tr32(SNDDATAI_STATSCTRL));
7636
7637         /* Send data completion control block */
7638         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
7639
7640         /* Send BD ring selector block */
7641         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
7642                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
7643
7644         /* Send BD initiator control block */
7645         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
7646                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
7647
7648         /* Send BD completion control block */
7649         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
7650
7651         /* Receive list placement control block */
7652         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
7653                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
7654         printk("       RCVLPC_STATSCTRL[%08x]\n",
7655                tr32(RCVLPC_STATSCTRL));
7656
7657         /* Receive data and receive BD initiator control block */
7658         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
7659                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
7660
7661         /* Receive data completion control block */
7662         printk("DEBUG: RCVDCC_MODE[%08x]\n",
7663                tr32(RCVDCC_MODE));
7664
7665         /* Receive BD initiator control block */
7666         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
7667                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
7668
7669         /* Receive BD completion control block */
7670         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
7671                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
7672
7673         /* Receive list selector control block */
7674         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
7675                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
7676
7677         /* Mbuf cluster free block */
7678         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
7679                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
7680
7681         /* Host coalescing control block */
7682         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
7683                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
7684         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
7685                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7686                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7687         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
7688                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7689                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7690         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
7691                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
7692         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
7693                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
7694
7695         /* Memory arbiter control block */
7696         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
7697                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
7698
7699         /* Buffer manager control block */
7700         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
7701                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
7702         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
7703                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
7704         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
7705                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
7706                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
7707                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
7708
7709         /* Read DMA control block */
7710         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
7711                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
7712
7713         /* Write DMA control block */
7714         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
7715                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
7716
7717         /* DMA completion block */
7718         printk("DEBUG: DMAC_MODE[%08x]\n",
7719                tr32(DMAC_MODE));
7720
7721         /* GRC block */
7722         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
7723                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
7724         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
7725                tr32(GRC_LOCAL_CTRL));
7726
7727         /* TG3_BDINFOs */
7728         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
7729                tr32(RCVDBDI_JUMBO_BD + 0x0),
7730                tr32(RCVDBDI_JUMBO_BD + 0x4),
7731                tr32(RCVDBDI_JUMBO_BD + 0x8),
7732                tr32(RCVDBDI_JUMBO_BD + 0xc));
7733         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
7734                tr32(RCVDBDI_STD_BD + 0x0),
7735                tr32(RCVDBDI_STD_BD + 0x4),
7736                tr32(RCVDBDI_STD_BD + 0x8),
7737                tr32(RCVDBDI_STD_BD + 0xc));
7738         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
7739                tr32(RCVDBDI_MINI_BD + 0x0),
7740                tr32(RCVDBDI_MINI_BD + 0x4),
7741                tr32(RCVDBDI_MINI_BD + 0x8),
7742                tr32(RCVDBDI_MINI_BD + 0xc));
7743
7744         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
7745         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
7746         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
7747         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
7748         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
7749                val32, val32_2, val32_3, val32_4);
7750
7751         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
7752         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
7753         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
7754         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
7755         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
7756                val32, val32_2, val32_3, val32_4);
7757
7758         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
7759         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
7760         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
7761         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
7762         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
7763         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
7764                val32, val32_2, val32_3, val32_4, val32_5);
7765
7766         /* SW status block */
7767         printk("DEBUG: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
7768                tp->hw_status->status,
7769                tp->hw_status->status_tag,
7770                tp->hw_status->rx_jumbo_consumer,
7771                tp->hw_status->rx_consumer,
7772                tp->hw_status->rx_mini_consumer,
7773                tp->hw_status->idx[0].rx_producer,
7774                tp->hw_status->idx[0].tx_consumer);
7775
7776         /* SW statistics block */
7777         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
7778                ((u32 *)tp->hw_stats)[0],
7779                ((u32 *)tp->hw_stats)[1],
7780                ((u32 *)tp->hw_stats)[2],
7781                ((u32 *)tp->hw_stats)[3]);
7782
7783         /* Mailboxes */
7784         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
7785                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
7786                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
7787                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
7788                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
7789
7790         /* NIC side send descriptors. */
7791         for (i = 0; i < 6; i++) {
7792                 unsigned long txd;
7793
7794                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
7795                         + (i * sizeof(struct tg3_tx_buffer_desc));
7796                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
7797                        i,
7798                        readl(txd + 0x0), readl(txd + 0x4),
7799                        readl(txd + 0x8), readl(txd + 0xc));
7800         }
7801
7802         /* NIC side RX descriptors. */
7803         for (i = 0; i < 6; i++) {
7804                 unsigned long rxd;
7805
7806                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
7807                         + (i * sizeof(struct tg3_rx_buffer_desc));
7808                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
7809                        i,
7810                        readl(rxd + 0x0), readl(rxd + 0x4),
7811                        readl(rxd + 0x8), readl(rxd + 0xc));
7812                 rxd += (4 * sizeof(u32));
7813                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
7814                        i,
7815                        readl(rxd + 0x0), readl(rxd + 0x4),
7816                        readl(rxd + 0x8), readl(rxd + 0xc));
7817         }
7818
7819         for (i = 0; i < 6; i++) {
7820                 unsigned long rxd;
7821
7822                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
7823                         + (i * sizeof(struct tg3_rx_buffer_desc));
7824                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
7825                        i,
7826                        readl(rxd + 0x0), readl(rxd + 0x4),
7827                        readl(rxd + 0x8), readl(rxd + 0xc));
7828                 rxd += (4 * sizeof(u32));
7829                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
7830                        i,
7831                        readl(rxd + 0x0), readl(rxd + 0x4),
7832                        readl(rxd + 0x8), readl(rxd + 0xc));
7833         }
7834 }
7835 #endif
7836
7837 static struct net_device_stats *tg3_get_stats(struct net_device *);
7838 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
7839
7840 static int tg3_close(struct net_device *dev)
7841 {
7842         struct tg3 *tp = netdev_priv(dev);
7843
7844         napi_disable(&tp->napi);
7845         cancel_work_sync(&tp->reset_task);
7846
7847         netif_stop_queue(dev);
7848
7849         del_timer_sync(&tp->timer);
7850
7851         tg3_full_lock(tp, 1);
7852 #if 0
7853         tg3_dump_state(tp);
7854 #endif
7855
7856         tg3_disable_ints(tp);
7857
7858         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7859         tg3_free_rings(tp);
7860         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7861
7862         tg3_full_unlock(tp);
7863
7864         free_irq(tp->pdev->irq, dev);
7865         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7866                 pci_disable_msi(tp->pdev);
7867                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7868         }
7869
7870         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
7871                sizeof(tp->net_stats_prev));
7872         memcpy(&tp->estats_prev, tg3_get_estats(tp),
7873                sizeof(tp->estats_prev));
7874
7875         tg3_free_consistent(tp);
7876
7877         tg3_set_power_state(tp, PCI_D3hot);
7878
7879         netif_carrier_off(tp->dev);
7880
7881         return 0;
7882 }
7883
7884 static inline unsigned long get_stat64(tg3_stat64_t *val)
7885 {
7886         unsigned long ret;
7887
7888 #if (BITS_PER_LONG == 32)
7889         ret = val->low;
7890 #else
7891         ret = ((u64)val->high << 32) | ((u64)val->low);
7892 #endif
7893         return ret;
7894 }
7895
7896 static unsigned long calc_crc_errors(struct tg3 *tp)
7897 {
7898         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7899
7900         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7901             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
7902              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
7903                 u32 val;
7904
7905                 spin_lock_bh(&tp->lock);
7906                 if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
7907                         tg3_writephy(tp, MII_TG3_TEST1,
7908                                      val | MII_TG3_TEST1_CRC_EN);
7909                         tg3_readphy(tp, 0x14, &val);
7910                 } else
7911                         val = 0;
7912                 spin_unlock_bh(&tp->lock);
7913
7914                 tp->phy_crc_errors += val;
7915
7916                 return tp->phy_crc_errors;
7917         }
7918
7919         return get_stat64(&hw_stats->rx_fcs_errors);
7920 }
7921
7922 #define ESTAT_ADD(member) \
7923         estats->member =        old_estats->member + \
7924                                 get_stat64(&hw_stats->member)
7925
7926 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
7927 {
7928         struct tg3_ethtool_stats *estats = &tp->estats;
7929         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
7930         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7931
7932         if (!hw_stats)
7933                 return old_estats;
7934
7935         ESTAT_ADD(rx_octets);
7936         ESTAT_ADD(rx_fragments);
7937         ESTAT_ADD(rx_ucast_packets);
7938         ESTAT_ADD(rx_mcast_packets);
7939         ESTAT_ADD(rx_bcast_packets);
7940         ESTAT_ADD(rx_fcs_errors);
7941         ESTAT_ADD(rx_align_errors);
7942         ESTAT_ADD(rx_xon_pause_rcvd);
7943         ESTAT_ADD(rx_xoff_pause_rcvd);
7944         ESTAT_ADD(rx_mac_ctrl_rcvd);
7945         ESTAT_ADD(rx_xoff_entered);
7946         ESTAT_ADD(rx_frame_too_long_errors);
7947         ESTAT_ADD(rx_jabbers);
7948         ESTAT_ADD(rx_undersize_packets);
7949         ESTAT_ADD(rx_in_length_errors);
7950         ESTAT_ADD(rx_out_length_errors);
7951         ESTAT_ADD(rx_64_or_less_octet_packets);
7952         ESTAT_ADD(rx_65_to_127_octet_packets);
7953         ESTAT_ADD(rx_128_to_255_octet_packets);
7954         ESTAT_ADD(rx_256_to_511_octet_packets);
7955         ESTAT_ADD(rx_512_to_1023_octet_packets);
7956         ESTAT_ADD(rx_1024_to_1522_octet_packets);
7957         ESTAT_ADD(rx_1523_to_2047_octet_packets);
7958         ESTAT_ADD(rx_2048_to_4095_octet_packets);
7959         ESTAT_ADD(rx_4096_to_8191_octet_packets);
7960         ESTAT_ADD(rx_8192_to_9022_octet_packets);
7961
7962         ESTAT_ADD(tx_octets);
7963         ESTAT_ADD(tx_collisions);
7964         ESTAT_ADD(tx_xon_sent);
7965         ESTAT_ADD(tx_xoff_sent);
7966         ESTAT_ADD(tx_flow_control);
7967         ESTAT_ADD(tx_mac_errors);
7968         ESTAT_ADD(tx_single_collisions);
7969         ESTAT_ADD(tx_mult_collisions);
7970         ESTAT_ADD(tx_deferred);
7971         ESTAT_ADD(tx_excessive_collisions);
7972         ESTAT_ADD(tx_late_collisions);
7973         ESTAT_ADD(tx_collide_2times);
7974         ESTAT_ADD(tx_collide_3times);
7975         ESTAT_ADD(tx_collide_4times);
7976         ESTAT_ADD(tx_collide_5times);
7977         ESTAT_ADD(tx_collide_6times);
7978         ESTAT_ADD(tx_collide_7times);
7979         ESTAT_ADD(tx_collide_8times);
7980         ESTAT_ADD(tx_collide_9times);
7981         ESTAT_ADD(tx_collide_10times);
7982         ESTAT_ADD(tx_collide_11times);
7983         ESTAT_ADD(tx_collide_12times);
7984         ESTAT_ADD(tx_collide_13times);
7985         ESTAT_ADD(tx_collide_14times);
7986         ESTAT_ADD(tx_collide_15times);
7987         ESTAT_ADD(tx_ucast_packets);
7988         ESTAT_ADD(tx_mcast_packets);
7989         ESTAT_ADD(tx_bcast_packets);
7990         ESTAT_ADD(tx_carrier_sense_errors);
7991         ESTAT_ADD(tx_discards);
7992         ESTAT_ADD(tx_errors);
7993
7994         ESTAT_ADD(dma_writeq_full);
7995         ESTAT_ADD(dma_write_prioq_full);
7996         ESTAT_ADD(rxbds_empty);
7997         ESTAT_ADD(rx_discards);
7998         ESTAT_ADD(rx_errors);
7999         ESTAT_ADD(rx_threshold_hit);
8000
8001         ESTAT_ADD(dma_readq_full);
8002         ESTAT_ADD(dma_read_prioq_full);
8003         ESTAT_ADD(tx_comp_queue_full);
8004
8005         ESTAT_ADD(ring_set_send_prod_index);
8006         ESTAT_ADD(ring_status_update);
8007         ESTAT_ADD(nic_irqs);
8008         ESTAT_ADD(nic_avoided_irqs);
8009         ESTAT_ADD(nic_tx_threshold_hit);
8010
8011         return estats;
8012 }
8013
8014 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
8015 {
8016         struct tg3 *tp = netdev_priv(dev);
8017         struct net_device_stats *stats = &tp->net_stats;
8018         struct net_device_stats *old_stats = &tp->net_stats_prev;
8019         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8020
8021         if (!hw_stats)
8022                 return old_stats;
8023
8024         stats->rx_packets = old_stats->rx_packets +
8025                 get_stat64(&hw_stats->rx_ucast_packets) +
8026                 get_stat64(&hw_stats->rx_mcast_packets) +
8027                 get_stat64(&hw_stats->rx_bcast_packets);
8028
8029         stats->tx_packets = old_stats->tx_packets +
8030                 get_stat64(&hw_stats->tx_ucast_packets) +
8031                 get_stat64(&hw_stats->tx_mcast_packets) +
8032                 get_stat64(&hw_stats->tx_bcast_packets);
8033
8034         stats->rx_bytes = old_stats->rx_bytes +
8035                 get_stat64(&hw_stats->rx_octets);
8036         stats->tx_bytes = old_stats->tx_bytes +
8037                 get_stat64(&hw_stats->tx_octets);
8038
8039         stats->rx_errors = old_stats->rx_errors +
8040                 get_stat64(&hw_stats->rx_errors);
8041         stats->tx_errors = old_stats->tx_errors +
8042                 get_stat64(&hw_stats->tx_errors) +
8043                 get_stat64(&hw_stats->tx_mac_errors) +
8044                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
8045                 get_stat64(&hw_stats->tx_discards);
8046
8047         stats->multicast = old_stats->multicast +
8048                 get_stat64(&hw_stats->rx_mcast_packets);
8049         stats->collisions = old_stats->collisions +
8050                 get_stat64(&hw_stats->tx_collisions);
8051
8052         stats->rx_length_errors = old_stats->rx_length_errors +
8053                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
8054                 get_stat64(&hw_stats->rx_undersize_packets);
8055
8056         stats->rx_over_errors = old_stats->rx_over_errors +
8057                 get_stat64(&hw_stats->rxbds_empty);
8058         stats->rx_frame_errors = old_stats->rx_frame_errors +
8059                 get_stat64(&hw_stats->rx_align_errors);
8060         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
8061                 get_stat64(&hw_stats->tx_discards);
8062         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
8063                 get_stat64(&hw_stats->tx_carrier_sense_errors);
8064
8065         stats->rx_crc_errors = old_stats->rx_crc_errors +
8066                 calc_crc_errors(tp);
8067
8068         stats->rx_missed_errors = old_stats->rx_missed_errors +
8069                 get_stat64(&hw_stats->rx_discards);
8070
8071         return stats;
8072 }
8073
8074 static inline u32 calc_crc(unsigned char *buf, int len)
8075 {
8076         u32 reg;
8077         u32 tmp;
8078         int j, k;
8079
8080         reg = 0xffffffff;
8081
8082         for (j = 0; j < len; j++) {
8083                 reg ^= buf[j];
8084
8085                 for (k = 0; k < 8; k++) {
8086                         tmp = reg & 0x01;
8087
8088                         reg >>= 1;
8089
8090                         if (tmp) {
8091                                 reg ^= 0xedb88320;
8092                         }
8093                 }
8094         }
8095
8096         return ~reg;
8097 }
8098
8099 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
8100 {
8101         /* accept or reject all multicast frames */
8102         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
8103         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
8104         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
8105         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
8106 }
8107
8108 static void __tg3_set_rx_mode(struct net_device *dev)
8109 {
8110         struct tg3 *tp = netdev_priv(dev);
8111         u32 rx_mode;
8112
8113         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
8114                                   RX_MODE_KEEP_VLAN_TAG);
8115
8116         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
8117          * flag clear.
8118          */
8119 #if TG3_VLAN_TAG_USED
8120         if (!tp->vlgrp &&
8121             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8122                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8123 #else
8124         /* By definition, VLAN is disabled always in this
8125          * case.
8126          */
8127         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8128                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8129 #endif
8130
8131         if (dev->flags & IFF_PROMISC) {
8132                 /* Promiscuous mode. */
8133                 rx_mode |= RX_MODE_PROMISC;
8134         } else if (dev->flags & IFF_ALLMULTI) {
8135                 /* Accept all multicast. */
8136                 tg3_set_multi (tp, 1);
8137         } else if (dev->mc_count < 1) {
8138                 /* Reject all multicast. */
8139                 tg3_set_multi (tp, 0);
8140         } else {
8141                 /* Accept one or more multicast(s). */
8142                 struct dev_mc_list *mclist;
8143                 unsigned int i;
8144                 u32 mc_filter[4] = { 0, };
8145                 u32 regidx;
8146                 u32 bit;
8147                 u32 crc;
8148
8149                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
8150                      i++, mclist = mclist->next) {
8151
8152                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
8153                         bit = ~crc & 0x7f;
8154                         regidx = (bit & 0x60) >> 5;
8155                         bit &= 0x1f;
8156                         mc_filter[regidx] |= (1 << bit);
8157                 }
8158
8159                 tw32(MAC_HASH_REG_0, mc_filter[0]);
8160                 tw32(MAC_HASH_REG_1, mc_filter[1]);
8161                 tw32(MAC_HASH_REG_2, mc_filter[2]);
8162                 tw32(MAC_HASH_REG_3, mc_filter[3]);
8163         }
8164
8165         if (rx_mode != tp->rx_mode) {
8166                 tp->rx_mode = rx_mode;
8167                 tw32_f(MAC_RX_MODE, rx_mode);
8168                 udelay(10);
8169         }
8170 }
8171
8172 static void tg3_set_rx_mode(struct net_device *dev)
8173 {
8174         struct tg3 *tp = netdev_priv(dev);
8175
8176         if (!netif_running(dev))
8177                 return;
8178
8179         tg3_full_lock(tp, 0);
8180         __tg3_set_rx_mode(dev);
8181         tg3_full_unlock(tp);
8182 }
8183
8184 #define TG3_REGDUMP_LEN         (32 * 1024)
8185
8186 static int tg3_get_regs_len(struct net_device *dev)
8187 {
8188         return TG3_REGDUMP_LEN;
8189 }
8190
8191 static void tg3_get_regs(struct net_device *dev,
8192                 struct ethtool_regs *regs, void *_p)
8193 {
8194         u32 *p = _p;
8195         struct tg3 *tp = netdev_priv(dev);
8196         u8 *orig_p = _p;
8197         int i;
8198
8199         regs->version = 0;
8200
8201         memset(p, 0, TG3_REGDUMP_LEN);
8202
8203         if (tp->link_config.phy_is_low_power)
8204                 return;
8205
8206         tg3_full_lock(tp, 0);
8207
8208 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
8209 #define GET_REG32_LOOP(base,len)                \
8210 do {    p = (u32 *)(orig_p + (base));           \
8211         for (i = 0; i < len; i += 4)            \
8212                 __GET_REG32((base) + i);        \
8213 } while (0)
8214 #define GET_REG32_1(reg)                        \
8215 do {    p = (u32 *)(orig_p + (reg));            \
8216         __GET_REG32((reg));                     \
8217 } while (0)
8218
8219         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
8220         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
8221         GET_REG32_LOOP(MAC_MODE, 0x4f0);
8222         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
8223         GET_REG32_1(SNDDATAC_MODE);
8224         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
8225         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
8226         GET_REG32_1(SNDBDC_MODE);
8227         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
8228         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
8229         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
8230         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
8231         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
8232         GET_REG32_1(RCVDCC_MODE);
8233         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
8234         GET_REG32_LOOP(RCVCC_MODE, 0x14);
8235         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
8236         GET_REG32_1(MBFREE_MODE);
8237         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
8238         GET_REG32_LOOP(MEMARB_MODE, 0x10);
8239         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
8240         GET_REG32_LOOP(RDMAC_MODE, 0x08);
8241         GET_REG32_LOOP(WDMAC_MODE, 0x08);
8242         GET_REG32_1(RX_CPU_MODE);
8243         GET_REG32_1(RX_CPU_STATE);
8244         GET_REG32_1(RX_CPU_PGMCTR);
8245         GET_REG32_1(RX_CPU_HWBKPT);
8246         GET_REG32_1(TX_CPU_MODE);
8247         GET_REG32_1(TX_CPU_STATE);
8248         GET_REG32_1(TX_CPU_PGMCTR);
8249         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
8250         GET_REG32_LOOP(FTQ_RESET, 0x120);
8251         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
8252         GET_REG32_1(DMAC_MODE);
8253         GET_REG32_LOOP(GRC_MODE, 0x4c);
8254         if (tp->tg3_flags & TG3_FLAG_NVRAM)
8255                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
8256
8257 #undef __GET_REG32
8258 #undef GET_REG32_LOOP
8259 #undef GET_REG32_1
8260
8261         tg3_full_unlock(tp);
8262 }
8263
8264 static int tg3_get_eeprom_len(struct net_device *dev)
8265 {
8266         struct tg3 *tp = netdev_priv(dev);
8267
8268         return tp->nvram_size;
8269 }
8270
8271 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val);
8272 static int tg3_nvram_read_le(struct tg3 *tp, u32 offset, __le32 *val);
8273 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val);
8274
8275 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8276 {
8277         struct tg3 *tp = netdev_priv(dev);
8278         int ret;
8279         u8  *pd;
8280         u32 i, offset, len, b_offset, b_count;
8281         __le32 val;
8282
8283         if (tp->link_config.phy_is_low_power)
8284                 return -EAGAIN;
8285
8286         offset = eeprom->offset;
8287         len = eeprom->len;
8288         eeprom->len = 0;
8289
8290         eeprom->magic = TG3_EEPROM_MAGIC;
8291
8292         if (offset & 3) {
8293                 /* adjustments to start on required 4 byte boundary */
8294                 b_offset = offset & 3;
8295                 b_count = 4 - b_offset;
8296                 if (b_count > len) {
8297                         /* i.e. offset=1 len=2 */
8298                         b_count = len;
8299                 }
8300                 ret = tg3_nvram_read_le(tp, offset-b_offset, &val);
8301                 if (ret)
8302                         return ret;
8303                 memcpy(data, ((char*)&val) + b_offset, b_count);
8304                 len -= b_count;
8305                 offset += b_count;
8306                 eeprom->len += b_count;
8307         }
8308
8309         /* read bytes upto the last 4 byte boundary */
8310         pd = &data[eeprom->len];
8311         for (i = 0; i < (len - (len & 3)); i += 4) {
8312                 ret = tg3_nvram_read_le(tp, offset + i, &val);
8313                 if (ret) {
8314                         eeprom->len += i;
8315                         return ret;
8316                 }
8317                 memcpy(pd + i, &val, 4);
8318         }
8319         eeprom->len += i;
8320
8321         if (len & 3) {
8322                 /* read last bytes not ending on 4 byte boundary */
8323                 pd = &data[eeprom->len];
8324                 b_count = len & 3;
8325                 b_offset = offset + len - b_count;
8326                 ret = tg3_nvram_read_le(tp, b_offset, &val);
8327                 if (ret)
8328                         return ret;
8329                 memcpy(pd, &val, b_count);
8330                 eeprom->len += b_count;
8331         }
8332         return 0;
8333 }
8334
8335 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
8336
8337 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8338 {
8339         struct tg3 *tp = netdev_priv(dev);
8340         int ret;
8341         u32 offset, len, b_offset, odd_len;
8342         u8 *buf;
8343         __le32 start, end;
8344
8345         if (tp->link_config.phy_is_low_power)
8346                 return -EAGAIN;
8347
8348         if (eeprom->magic != TG3_EEPROM_MAGIC)
8349                 return -EINVAL;
8350
8351         offset = eeprom->offset;
8352         len = eeprom->len;
8353
8354         if ((b_offset = (offset & 3))) {
8355                 /* adjustments to start on required 4 byte boundary */
8356                 ret = tg3_nvram_read_le(tp, offset-b_offset, &start);
8357                 if (ret)
8358                         return ret;
8359                 len += b_offset;
8360                 offset &= ~3;
8361                 if (len < 4)
8362                         len = 4;
8363         }
8364
8365         odd_len = 0;
8366         if (len & 3) {
8367                 /* adjustments to end on required 4 byte boundary */
8368                 odd_len = 1;
8369                 len = (len + 3) & ~3;
8370                 ret = tg3_nvram_read_le(tp, offset+len-4, &end);
8371                 if (ret)
8372                         return ret;
8373         }
8374
8375         buf = data;
8376         if (b_offset || odd_len) {
8377                 buf = kmalloc(len, GFP_KERNEL);
8378                 if (!buf)
8379                         return -ENOMEM;
8380                 if (b_offset)
8381                         memcpy(buf, &start, 4);
8382                 if (odd_len)
8383                         memcpy(buf+len-4, &end, 4);
8384                 memcpy(buf + b_offset, data, eeprom->len);
8385         }
8386
8387         ret = tg3_nvram_write_block(tp, offset, len, buf);
8388
8389         if (buf != data)
8390                 kfree(buf);
8391
8392         return ret;
8393 }
8394
8395 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8396 {
8397         struct tg3 *tp = netdev_priv(dev);
8398
8399         cmd->supported = (SUPPORTED_Autoneg);
8400
8401         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
8402                 cmd->supported |= (SUPPORTED_1000baseT_Half |
8403                                    SUPPORTED_1000baseT_Full);
8404
8405         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
8406                 cmd->supported |= (SUPPORTED_100baseT_Half |
8407                                   SUPPORTED_100baseT_Full |
8408                                   SUPPORTED_10baseT_Half |
8409                                   SUPPORTED_10baseT_Full |
8410                                   SUPPORTED_TP);
8411                 cmd->port = PORT_TP;
8412         } else {
8413                 cmd->supported |= SUPPORTED_FIBRE;
8414                 cmd->port = PORT_FIBRE;
8415         }
8416
8417         cmd->advertising = tp->link_config.advertising;
8418         if (netif_running(dev)) {
8419                 cmd->speed = tp->link_config.active_speed;
8420                 cmd->duplex = tp->link_config.active_duplex;
8421         }
8422         cmd->phy_address = PHY_ADDR;
8423         cmd->transceiver = 0;
8424         cmd->autoneg = tp->link_config.autoneg;
8425         cmd->maxtxpkt = 0;
8426         cmd->maxrxpkt = 0;
8427         return 0;
8428 }
8429
8430 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8431 {
8432         struct tg3 *tp = netdev_priv(dev);
8433
8434         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
8435                 /* These are the only valid advertisement bits allowed.  */
8436                 if (cmd->autoneg == AUTONEG_ENABLE &&
8437                     (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
8438                                           ADVERTISED_1000baseT_Full |
8439                                           ADVERTISED_Autoneg |
8440                                           ADVERTISED_FIBRE)))
8441                         return -EINVAL;
8442                 /* Fiber can only do SPEED_1000.  */
8443                 else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8444                          (cmd->speed != SPEED_1000))
8445                         return -EINVAL;
8446         /* Copper cannot force SPEED_1000.  */
8447         } else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8448                    (cmd->speed == SPEED_1000))
8449                 return -EINVAL;
8450         else if ((cmd->speed == SPEED_1000) &&
8451                  (tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
8452                 return -EINVAL;
8453
8454         tg3_full_lock(tp, 0);
8455
8456         tp->link_config.autoneg = cmd->autoneg;
8457         if (cmd->autoneg == AUTONEG_ENABLE) {
8458                 tp->link_config.advertising = (cmd->advertising |
8459                                               ADVERTISED_Autoneg);
8460                 tp->link_config.speed = SPEED_INVALID;
8461                 tp->link_config.duplex = DUPLEX_INVALID;
8462         } else {
8463                 tp->link_config.advertising = 0;
8464                 tp->link_config.speed = cmd->speed;
8465                 tp->link_config.duplex = cmd->duplex;
8466         }
8467
8468         tp->link_config.orig_speed = tp->link_config.speed;
8469         tp->link_config.orig_duplex = tp->link_config.duplex;
8470         tp->link_config.orig_autoneg = tp->link_config.autoneg;
8471
8472         if (netif_running(dev))
8473                 tg3_setup_phy(tp, 1);
8474
8475         tg3_full_unlock(tp);
8476
8477         return 0;
8478 }
8479
8480 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
8481 {
8482         struct tg3 *tp = netdev_priv(dev);
8483
8484         strcpy(info->driver, DRV_MODULE_NAME);
8485         strcpy(info->version, DRV_MODULE_VERSION);
8486         strcpy(info->fw_version, tp->fw_ver);
8487         strcpy(info->bus_info, pci_name(tp->pdev));
8488 }
8489
8490 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8491 {
8492         struct tg3 *tp = netdev_priv(dev);
8493
8494         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
8495                 wol->supported = WAKE_MAGIC;
8496         else
8497                 wol->supported = 0;
8498         wol->wolopts = 0;
8499         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)
8500                 wol->wolopts = WAKE_MAGIC;
8501         memset(&wol->sopass, 0, sizeof(wol->sopass));
8502 }
8503
8504 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8505 {
8506         struct tg3 *tp = netdev_priv(dev);
8507
8508         if (wol->wolopts & ~WAKE_MAGIC)
8509                 return -EINVAL;
8510         if ((wol->wolopts & WAKE_MAGIC) &&
8511             !(tp->tg3_flags & TG3_FLAG_WOL_CAP))
8512                 return -EINVAL;
8513
8514         spin_lock_bh(&tp->lock);
8515         if (wol->wolopts & WAKE_MAGIC)
8516                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
8517         else
8518                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
8519         spin_unlock_bh(&tp->lock);
8520
8521         return 0;
8522 }
8523
8524 static u32 tg3_get_msglevel(struct net_device *dev)
8525 {
8526         struct tg3 *tp = netdev_priv(dev);
8527         return tp->msg_enable;
8528 }
8529
8530 static void tg3_set_msglevel(struct net_device *dev, u32 value)
8531 {
8532         struct tg3 *tp = netdev_priv(dev);
8533         tp->msg_enable = value;
8534 }
8535
8536 static int tg3_set_tso(struct net_device *dev, u32 value)
8537 {
8538         struct tg3 *tp = netdev_priv(dev);
8539
8540         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
8541                 if (value)
8542                         return -EINVAL;
8543                 return 0;
8544         }
8545         if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
8546             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)) {
8547                 if (value) {
8548                         dev->features |= NETIF_F_TSO6;
8549                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8550                                 dev->features |= NETIF_F_TSO_ECN;
8551                 } else
8552                         dev->features &= ~(NETIF_F_TSO6 | NETIF_F_TSO_ECN);
8553         }
8554         return ethtool_op_set_tso(dev, value);
8555 }
8556
8557 static int tg3_nway_reset(struct net_device *dev)
8558 {
8559         struct tg3 *tp = netdev_priv(dev);
8560         u32 bmcr;
8561         int r;
8562
8563         if (!netif_running(dev))
8564                 return -EAGAIN;
8565
8566         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8567                 return -EINVAL;
8568
8569         spin_lock_bh(&tp->lock);
8570         r = -EINVAL;
8571         tg3_readphy(tp, MII_BMCR, &bmcr);
8572         if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
8573             ((bmcr & BMCR_ANENABLE) ||
8574              (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
8575                 tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
8576                                            BMCR_ANENABLE);
8577                 r = 0;
8578         }
8579         spin_unlock_bh(&tp->lock);
8580
8581         return r;
8582 }
8583
8584 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8585 {
8586         struct tg3 *tp = netdev_priv(dev);
8587
8588         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
8589         ering->rx_mini_max_pending = 0;
8590         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8591                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
8592         else
8593                 ering->rx_jumbo_max_pending = 0;
8594
8595         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
8596
8597         ering->rx_pending = tp->rx_pending;
8598         ering->rx_mini_pending = 0;
8599         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8600                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
8601         else
8602                 ering->rx_jumbo_pending = 0;
8603
8604         ering->tx_pending = tp->tx_pending;
8605 }
8606
8607 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8608 {
8609         struct tg3 *tp = netdev_priv(dev);
8610         int irq_sync = 0, err = 0;
8611
8612         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
8613             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
8614             (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
8615             (ering->tx_pending <= MAX_SKB_FRAGS) ||
8616             ((tp->tg3_flags2 & TG3_FLG2_TSO_BUG) &&
8617              (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
8618                 return -EINVAL;
8619
8620         if (netif_running(dev)) {
8621                 tg3_netif_stop(tp);
8622                 irq_sync = 1;
8623         }
8624
8625         tg3_full_lock(tp, irq_sync);
8626
8627         tp->rx_pending = ering->rx_pending;
8628
8629         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
8630             tp->rx_pending > 63)
8631                 tp->rx_pending = 63;
8632         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
8633         tp->tx_pending = ering->tx_pending;
8634
8635         if (netif_running(dev)) {
8636                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8637                 err = tg3_restart_hw(tp, 1);
8638                 if (!err)
8639                         tg3_netif_start(tp);
8640         }
8641
8642         tg3_full_unlock(tp);
8643
8644         return err;
8645 }
8646
8647 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8648 {
8649         struct tg3 *tp = netdev_priv(dev);
8650
8651         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
8652
8653         if (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_RX)
8654                 epause->rx_pause = 1;
8655         else
8656                 epause->rx_pause = 0;
8657
8658         if (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_TX)
8659                 epause->tx_pause = 1;
8660         else
8661                 epause->tx_pause = 0;
8662 }
8663
8664 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8665 {
8666         struct tg3 *tp = netdev_priv(dev);
8667         int irq_sync = 0, err = 0;
8668
8669         if (netif_running(dev)) {
8670                 tg3_netif_stop(tp);
8671                 irq_sync = 1;
8672         }
8673
8674         tg3_full_lock(tp, irq_sync);
8675
8676         if (epause->autoneg)
8677                 tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
8678         else
8679                 tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
8680         if (epause->rx_pause)
8681                 tp->link_config.flowctrl |= TG3_FLOW_CTRL_RX;
8682         else
8683                 tp->link_config.flowctrl &= ~TG3_FLOW_CTRL_RX;
8684         if (epause->tx_pause)
8685                 tp->link_config.flowctrl |= TG3_FLOW_CTRL_TX;
8686         else
8687                 tp->link_config.flowctrl &= ~TG3_FLOW_CTRL_TX;
8688
8689         if (netif_running(dev)) {
8690                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8691                 err = tg3_restart_hw(tp, 1);
8692                 if (!err)
8693                         tg3_netif_start(tp);
8694         }
8695
8696         tg3_full_unlock(tp);
8697
8698         return err;
8699 }
8700
8701 static u32 tg3_get_rx_csum(struct net_device *dev)
8702 {
8703         struct tg3 *tp = netdev_priv(dev);
8704         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
8705 }
8706
8707 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
8708 {
8709         struct tg3 *tp = netdev_priv(dev);
8710
8711         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8712                 if (data != 0)
8713                         return -EINVAL;
8714                 return 0;
8715         }
8716
8717         spin_lock_bh(&tp->lock);
8718         if (data)
8719                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
8720         else
8721                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
8722         spin_unlock_bh(&tp->lock);
8723
8724         return 0;
8725 }
8726
8727 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
8728 {
8729         struct tg3 *tp = netdev_priv(dev);
8730
8731         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8732                 if (data != 0)
8733                         return -EINVAL;
8734                 return 0;
8735         }
8736
8737         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8738             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
8739             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
8740             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8741                 ethtool_op_set_tx_ipv6_csum(dev, data);
8742         else
8743                 ethtool_op_set_tx_csum(dev, data);
8744
8745         return 0;
8746 }
8747
8748 static int tg3_get_sset_count (struct net_device *dev, int sset)
8749 {
8750         switch (sset) {
8751         case ETH_SS_TEST:
8752                 return TG3_NUM_TEST;
8753         case ETH_SS_STATS:
8754                 return TG3_NUM_STATS;
8755         default:
8756                 return -EOPNOTSUPP;
8757         }
8758 }
8759
8760 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
8761 {
8762         switch (stringset) {
8763         case ETH_SS_STATS:
8764                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
8765                 break;
8766         case ETH_SS_TEST:
8767                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
8768                 break;
8769         default:
8770                 WARN_ON(1);     /* we need a WARN() */
8771                 break;
8772         }
8773 }
8774
8775 static int tg3_phys_id(struct net_device *dev, u32 data)
8776 {
8777         struct tg3 *tp = netdev_priv(dev);
8778         int i;
8779
8780         if (!netif_running(tp->dev))
8781                 return -EAGAIN;
8782
8783         if (data == 0)
8784                 data = 2;
8785
8786         for (i = 0; i < (data * 2); i++) {
8787                 if ((i % 2) == 0)
8788                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8789                                            LED_CTRL_1000MBPS_ON |
8790                                            LED_CTRL_100MBPS_ON |
8791                                            LED_CTRL_10MBPS_ON |
8792                                            LED_CTRL_TRAFFIC_OVERRIDE |
8793                                            LED_CTRL_TRAFFIC_BLINK |
8794                                            LED_CTRL_TRAFFIC_LED);
8795
8796                 else
8797                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8798                                            LED_CTRL_TRAFFIC_OVERRIDE);
8799
8800                 if (msleep_interruptible(500))
8801                         break;
8802         }
8803         tw32(MAC_LED_CTRL, tp->led_ctrl);
8804         return 0;
8805 }
8806
8807 static void tg3_get_ethtool_stats (struct net_device *dev,
8808                                    struct ethtool_stats *estats, u64 *tmp_stats)
8809 {
8810         struct tg3 *tp = netdev_priv(dev);
8811         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
8812 }
8813
8814 #define NVRAM_TEST_SIZE 0x100
8815 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE   0x14
8816 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE   0x18
8817 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE   0x1c
8818 #define NVRAM_SELFBOOT_HW_SIZE 0x20
8819 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
8820
8821 static int tg3_test_nvram(struct tg3 *tp)
8822 {
8823         u32 csum, magic;
8824         __le32 *buf;
8825         int i, j, k, err = 0, size;
8826
8827         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8828                 return -EIO;
8829
8830         if (magic == TG3_EEPROM_MAGIC)
8831                 size = NVRAM_TEST_SIZE;
8832         else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
8833                 if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
8834                     TG3_EEPROM_SB_FORMAT_1) {
8835                         switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
8836                         case TG3_EEPROM_SB_REVISION_0:
8837                                 size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
8838                                 break;
8839                         case TG3_EEPROM_SB_REVISION_2:
8840                                 size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
8841                                 break;
8842                         case TG3_EEPROM_SB_REVISION_3:
8843                                 size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
8844                                 break;
8845                         default:
8846                                 return 0;
8847                         }
8848                 } else
8849                         return 0;
8850         } else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
8851                 size = NVRAM_SELFBOOT_HW_SIZE;
8852         else
8853                 return -EIO;
8854
8855         buf = kmalloc(size, GFP_KERNEL);
8856         if (buf == NULL)
8857                 return -ENOMEM;
8858
8859         err = -EIO;
8860         for (i = 0, j = 0; i < size; i += 4, j++) {
8861                 if ((err = tg3_nvram_read_le(tp, i, &buf[j])) != 0)
8862                         break;
8863         }
8864         if (i < size)
8865                 goto out;
8866
8867         /* Selfboot format */
8868         magic = swab32(le32_to_cpu(buf[0]));
8869         if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
8870             TG3_EEPROM_MAGIC_FW) {
8871                 u8 *buf8 = (u8 *) buf, csum8 = 0;
8872
8873                 if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
8874                     TG3_EEPROM_SB_REVISION_2) {
8875                         /* For rev 2, the csum doesn't include the MBA. */
8876                         for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
8877                                 csum8 += buf8[i];
8878                         for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
8879                                 csum8 += buf8[i];
8880                 } else {
8881                         for (i = 0; i < size; i++)
8882                                 csum8 += buf8[i];
8883                 }
8884
8885                 if (csum8 == 0) {
8886                         err = 0;
8887                         goto out;
8888                 }
8889
8890                 err = -EIO;
8891                 goto out;
8892         }
8893
8894         if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
8895             TG3_EEPROM_MAGIC_HW) {
8896                 u8 data[NVRAM_SELFBOOT_DATA_SIZE];
8897                 u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
8898                 u8 *buf8 = (u8 *) buf;
8899
8900                 /* Separate the parity bits and the data bytes.  */
8901                 for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
8902                         if ((i == 0) || (i == 8)) {
8903                                 int l;
8904                                 u8 msk;
8905
8906                                 for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
8907                                         parity[k++] = buf8[i] & msk;
8908                                 i++;
8909                         }
8910                         else if (i == 16) {
8911                                 int l;
8912                                 u8 msk;
8913
8914                                 for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
8915                                         parity[k++] = buf8[i] & msk;
8916                                 i++;
8917
8918                                 for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
8919                                         parity[k++] = buf8[i] & msk;
8920                                 i++;
8921                         }
8922                         data[j++] = buf8[i];
8923                 }
8924
8925                 err = -EIO;
8926                 for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
8927                         u8 hw8 = hweight8(data[i]);
8928
8929                         if ((hw8 & 0x1) && parity[i])
8930                                 goto out;
8931                         else if (!(hw8 & 0x1) && !parity[i])
8932                                 goto out;
8933                 }
8934                 err = 0;
8935                 goto out;
8936         }
8937
8938         /* Bootstrap checksum at offset 0x10 */
8939         csum = calc_crc((unsigned char *) buf, 0x10);
8940         if(csum != le32_to_cpu(buf[0x10/4]))
8941                 goto out;
8942
8943         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
8944         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
8945         if (csum != le32_to_cpu(buf[0xfc/4]))
8946                  goto out;
8947
8948         err = 0;
8949
8950 out:
8951         kfree(buf);
8952         return err;
8953 }
8954
8955 #define TG3_SERDES_TIMEOUT_SEC  2
8956 #define TG3_COPPER_TIMEOUT_SEC  6
8957
8958 static int tg3_test_link(struct tg3 *tp)
8959 {
8960         int i, max;
8961
8962         if (!netif_running(tp->dev))
8963                 return -ENODEV;
8964
8965         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
8966                 max = TG3_SERDES_TIMEOUT_SEC;
8967         else
8968                 max = TG3_COPPER_TIMEOUT_SEC;
8969
8970         for (i = 0; i < max; i++) {
8971                 if (netif_carrier_ok(tp->dev))
8972                         return 0;
8973
8974                 if (msleep_interruptible(1000))
8975                         break;
8976         }
8977
8978         return -EIO;
8979 }
8980
8981 /* Only test the commonly used registers */
8982 static int tg3_test_registers(struct tg3 *tp)
8983 {
8984         int i, is_5705, is_5750;
8985         u32 offset, read_mask, write_mask, val, save_val, read_val;
8986         static struct {
8987                 u16 offset;
8988                 u16 flags;
8989 #define TG3_FL_5705     0x1
8990 #define TG3_FL_NOT_5705 0x2
8991 #define TG3_FL_NOT_5788 0x4
8992 #define TG3_FL_NOT_5750 0x8
8993                 u32 read_mask;
8994                 u32 write_mask;
8995         } reg_tbl[] = {
8996                 /* MAC Control Registers */
8997                 { MAC_MODE, TG3_FL_NOT_5705,
8998                         0x00000000, 0x00ef6f8c },
8999                 { MAC_MODE, TG3_FL_5705,
9000                         0x00000000, 0x01ef6b8c },
9001                 { MAC_STATUS, TG3_FL_NOT_5705,
9002                         0x03800107, 0x00000000 },
9003                 { MAC_STATUS, TG3_FL_5705,
9004                         0x03800100, 0x00000000 },
9005                 { MAC_ADDR_0_HIGH, 0x0000,
9006                         0x00000000, 0x0000ffff },
9007                 { MAC_ADDR_0_LOW, 0x0000,
9008                         0x00000000, 0xffffffff },
9009                 { MAC_RX_MTU_SIZE, 0x0000,
9010                         0x00000000, 0x0000ffff },
9011                 { MAC_TX_MODE, 0x0000,
9012                         0x00000000, 0x00000070 },
9013                 { MAC_TX_LENGTHS, 0x0000,
9014                         0x00000000, 0x00003fff },
9015                 { MAC_RX_MODE, TG3_FL_NOT_5705,
9016                         0x00000000, 0x000007fc },
9017                 { MAC_RX_MODE, TG3_FL_5705,
9018                         0x00000000, 0x000007dc },
9019                 { MAC_HASH_REG_0, 0x0000,
9020                         0x00000000, 0xffffffff },
9021                 { MAC_HASH_REG_1, 0x0000,
9022                         0x00000000, 0xffffffff },
9023                 { MAC_HASH_REG_2, 0x0000,
9024                         0x00000000, 0xffffffff },
9025                 { MAC_HASH_REG_3, 0x0000,
9026                         0x00000000, 0xffffffff },
9027
9028                 /* Receive Data and Receive BD Initiator Control Registers. */
9029                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
9030                         0x00000000, 0xffffffff },
9031                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
9032                         0x00000000, 0xffffffff },
9033                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
9034                         0x00000000, 0x00000003 },
9035                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
9036                         0x00000000, 0xffffffff },
9037                 { RCVDBDI_STD_BD+0, 0x0000,
9038                         0x00000000, 0xffffffff },
9039                 { RCVDBDI_STD_BD+4, 0x0000,
9040                         0x00000000, 0xffffffff },
9041                 { RCVDBDI_STD_BD+8, 0x0000,
9042                         0x00000000, 0xffff0002 },
9043                 { RCVDBDI_STD_BD+0xc, 0x0000,
9044                         0x00000000, 0xffffffff },
9045
9046                 /* Receive BD Initiator Control Registers. */
9047                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
9048                         0x00000000, 0xffffffff },
9049                 { RCVBDI_STD_THRESH, TG3_FL_5705,
9050                         0x00000000, 0x000003ff },
9051                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
9052                         0x00000000, 0xffffffff },
9053
9054                 /* Host Coalescing Control Registers. */
9055                 { HOSTCC_MODE, TG3_FL_NOT_5705,
9056                         0x00000000, 0x00000004 },
9057                 { HOSTCC_MODE, TG3_FL_5705,
9058                         0x00000000, 0x000000f6 },
9059                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
9060                         0x00000000, 0xffffffff },
9061                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
9062                         0x00000000, 0x000003ff },
9063                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
9064                         0x00000000, 0xffffffff },
9065                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
9066                         0x00000000, 0x000003ff },
9067                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
9068                         0x00000000, 0xffffffff },
9069                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
9070                         0x00000000, 0x000000ff },
9071                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
9072                         0x00000000, 0xffffffff },
9073                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
9074                         0x00000000, 0x000000ff },
9075                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
9076                         0x00000000, 0xffffffff },
9077                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
9078                         0x00000000, 0xffffffff },
9079                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
9080                         0x00000000, 0xffffffff },
9081                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
9082                         0x00000000, 0x000000ff },
9083                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
9084                         0x00000000, 0xffffffff },
9085                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
9086                         0x00000000, 0x000000ff },
9087                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
9088                         0x00000000, 0xffffffff },
9089                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
9090                         0x00000000, 0xffffffff },
9091                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
9092                         0x00000000, 0xffffffff },
9093                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
9094                         0x00000000, 0xffffffff },
9095                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
9096                         0x00000000, 0xffffffff },
9097                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
9098                         0xffffffff, 0x00000000 },
9099                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
9100                         0xffffffff, 0x00000000 },
9101
9102                 /* Buffer Manager Control Registers. */
9103                 { BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
9104                         0x00000000, 0x007fff80 },
9105                 { BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
9106                         0x00000000, 0x007fffff },
9107                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
9108                         0x00000000, 0x0000003f },
9109                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
9110                         0x00000000, 0x000001ff },
9111                 { BUFMGR_MB_HIGH_WATER, 0x0000,
9112                         0x00000000, 0x000001ff },
9113                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
9114                         0xffffffff, 0x00000000 },
9115                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
9116                         0xffffffff, 0x00000000 },
9117
9118                 /* Mailbox Registers */
9119                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
9120                         0x00000000, 0x000001ff },
9121                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
9122                         0x00000000, 0x000001ff },
9123                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
9124                         0x00000000, 0x000007ff },
9125                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
9126                         0x00000000, 0x000001ff },
9127
9128                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
9129         };
9130
9131         is_5705 = is_5750 = 0;
9132         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9133                 is_5705 = 1;
9134                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9135                         is_5750 = 1;
9136         }
9137
9138         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
9139                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
9140                         continue;
9141
9142                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
9143                         continue;
9144
9145                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
9146                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
9147                         continue;
9148
9149                 if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
9150                         continue;
9151
9152                 offset = (u32) reg_tbl[i].offset;
9153                 read_mask = reg_tbl[i].read_mask;
9154                 write_mask = reg_tbl[i].write_mask;
9155
9156                 /* Save the original register content */
9157                 save_val = tr32(offset);
9158
9159                 /* Determine the read-only value. */
9160                 read_val = save_val & read_mask;
9161
9162                 /* Write zero to the register, then make sure the read-only bits
9163                  * are not changed and the read/write bits are all zeros.
9164                  */
9165                 tw32(offset, 0);
9166
9167                 val = tr32(offset);
9168
9169                 /* Test the read-only and read/write bits. */
9170                 if (((val & read_mask) != read_val) || (val & write_mask))
9171                         goto out;
9172
9173                 /* Write ones to all the bits defined by RdMask and WrMask, then
9174                  * make sure the read-only bits are not changed and the
9175                  * read/write bits are all ones.
9176                  */
9177                 tw32(offset, read_mask | write_mask);
9178
9179                 val = tr32(offset);
9180
9181                 /* Test the read-only bits. */
9182                 if ((val & read_mask) != read_val)
9183                         goto out;
9184
9185                 /* Test the read/write bits. */
9186                 if ((val & write_mask) != write_mask)
9187                         goto out;
9188
9189                 tw32(offset, save_val);
9190         }
9191
9192         return 0;
9193
9194 out:
9195         if (netif_msg_hw(tp))
9196                 printk(KERN_ERR PFX "Register test failed at offset %x\n",
9197                        offset);
9198         tw32(offset, save_val);
9199         return -EIO;
9200 }
9201
9202 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
9203 {
9204         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
9205         int i;
9206         u32 j;
9207
9208         for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
9209                 for (j = 0; j < len; j += 4) {
9210                         u32 val;
9211
9212                         tg3_write_mem(tp, offset + j, test_pattern[i]);
9213                         tg3_read_mem(tp, offset + j, &val);
9214                         if (val != test_pattern[i])
9215                                 return -EIO;
9216                 }
9217         }
9218         return 0;
9219 }
9220
9221 static int tg3_test_memory(struct tg3 *tp)
9222 {
9223         static struct mem_entry {
9224                 u32 offset;
9225                 u32 len;
9226         } mem_tbl_570x[] = {
9227                 { 0x00000000, 0x00b50},
9228                 { 0x00002000, 0x1c000},
9229                 { 0xffffffff, 0x00000}
9230         }, mem_tbl_5705[] = {
9231                 { 0x00000100, 0x0000c},
9232                 { 0x00000200, 0x00008},
9233                 { 0x00004000, 0x00800},
9234                 { 0x00006000, 0x01000},
9235                 { 0x00008000, 0x02000},
9236                 { 0x00010000, 0x0e000},
9237                 { 0xffffffff, 0x00000}
9238         }, mem_tbl_5755[] = {
9239                 { 0x00000200, 0x00008},
9240                 { 0x00004000, 0x00800},
9241                 { 0x00006000, 0x00800},
9242                 { 0x00008000, 0x02000},
9243                 { 0x00010000, 0x0c000},
9244                 { 0xffffffff, 0x00000}
9245         }, mem_tbl_5906[] = {
9246                 { 0x00000200, 0x00008},
9247                 { 0x00004000, 0x00400},
9248                 { 0x00006000, 0x00400},
9249                 { 0x00008000, 0x01000},
9250                 { 0x00010000, 0x01000},
9251                 { 0xffffffff, 0x00000}
9252         };
9253         struct mem_entry *mem_tbl;
9254         int err = 0;
9255         int i;
9256
9257         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9258                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
9259                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
9260                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9261                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
9262                         mem_tbl = mem_tbl_5755;
9263                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
9264                         mem_tbl = mem_tbl_5906;
9265                 else
9266                         mem_tbl = mem_tbl_5705;
9267         } else
9268                 mem_tbl = mem_tbl_570x;
9269
9270         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
9271                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
9272                     mem_tbl[i].len)) != 0)
9273                         break;
9274         }
9275
9276         return err;
9277 }
9278
9279 #define TG3_MAC_LOOPBACK        0
9280 #define TG3_PHY_LOOPBACK        1
9281
9282 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
9283 {
9284         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
9285         u32 desc_idx;
9286         struct sk_buff *skb, *rx_skb;
9287         u8 *tx_data;
9288         dma_addr_t map;
9289         int num_pkts, tx_len, rx_len, i, err;
9290         struct tg3_rx_buffer_desc *desc;
9291
9292         if (loopback_mode == TG3_MAC_LOOPBACK) {
9293                 /* HW errata - mac loopback fails in some cases on 5780.
9294                  * Normal traffic and PHY loopback are not affected by
9295                  * errata.
9296                  */
9297                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
9298                         return 0;
9299
9300                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
9301                            MAC_MODE_PORT_INT_LPBACK;
9302                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9303                         mac_mode |= MAC_MODE_LINK_POLARITY;
9304                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
9305                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9306                 else
9307                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9308                 tw32(MAC_MODE, mac_mode);
9309         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
9310                 u32 val;
9311
9312                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9313                         u32 phytest;
9314
9315                         if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &phytest)) {
9316                                 u32 phy;
9317
9318                                 tg3_writephy(tp, MII_TG3_EPHY_TEST,
9319                                              phytest | MII_TG3_EPHY_SHADOW_EN);
9320                                 if (!tg3_readphy(tp, 0x1b, &phy))
9321                                         tg3_writephy(tp, 0x1b, phy & ~0x20);
9322                                 tg3_writephy(tp, MII_TG3_EPHY_TEST, phytest);
9323                         }
9324                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED100;
9325                 } else
9326                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED1000;
9327
9328                 tg3_phy_toggle_automdix(tp, 0);
9329
9330                 tg3_writephy(tp, MII_BMCR, val);
9331                 udelay(40);
9332
9333                 mac_mode = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
9334                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9335                         tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x1800);
9336                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9337                 } else
9338                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9339
9340                 /* reset to prevent losing 1st rx packet intermittently */
9341                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
9342                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
9343                         udelay(10);
9344                         tw32_f(MAC_RX_MODE, tp->rx_mode);
9345                 }
9346                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
9347                         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)
9348                                 mac_mode &= ~MAC_MODE_LINK_POLARITY;
9349                         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411)
9350                                 mac_mode |= MAC_MODE_LINK_POLARITY;
9351                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
9352                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
9353                 }
9354                 tw32(MAC_MODE, mac_mode);
9355         }
9356         else
9357                 return -EINVAL;
9358
9359         err = -EIO;
9360
9361         tx_len = 1514;
9362         skb = netdev_alloc_skb(tp->dev, tx_len);
9363         if (!skb)
9364                 return -ENOMEM;
9365
9366         tx_data = skb_put(skb, tx_len);
9367         memcpy(tx_data, tp->dev->dev_addr, 6);
9368         memset(tx_data + 6, 0x0, 8);
9369
9370         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
9371
9372         for (i = 14; i < tx_len; i++)
9373                 tx_data[i] = (u8) (i & 0xff);
9374
9375         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
9376
9377         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9378              HOSTCC_MODE_NOW);
9379
9380         udelay(10);
9381
9382         rx_start_idx = tp->hw_status->idx[0].rx_producer;
9383
9384         num_pkts = 0;
9385
9386         tg3_set_txd(tp, tp->tx_prod, map, tx_len, 0, 1);
9387
9388         tp->tx_prod++;
9389         num_pkts++;
9390
9391         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW,
9392                      tp->tx_prod);
9393         tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW);
9394
9395         udelay(10);
9396
9397         /* 250 usec to allow enough time on some 10/100 Mbps devices.  */
9398         for (i = 0; i < 25; i++) {
9399                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9400                        HOSTCC_MODE_NOW);
9401
9402                 udelay(10);
9403
9404                 tx_idx = tp->hw_status->idx[0].tx_consumer;
9405                 rx_idx = tp->hw_status->idx[0].rx_producer;
9406                 if ((tx_idx == tp->tx_prod) &&
9407                     (rx_idx == (rx_start_idx + num_pkts)))
9408                         break;
9409         }
9410
9411         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
9412         dev_kfree_skb(skb);
9413
9414         if (tx_idx != tp->tx_prod)
9415                 goto out;
9416
9417         if (rx_idx != rx_start_idx + num_pkts)
9418                 goto out;
9419
9420         desc = &tp->rx_rcb[rx_start_idx];
9421         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
9422         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
9423         if (opaque_key != RXD_OPAQUE_RING_STD)
9424                 goto out;
9425
9426         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
9427             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
9428                 goto out;
9429
9430         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
9431         if (rx_len != tx_len)
9432                 goto out;
9433
9434         rx_skb = tp->rx_std_buffers[desc_idx].skb;
9435
9436         map = pci_unmap_addr(&tp->rx_std_buffers[desc_idx], mapping);
9437         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
9438
9439         for (i = 14; i < tx_len; i++) {
9440                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
9441                         goto out;
9442         }
9443         err = 0;
9444
9445         /* tg3_free_rings will unmap and free the rx_skb */
9446 out:
9447         return err;
9448 }
9449
9450 #define TG3_MAC_LOOPBACK_FAILED         1
9451 #define TG3_PHY_LOOPBACK_FAILED         2
9452 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
9453                                          TG3_PHY_LOOPBACK_FAILED)
9454
9455 static int tg3_test_loopback(struct tg3 *tp)
9456 {
9457         int err = 0;
9458         u32 cpmuctrl = 0;
9459
9460         if (!netif_running(tp->dev))
9461                 return TG3_LOOPBACK_FAILED;
9462
9463         err = tg3_reset_hw(tp, 1);
9464         if (err)
9465                 return TG3_LOOPBACK_FAILED;
9466
9467         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
9468                 int i;
9469                 u32 status;
9470
9471                 tw32(TG3_CPMU_MUTEX_REQ, CPMU_MUTEX_REQ_DRIVER);
9472
9473                 /* Wait for up to 40 microseconds to acquire lock. */
9474                 for (i = 0; i < 4; i++) {
9475                         status = tr32(TG3_CPMU_MUTEX_GNT);
9476                         if (status == CPMU_MUTEX_GNT_DRIVER)
9477                                 break;
9478                         udelay(10);
9479                 }
9480
9481                 if (status != CPMU_MUTEX_GNT_DRIVER)
9482                         return TG3_LOOPBACK_FAILED;
9483
9484                 /* Turn off power management based on link speed. */
9485                 cpmuctrl = tr32(TG3_CPMU_CTRL);
9486                 tw32(TG3_CPMU_CTRL,
9487                      cpmuctrl & ~(CPMU_CTRL_LINK_SPEED_MODE |
9488                                   CPMU_CTRL_LINK_AWARE_MODE));
9489         }
9490
9491         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
9492                 err |= TG3_MAC_LOOPBACK_FAILED;
9493
9494         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
9495                 tw32(TG3_CPMU_CTRL, cpmuctrl);
9496
9497                 /* Release the mutex */
9498                 tw32(TG3_CPMU_MUTEX_GNT, CPMU_MUTEX_GNT_DRIVER);
9499         }
9500
9501         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
9502                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
9503                         err |= TG3_PHY_LOOPBACK_FAILED;
9504         }
9505
9506         return err;
9507 }
9508
9509 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
9510                           u64 *data)
9511 {
9512         struct tg3 *tp = netdev_priv(dev);
9513
9514         if (tp->link_config.phy_is_low_power)
9515                 tg3_set_power_state(tp, PCI_D0);
9516
9517         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
9518
9519         if (tg3_test_nvram(tp) != 0) {
9520                 etest->flags |= ETH_TEST_FL_FAILED;
9521                 data[0] = 1;
9522         }
9523         if (tg3_test_link(tp) != 0) {
9524                 etest->flags |= ETH_TEST_FL_FAILED;
9525                 data[1] = 1;
9526         }
9527         if (etest->flags & ETH_TEST_FL_OFFLINE) {
9528                 int err, irq_sync = 0;
9529
9530                 if (netif_running(dev)) {
9531                         tg3_netif_stop(tp);
9532                         irq_sync = 1;
9533                 }
9534
9535                 tg3_full_lock(tp, irq_sync);
9536
9537                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
9538                 err = tg3_nvram_lock(tp);
9539                 tg3_halt_cpu(tp, RX_CPU_BASE);
9540                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9541                         tg3_halt_cpu(tp, TX_CPU_BASE);
9542                 if (!err)
9543                         tg3_nvram_unlock(tp);
9544
9545                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
9546                         tg3_phy_reset(tp);
9547
9548                 if (tg3_test_registers(tp) != 0) {
9549                         etest->flags |= ETH_TEST_FL_FAILED;
9550                         data[2] = 1;
9551                 }
9552                 if (tg3_test_memory(tp) != 0) {
9553                         etest->flags |= ETH_TEST_FL_FAILED;
9554                         data[3] = 1;
9555                 }
9556                 if ((data[4] = tg3_test_loopback(tp)) != 0)
9557                         etest->flags |= ETH_TEST_FL_FAILED;
9558
9559                 tg3_full_unlock(tp);
9560
9561                 if (tg3_test_interrupt(tp) != 0) {
9562                         etest->flags |= ETH_TEST_FL_FAILED;
9563                         data[5] = 1;
9564                 }
9565
9566                 tg3_full_lock(tp, 0);
9567
9568                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9569                 if (netif_running(dev)) {
9570                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
9571                         if (!tg3_restart_hw(tp, 1))
9572                                 tg3_netif_start(tp);
9573                 }
9574
9575                 tg3_full_unlock(tp);
9576         }
9577         if (tp->link_config.phy_is_low_power)
9578                 tg3_set_power_state(tp, PCI_D3hot);
9579
9580 }
9581
9582 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9583 {
9584         struct mii_ioctl_data *data = if_mii(ifr);
9585         struct tg3 *tp = netdev_priv(dev);
9586         int err;
9587
9588         switch(cmd) {
9589         case SIOCGMIIPHY:
9590                 data->phy_id = PHY_ADDR;
9591
9592                 /* fallthru */
9593         case SIOCGMIIREG: {
9594                 u32 mii_regval;
9595
9596                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9597                         break;                  /* We have no PHY */
9598
9599                 if (tp->link_config.phy_is_low_power)
9600                         return -EAGAIN;
9601
9602                 spin_lock_bh(&tp->lock);
9603                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
9604                 spin_unlock_bh(&tp->lock);
9605
9606                 data->val_out = mii_regval;
9607
9608                 return err;
9609         }
9610
9611         case SIOCSMIIREG:
9612                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9613                         break;                  /* We have no PHY */
9614
9615                 if (!capable(CAP_NET_ADMIN))
9616                         return -EPERM;
9617
9618                 if (tp->link_config.phy_is_low_power)
9619                         return -EAGAIN;
9620
9621                 spin_lock_bh(&tp->lock);
9622                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
9623                 spin_unlock_bh(&tp->lock);
9624
9625                 return err;
9626
9627         default:
9628                 /* do nothing */
9629                 break;
9630         }
9631         return -EOPNOTSUPP;
9632 }
9633
9634 #if TG3_VLAN_TAG_USED
9635 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
9636 {
9637         struct tg3 *tp = netdev_priv(dev);
9638
9639         if (netif_running(dev))
9640                 tg3_netif_stop(tp);
9641
9642         tg3_full_lock(tp, 0);
9643
9644         tp->vlgrp = grp;
9645
9646         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
9647         __tg3_set_rx_mode(dev);
9648
9649         if (netif_running(dev))
9650                 tg3_netif_start(tp);
9651
9652         tg3_full_unlock(tp);
9653 }
9654 #endif
9655
9656 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9657 {
9658         struct tg3 *tp = netdev_priv(dev);
9659
9660         memcpy(ec, &tp->coal, sizeof(*ec));
9661         return 0;
9662 }
9663
9664 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9665 {
9666         struct tg3 *tp = netdev_priv(dev);
9667         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
9668         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
9669
9670         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
9671                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
9672                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
9673                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
9674                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
9675         }
9676
9677         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
9678             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
9679             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
9680             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
9681             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
9682             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
9683             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
9684             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
9685             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
9686             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
9687                 return -EINVAL;
9688
9689         /* No rx interrupts will be generated if both are zero */
9690         if ((ec->rx_coalesce_usecs == 0) &&
9691             (ec->rx_max_coalesced_frames == 0))
9692                 return -EINVAL;
9693
9694         /* No tx interrupts will be generated if both are zero */
9695         if ((ec->tx_coalesce_usecs == 0) &&
9696             (ec->tx_max_coalesced_frames == 0))
9697                 return -EINVAL;
9698
9699         /* Only copy relevant parameters, ignore all others. */
9700         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
9701         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
9702         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
9703         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
9704         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
9705         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
9706         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
9707         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
9708         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
9709
9710         if (netif_running(dev)) {
9711                 tg3_full_lock(tp, 0);
9712                 __tg3_set_coalesce(tp, &tp->coal);
9713                 tg3_full_unlock(tp);
9714         }
9715         return 0;
9716 }
9717
9718 static const struct ethtool_ops tg3_ethtool_ops = {
9719         .get_settings           = tg3_get_settings,
9720         .set_settings           = tg3_set_settings,
9721         .get_drvinfo            = tg3_get_drvinfo,
9722         .get_regs_len           = tg3_get_regs_len,
9723         .get_regs               = tg3_get_regs,
9724         .get_wol                = tg3_get_wol,
9725         .set_wol                = tg3_set_wol,
9726         .get_msglevel           = tg3_get_msglevel,
9727         .set_msglevel           = tg3_set_msglevel,
9728         .nway_reset             = tg3_nway_reset,
9729         .get_link               = ethtool_op_get_link,
9730         .get_eeprom_len         = tg3_get_eeprom_len,
9731         .get_eeprom             = tg3_get_eeprom,
9732         .set_eeprom             = tg3_set_eeprom,
9733         .get_ringparam          = tg3_get_ringparam,
9734         .set_ringparam          = tg3_set_ringparam,
9735         .get_pauseparam         = tg3_get_pauseparam,
9736         .set_pauseparam         = tg3_set_pauseparam,
9737         .get_rx_csum            = tg3_get_rx_csum,
9738         .set_rx_csum            = tg3_set_rx_csum,
9739         .set_tx_csum            = tg3_set_tx_csum,
9740         .set_sg                 = ethtool_op_set_sg,
9741         .set_tso                = tg3_set_tso,
9742         .self_test              = tg3_self_test,
9743         .get_strings            = tg3_get_strings,
9744         .phys_id                = tg3_phys_id,
9745         .get_ethtool_stats      = tg3_get_ethtool_stats,
9746         .get_coalesce           = tg3_get_coalesce,
9747         .set_coalesce           = tg3_set_coalesce,
9748         .get_sset_count         = tg3_get_sset_count,
9749 };
9750
9751 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
9752 {
9753         u32 cursize, val, magic;
9754
9755         tp->nvram_size = EEPROM_CHIP_SIZE;
9756
9757         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
9758                 return;
9759
9760         if ((magic != TG3_EEPROM_MAGIC) &&
9761             ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
9762             ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
9763                 return;
9764
9765         /*
9766          * Size the chip by reading offsets at increasing powers of two.
9767          * When we encounter our validation signature, we know the addressing
9768          * has wrapped around, and thus have our chip size.
9769          */
9770         cursize = 0x10;
9771
9772         while (cursize < tp->nvram_size) {
9773                 if (tg3_nvram_read_swab(tp, cursize, &val) != 0)
9774                         return;
9775
9776                 if (val == magic)
9777                         break;
9778
9779                 cursize <<= 1;
9780         }
9781
9782         tp->nvram_size = cursize;
9783 }
9784
9785 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
9786 {
9787         u32 val;
9788
9789         if (tg3_nvram_read_swab(tp, 0, &val) != 0)
9790                 return;
9791
9792         /* Selfboot format */
9793         if (val != TG3_EEPROM_MAGIC) {
9794                 tg3_get_eeprom_size(tp);
9795                 return;
9796         }
9797
9798         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
9799                 if (val != 0) {
9800                         tp->nvram_size = (val >> 16) * 1024;
9801                         return;
9802                 }
9803         }
9804         tp->nvram_size = 0x80000;
9805 }
9806
9807 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
9808 {
9809         u32 nvcfg1;
9810
9811         nvcfg1 = tr32(NVRAM_CFG1);
9812         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
9813                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
9814         }
9815         else {
9816                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9817                 tw32(NVRAM_CFG1, nvcfg1);
9818         }
9819
9820         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
9821             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
9822                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
9823                         case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
9824                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9825                                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9826                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9827                                 break;
9828                         case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
9829                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9830                                 tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
9831                                 break;
9832                         case FLASH_VENDOR_ATMEL_EEPROM:
9833                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9834                                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9835                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9836                                 break;
9837                         case FLASH_VENDOR_ST:
9838                                 tp->nvram_jedecnum = JEDEC_ST;
9839                                 tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
9840                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9841                                 break;
9842                         case FLASH_VENDOR_SAIFUN:
9843                                 tp->nvram_jedecnum = JEDEC_SAIFUN;
9844                                 tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
9845                                 break;
9846                         case FLASH_VENDOR_SST_SMALL:
9847                         case FLASH_VENDOR_SST_LARGE:
9848                                 tp->nvram_jedecnum = JEDEC_SST;
9849                                 tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
9850                                 break;
9851                 }
9852         }
9853         else {
9854                 tp->nvram_jedecnum = JEDEC_ATMEL;
9855                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9856                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9857         }
9858 }
9859
9860 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
9861 {
9862         u32 nvcfg1;
9863
9864         nvcfg1 = tr32(NVRAM_CFG1);
9865
9866         /* NVRAM protection for TPM */
9867         if (nvcfg1 & (1 << 27))
9868                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9869
9870         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9871                 case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
9872                 case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
9873                         tp->nvram_jedecnum = JEDEC_ATMEL;
9874                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9875                         break;
9876                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9877                         tp->nvram_jedecnum = JEDEC_ATMEL;
9878                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9879                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9880                         break;
9881                 case FLASH_5752VENDOR_ST_M45PE10:
9882                 case FLASH_5752VENDOR_ST_M45PE20:
9883                 case FLASH_5752VENDOR_ST_M45PE40:
9884                         tp->nvram_jedecnum = JEDEC_ST;
9885                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9886                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9887                         break;
9888         }
9889
9890         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
9891                 switch (nvcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
9892                         case FLASH_5752PAGE_SIZE_256:
9893                                 tp->nvram_pagesize = 256;
9894                                 break;
9895                         case FLASH_5752PAGE_SIZE_512:
9896                                 tp->nvram_pagesize = 512;
9897                                 break;
9898                         case FLASH_5752PAGE_SIZE_1K:
9899                                 tp->nvram_pagesize = 1024;
9900                                 break;
9901                         case FLASH_5752PAGE_SIZE_2K:
9902                                 tp->nvram_pagesize = 2048;
9903                                 break;
9904                         case FLASH_5752PAGE_SIZE_4K:
9905                                 tp->nvram_pagesize = 4096;
9906                                 break;
9907                         case FLASH_5752PAGE_SIZE_264:
9908                                 tp->nvram_pagesize = 264;
9909                                 break;
9910                 }
9911         }
9912         else {
9913                 /* For eeprom, set pagesize to maximum eeprom size */
9914                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9915
9916                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9917                 tw32(NVRAM_CFG1, nvcfg1);
9918         }
9919 }
9920
9921 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
9922 {
9923         u32 nvcfg1, protect = 0;
9924
9925         nvcfg1 = tr32(NVRAM_CFG1);
9926
9927         /* NVRAM protection for TPM */
9928         if (nvcfg1 & (1 << 27)) {
9929                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9930                 protect = 1;
9931         }
9932
9933         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
9934         switch (nvcfg1) {
9935                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9936                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9937                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9938                 case FLASH_5755VENDOR_ATMEL_FLASH_5:
9939                         tp->nvram_jedecnum = JEDEC_ATMEL;
9940                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9941                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9942                         tp->nvram_pagesize = 264;
9943                         if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
9944                             nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
9945                                 tp->nvram_size = (protect ? 0x3e200 : 0x80000);
9946                         else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
9947                                 tp->nvram_size = (protect ? 0x1f200 : 0x40000);
9948                         else
9949                                 tp->nvram_size = (protect ? 0x1f200 : 0x20000);
9950                         break;
9951                 case FLASH_5752VENDOR_ST_M45PE10:
9952                 case FLASH_5752VENDOR_ST_M45PE20:
9953                 case FLASH_5752VENDOR_ST_M45PE40:
9954                         tp->nvram_jedecnum = JEDEC_ST;
9955                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9956                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9957                         tp->nvram_pagesize = 256;
9958                         if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
9959                                 tp->nvram_size = (protect ? 0x10000 : 0x20000);
9960                         else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
9961                                 tp->nvram_size = (protect ? 0x10000 : 0x40000);
9962                         else
9963                                 tp->nvram_size = (protect ? 0x20000 : 0x80000);
9964                         break;
9965         }
9966 }
9967
9968 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
9969 {
9970         u32 nvcfg1;
9971
9972         nvcfg1 = tr32(NVRAM_CFG1);
9973
9974         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9975                 case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
9976                 case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
9977                 case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
9978                 case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
9979                         tp->nvram_jedecnum = JEDEC_ATMEL;
9980                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9981                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9982
9983                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9984                         tw32(NVRAM_CFG1, nvcfg1);
9985                         break;
9986                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9987                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9988                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9989                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9990                         tp->nvram_jedecnum = JEDEC_ATMEL;
9991                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9992                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9993                         tp->nvram_pagesize = 264;
9994                         break;
9995                 case FLASH_5752VENDOR_ST_M45PE10:
9996                 case FLASH_5752VENDOR_ST_M45PE20:
9997                 case FLASH_5752VENDOR_ST_M45PE40:
9998                         tp->nvram_jedecnum = JEDEC_ST;
9999                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10000                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10001                         tp->nvram_pagesize = 256;
10002                         break;
10003         }
10004 }
10005
10006 static void __devinit tg3_get_5761_nvram_info(struct tg3 *tp)
10007 {
10008         u32 nvcfg1, protect = 0;
10009
10010         nvcfg1 = tr32(NVRAM_CFG1);
10011
10012         /* NVRAM protection for TPM */
10013         if (nvcfg1 & (1 << 27)) {
10014                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
10015                 protect = 1;
10016         }
10017
10018         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
10019         switch (nvcfg1) {
10020                 case FLASH_5761VENDOR_ATMEL_ADB021D:
10021                 case FLASH_5761VENDOR_ATMEL_ADB041D:
10022                 case FLASH_5761VENDOR_ATMEL_ADB081D:
10023                 case FLASH_5761VENDOR_ATMEL_ADB161D:
10024                 case FLASH_5761VENDOR_ATMEL_MDB021D:
10025                 case FLASH_5761VENDOR_ATMEL_MDB041D:
10026                 case FLASH_5761VENDOR_ATMEL_MDB081D:
10027                 case FLASH_5761VENDOR_ATMEL_MDB161D:
10028                         tp->nvram_jedecnum = JEDEC_ATMEL;
10029                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10030                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10031                         tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
10032                         tp->nvram_pagesize = 256;
10033                         break;
10034                 case FLASH_5761VENDOR_ST_A_M45PE20:
10035                 case FLASH_5761VENDOR_ST_A_M45PE40:
10036                 case FLASH_5761VENDOR_ST_A_M45PE80:
10037                 case FLASH_5761VENDOR_ST_A_M45PE16:
10038                 case FLASH_5761VENDOR_ST_M_M45PE20:
10039                 case FLASH_5761VENDOR_ST_M_M45PE40:
10040                 case FLASH_5761VENDOR_ST_M_M45PE80:
10041                 case FLASH_5761VENDOR_ST_M_M45PE16:
10042                         tp->nvram_jedecnum = JEDEC_ST;
10043                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10044                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10045                         tp->nvram_pagesize = 256;
10046                         break;
10047         }
10048
10049         if (protect) {
10050                 tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
10051         } else {
10052                 switch (nvcfg1) {
10053                         case FLASH_5761VENDOR_ATMEL_ADB161D:
10054                         case FLASH_5761VENDOR_ATMEL_MDB161D:
10055                         case FLASH_5761VENDOR_ST_A_M45PE16:
10056                         case FLASH_5761VENDOR_ST_M_M45PE16:
10057                                 tp->nvram_size = 0x100000;
10058                                 break;
10059                         case FLASH_5761VENDOR_ATMEL_ADB081D:
10060                         case FLASH_5761VENDOR_ATMEL_MDB081D:
10061                         case FLASH_5761VENDOR_ST_A_M45PE80:
10062                         case FLASH_5761VENDOR_ST_M_M45PE80:
10063                                 tp->nvram_size = 0x80000;
10064                                 break;
10065                         case FLASH_5761VENDOR_ATMEL_ADB041D:
10066                         case FLASH_5761VENDOR_ATMEL_MDB041D:
10067                         case FLASH_5761VENDOR_ST_A_M45PE40:
10068                         case FLASH_5761VENDOR_ST_M_M45PE40:
10069                                 tp->nvram_size = 0x40000;
10070                                 break;
10071                         case FLASH_5761VENDOR_ATMEL_ADB021D:
10072                         case FLASH_5761VENDOR_ATMEL_MDB021D:
10073                         case FLASH_5761VENDOR_ST_A_M45PE20:
10074                         case FLASH_5761VENDOR_ST_M_M45PE20:
10075                                 tp->nvram_size = 0x20000;
10076                                 break;
10077                 }
10078         }
10079 }
10080
10081 static void __devinit tg3_get_5906_nvram_info(struct tg3 *tp)
10082 {
10083         tp->nvram_jedecnum = JEDEC_ATMEL;
10084         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10085         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10086 }
10087
10088 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
10089 static void __devinit tg3_nvram_init(struct tg3 *tp)
10090 {
10091         tw32_f(GRC_EEPROM_ADDR,
10092              (EEPROM_ADDR_FSM_RESET |
10093               (EEPROM_DEFAULT_CLOCK_PERIOD <<
10094                EEPROM_ADDR_CLKPERD_SHIFT)));
10095
10096         msleep(1);
10097
10098         /* Enable seeprom accesses. */
10099         tw32_f(GRC_LOCAL_CTRL,
10100              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
10101         udelay(100);
10102
10103         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
10104             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
10105                 tp->tg3_flags |= TG3_FLAG_NVRAM;
10106
10107                 if (tg3_nvram_lock(tp)) {
10108                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
10109                                "tg3_nvram_init failed.\n", tp->dev->name);
10110                         return;
10111                 }
10112                 tg3_enable_nvram_access(tp);
10113
10114                 tp->nvram_size = 0;
10115
10116                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
10117                         tg3_get_5752_nvram_info(tp);
10118                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10119                         tg3_get_5755_nvram_info(tp);
10120                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
10121                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
10122                         tg3_get_5787_nvram_info(tp);
10123                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
10124                         tg3_get_5761_nvram_info(tp);
10125                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10126                         tg3_get_5906_nvram_info(tp);
10127                 else
10128                         tg3_get_nvram_info(tp);
10129
10130                 if (tp->nvram_size == 0)
10131                         tg3_get_nvram_size(tp);
10132
10133                 tg3_disable_nvram_access(tp);
10134                 tg3_nvram_unlock(tp);
10135
10136         } else {
10137                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
10138
10139                 tg3_get_eeprom_size(tp);
10140         }
10141 }
10142
10143 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
10144                                         u32 offset, u32 *val)
10145 {
10146         u32 tmp;
10147         int i;
10148
10149         if (offset > EEPROM_ADDR_ADDR_MASK ||
10150             (offset % 4) != 0)
10151                 return -EINVAL;
10152
10153         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
10154                                         EEPROM_ADDR_DEVID_MASK |
10155                                         EEPROM_ADDR_READ);
10156         tw32(GRC_EEPROM_ADDR,
10157              tmp |
10158              (0 << EEPROM_ADDR_DEVID_SHIFT) |
10159              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
10160               EEPROM_ADDR_ADDR_MASK) |
10161              EEPROM_ADDR_READ | EEPROM_ADDR_START);
10162
10163         for (i = 0; i < 1000; i++) {
10164                 tmp = tr32(GRC_EEPROM_ADDR);
10165
10166                 if (tmp & EEPROM_ADDR_COMPLETE)
10167                         break;
10168                 msleep(1);
10169         }
10170         if (!(tmp & EEPROM_ADDR_COMPLETE))
10171                 return -EBUSY;
10172
10173         *val = tr32(GRC_EEPROM_DATA);
10174         return 0;
10175 }
10176
10177 #define NVRAM_CMD_TIMEOUT 10000
10178
10179 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
10180 {
10181         int i;
10182
10183         tw32(NVRAM_CMD, nvram_cmd);
10184         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
10185                 udelay(10);
10186                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
10187                         udelay(10);
10188                         break;
10189                 }
10190         }
10191         if (i == NVRAM_CMD_TIMEOUT) {
10192                 return -EBUSY;
10193         }
10194         return 0;
10195 }
10196
10197 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
10198 {
10199         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10200             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10201             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10202            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10203             (tp->nvram_jedecnum == JEDEC_ATMEL))
10204
10205                 addr = ((addr / tp->nvram_pagesize) <<
10206                         ATMEL_AT45DB0X1B_PAGE_POS) +
10207                        (addr % tp->nvram_pagesize);
10208
10209         return addr;
10210 }
10211
10212 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
10213 {
10214         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10215             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10216             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10217            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10218             (tp->nvram_jedecnum == JEDEC_ATMEL))
10219
10220                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
10221                         tp->nvram_pagesize) +
10222                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
10223
10224         return addr;
10225 }
10226
10227 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
10228 {
10229         int ret;
10230
10231         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
10232                 return tg3_nvram_read_using_eeprom(tp, offset, val);
10233
10234         offset = tg3_nvram_phys_addr(tp, offset);
10235
10236         if (offset > NVRAM_ADDR_MSK)
10237                 return -EINVAL;
10238
10239         ret = tg3_nvram_lock(tp);
10240         if (ret)
10241                 return ret;
10242
10243         tg3_enable_nvram_access(tp);
10244
10245         tw32(NVRAM_ADDR, offset);
10246         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
10247                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
10248
10249         if (ret == 0)
10250                 *val = swab32(tr32(NVRAM_RDDATA));
10251
10252         tg3_disable_nvram_access(tp);
10253
10254         tg3_nvram_unlock(tp);
10255
10256         return ret;
10257 }
10258
10259 static int tg3_nvram_read_le(struct tg3 *tp, u32 offset, __le32 *val)
10260 {
10261         u32 v;
10262         int res = tg3_nvram_read(tp, offset, &v);
10263         if (!res)
10264                 *val = cpu_to_le32(v);
10265         return res;
10266 }
10267
10268 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val)
10269 {
10270         int err;
10271         u32 tmp;
10272
10273         err = tg3_nvram_read(tp, offset, &tmp);
10274         *val = swab32(tmp);
10275         return err;
10276 }
10277
10278 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
10279                                     u32 offset, u32 len, u8 *buf)
10280 {
10281         int i, j, rc = 0;
10282         u32 val;
10283
10284         for (i = 0; i < len; i += 4) {
10285                 u32 addr;
10286                 __le32 data;
10287
10288                 addr = offset + i;
10289
10290                 memcpy(&data, buf + i, 4);
10291
10292                 tw32(GRC_EEPROM_DATA, le32_to_cpu(data));
10293
10294                 val = tr32(GRC_EEPROM_ADDR);
10295                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
10296
10297                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
10298                         EEPROM_ADDR_READ);
10299                 tw32(GRC_EEPROM_ADDR, val |
10300                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
10301                         (addr & EEPROM_ADDR_ADDR_MASK) |
10302                         EEPROM_ADDR_START |
10303                         EEPROM_ADDR_WRITE);
10304
10305                 for (j = 0; j < 1000; j++) {
10306                         val = tr32(GRC_EEPROM_ADDR);
10307
10308                         if (val & EEPROM_ADDR_COMPLETE)
10309                                 break;
10310                         msleep(1);
10311                 }
10312                 if (!(val & EEPROM_ADDR_COMPLETE)) {
10313                         rc = -EBUSY;
10314                         break;
10315                 }
10316         }
10317
10318         return rc;
10319 }
10320
10321 /* offset and length are dword aligned */
10322 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
10323                 u8 *buf)
10324 {
10325         int ret = 0;
10326         u32 pagesize = tp->nvram_pagesize;
10327         u32 pagemask = pagesize - 1;
10328         u32 nvram_cmd;
10329         u8 *tmp;
10330
10331         tmp = kmalloc(pagesize, GFP_KERNEL);
10332         if (tmp == NULL)
10333                 return -ENOMEM;
10334
10335         while (len) {
10336                 int j;
10337                 u32 phy_addr, page_off, size;
10338
10339                 phy_addr = offset & ~pagemask;
10340
10341                 for (j = 0; j < pagesize; j += 4) {
10342                         if ((ret = tg3_nvram_read_le(tp, phy_addr + j,
10343                                                 (__le32 *) (tmp + j))))
10344                                 break;
10345                 }
10346                 if (ret)
10347                         break;
10348
10349                 page_off = offset & pagemask;
10350                 size = pagesize;
10351                 if (len < size)
10352                         size = len;
10353
10354                 len -= size;
10355
10356                 memcpy(tmp + page_off, buf, size);
10357
10358                 offset = offset + (pagesize - page_off);
10359
10360                 tg3_enable_nvram_access(tp);
10361
10362                 /*
10363                  * Before we can erase the flash page, we need
10364                  * to issue a special "write enable" command.
10365                  */
10366                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10367
10368                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10369                         break;
10370
10371                 /* Erase the target page */
10372                 tw32(NVRAM_ADDR, phy_addr);
10373
10374                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
10375                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
10376
10377                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10378                         break;
10379
10380                 /* Issue another write enable to start the write. */
10381                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10382
10383                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10384                         break;
10385
10386                 for (j = 0; j < pagesize; j += 4) {
10387                         __be32 data;
10388
10389                         data = *((__be32 *) (tmp + j));
10390                         /* swab32(le32_to_cpu(data)), actually */
10391                         tw32(NVRAM_WRDATA, be32_to_cpu(data));
10392
10393                         tw32(NVRAM_ADDR, phy_addr + j);
10394
10395                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
10396                                 NVRAM_CMD_WR;
10397
10398                         if (j == 0)
10399                                 nvram_cmd |= NVRAM_CMD_FIRST;
10400                         else if (j == (pagesize - 4))
10401                                 nvram_cmd |= NVRAM_CMD_LAST;
10402
10403                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10404                                 break;
10405                 }
10406                 if (ret)
10407                         break;
10408         }
10409
10410         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10411         tg3_nvram_exec_cmd(tp, nvram_cmd);
10412
10413         kfree(tmp);
10414
10415         return ret;
10416 }
10417
10418 /* offset and length are dword aligned */
10419 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
10420                 u8 *buf)
10421 {
10422         int i, ret = 0;
10423
10424         for (i = 0; i < len; i += 4, offset += 4) {
10425                 u32 page_off, phy_addr, nvram_cmd;
10426                 __be32 data;
10427
10428                 memcpy(&data, buf + i, 4);
10429                 tw32(NVRAM_WRDATA, be32_to_cpu(data));
10430
10431                 page_off = offset % tp->nvram_pagesize;
10432
10433                 phy_addr = tg3_nvram_phys_addr(tp, offset);
10434
10435                 tw32(NVRAM_ADDR, phy_addr);
10436
10437                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
10438
10439                 if ((page_off == 0) || (i == 0))
10440                         nvram_cmd |= NVRAM_CMD_FIRST;
10441                 if (page_off == (tp->nvram_pagesize - 4))
10442                         nvram_cmd |= NVRAM_CMD_LAST;
10443
10444                 if (i == (len - 4))
10445                         nvram_cmd |= NVRAM_CMD_LAST;
10446
10447                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752) &&
10448                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755) &&
10449                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787) &&
10450                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784) &&
10451                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) &&
10452                     (tp->nvram_jedecnum == JEDEC_ST) &&
10453                     (nvram_cmd & NVRAM_CMD_FIRST)) {
10454
10455                         if ((ret = tg3_nvram_exec_cmd(tp,
10456                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
10457                                 NVRAM_CMD_DONE)))
10458
10459                                 break;
10460                 }
10461                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10462                         /* We always do complete word writes to eeprom. */
10463                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
10464                 }
10465
10466                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10467                         break;
10468         }
10469         return ret;
10470 }
10471
10472 /* offset and length are dword aligned */
10473 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
10474 {
10475         int ret;
10476
10477         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10478                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
10479                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
10480                 udelay(40);
10481         }
10482
10483         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
10484                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
10485         }
10486         else {
10487                 u32 grc_mode;
10488
10489                 ret = tg3_nvram_lock(tp);
10490                 if (ret)
10491                         return ret;
10492
10493                 tg3_enable_nvram_access(tp);
10494                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
10495                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
10496                         tw32(NVRAM_WRITE1, 0x406);
10497
10498                 grc_mode = tr32(GRC_MODE);
10499                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
10500
10501                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
10502                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10503
10504                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
10505                                 buf);
10506                 }
10507                 else {
10508                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
10509                                 buf);
10510                 }
10511
10512                 grc_mode = tr32(GRC_MODE);
10513                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
10514
10515                 tg3_disable_nvram_access(tp);
10516                 tg3_nvram_unlock(tp);
10517         }
10518
10519         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10520                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10521                 udelay(40);
10522         }
10523
10524         return ret;
10525 }
10526
10527 struct subsys_tbl_ent {
10528         u16 subsys_vendor, subsys_devid;
10529         u32 phy_id;
10530 };
10531
10532 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
10533         /* Broadcom boards. */
10534         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
10535         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
10536         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
10537         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
10538         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
10539         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
10540         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
10541         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
10542         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
10543         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
10544         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
10545
10546         /* 3com boards. */
10547         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
10548         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
10549         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
10550         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
10551         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
10552
10553         /* DELL boards. */
10554         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
10555         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
10556         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
10557         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
10558
10559         /* Compaq boards. */
10560         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
10561         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
10562         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
10563         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
10564         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
10565
10566         /* IBM boards. */
10567         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
10568 };
10569
10570 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
10571 {
10572         int i;
10573
10574         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
10575                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
10576                      tp->pdev->subsystem_vendor) &&
10577                     (subsys_id_to_phy_id[i].subsys_devid ==
10578                      tp->pdev->subsystem_device))
10579                         return &subsys_id_to_phy_id[i];
10580         }
10581         return NULL;
10582 }
10583
10584 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
10585 {
10586         u32 val;
10587         u16 pmcsr;
10588
10589         /* On some early chips the SRAM cannot be accessed in D3hot state,
10590          * so need make sure we're in D0.
10591          */
10592         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
10593         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
10594         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
10595         msleep(1);
10596
10597         /* Make sure register accesses (indirect or otherwise)
10598          * will function correctly.
10599          */
10600         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10601                                tp->misc_host_ctrl);
10602
10603         /* The memory arbiter has to be enabled in order for SRAM accesses
10604          * to succeed.  Normally on powerup the tg3 chip firmware will make
10605          * sure it is enabled, but other entities such as system netboot
10606          * code might disable it.
10607          */
10608         val = tr32(MEMARB_MODE);
10609         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
10610
10611         tp->phy_id = PHY_ID_INVALID;
10612         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10613
10614         /* Assume an onboard device and WOL capable by default.  */
10615         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT | TG3_FLAG_WOL_CAP;
10616
10617         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
10618                 if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
10619                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10620                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10621                 }
10622                 val = tr32(VCPU_CFGSHDW);
10623                 if (val & VCPU_CFGSHDW_ASPM_DBNC)
10624                         tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10625                 if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
10626                     (val & VCPU_CFGSHDW_WOL_MAGPKT))
10627                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10628                 return;
10629         }
10630
10631         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
10632         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
10633                 u32 nic_cfg, led_cfg;
10634                 u32 nic_phy_id, ver, cfg2 = 0, eeprom_phy_id;
10635                 int eeprom_phy_serdes = 0;
10636
10637                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
10638                 tp->nic_sram_data_cfg = nic_cfg;
10639
10640                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
10641                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
10642                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
10643                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
10644                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
10645                     (ver > 0) && (ver < 0x100))
10646                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
10647
10648                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
10649                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
10650                         eeprom_phy_serdes = 1;
10651
10652                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
10653                 if (nic_phy_id != 0) {
10654                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
10655                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
10656
10657                         eeprom_phy_id  = (id1 >> 16) << 10;
10658                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
10659                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
10660                 } else
10661                         eeprom_phy_id = 0;
10662
10663                 tp->phy_id = eeprom_phy_id;
10664                 if (eeprom_phy_serdes) {
10665                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
10666                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
10667                         else
10668                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10669                 }
10670
10671                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10672                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
10673                                     SHASTA_EXT_LED_MODE_MASK);
10674                 else
10675                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
10676
10677                 switch (led_cfg) {
10678                 default:
10679                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
10680                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10681                         break;
10682
10683                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
10684                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10685                         break;
10686
10687                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
10688                         tp->led_ctrl = LED_CTRL_MODE_MAC;
10689
10690                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
10691                          * read on some older 5700/5701 bootcode.
10692                          */
10693                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
10694                             ASIC_REV_5700 ||
10695                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
10696                             ASIC_REV_5701)
10697                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10698
10699                         break;
10700
10701                 case SHASTA_EXT_LED_SHARED:
10702                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
10703                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
10704                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
10705                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10706                                                  LED_CTRL_MODE_PHY_2);
10707                         break;
10708
10709                 case SHASTA_EXT_LED_MAC:
10710                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
10711                         break;
10712
10713                 case SHASTA_EXT_LED_COMBO:
10714                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
10715                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
10716                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10717                                                  LED_CTRL_MODE_PHY_2);
10718                         break;
10719
10720                 };
10721
10722                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10723                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
10724                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
10725                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10726
10727                 if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
10728                     tp->pci_chip_rev_id == CHIPREV_ID_5784_A1)
10729                         tp->led_ctrl = LED_CTRL_MODE_MAC;
10730
10731                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
10732                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
10733                         if ((tp->pdev->subsystem_vendor ==
10734                              PCI_VENDOR_ID_ARIMA) &&
10735                             (tp->pdev->subsystem_device == 0x205a ||
10736                              tp->pdev->subsystem_device == 0x2063))
10737                                 tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10738                 } else {
10739                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10740                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10741                 }
10742
10743                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
10744                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
10745                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10746                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
10747                 }
10748                 if (nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE)
10749                         tp->tg3_flags3 |= TG3_FLG3_ENABLE_APE;
10750                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES &&
10751                     !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
10752                         tp->tg3_flags &= ~TG3_FLAG_WOL_CAP;
10753
10754                 if (tp->tg3_flags & TG3_FLAG_WOL_CAP &&
10755                     nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)
10756                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10757
10758                 if (cfg2 & (1 << 17))
10759                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
10760
10761                 /* serdes signal pre-emphasis in register 0x590 set by */
10762                 /* bootcode if bit 18 is set */
10763                 if (cfg2 & (1 << 18))
10764                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
10765
10766                 if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10767                         u32 cfg3;
10768
10769                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
10770                         if (cfg3 & NIC_SRAM_ASPM_DEBOUNCE)
10771                                 tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10772                 }
10773         }
10774 }
10775
10776 static int __devinit tg3_phy_probe(struct tg3 *tp)
10777 {
10778         u32 hw_phy_id_1, hw_phy_id_2;
10779         u32 hw_phy_id, hw_phy_id_masked;
10780         int err;
10781
10782         /* Reading the PHY ID register can conflict with ASF
10783          * firwmare access to the PHY hardware.
10784          */
10785         err = 0;
10786         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
10787             (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
10788                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
10789         } else {
10790                 /* Now read the physical PHY_ID from the chip and verify
10791                  * that it is sane.  If it doesn't look good, we fall back
10792                  * to either the hard-coded table based PHY_ID and failing
10793                  * that the value found in the eeprom area.
10794                  */
10795                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
10796                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
10797
10798                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
10799                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
10800                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
10801
10802                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
10803         }
10804
10805         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
10806                 tp->phy_id = hw_phy_id;
10807                 if (hw_phy_id_masked == PHY_ID_BCM8002)
10808                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10809                 else
10810                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
10811         } else {
10812                 if (tp->phy_id != PHY_ID_INVALID) {
10813                         /* Do nothing, phy ID already set up in
10814                          * tg3_get_eeprom_hw_cfg().
10815                          */
10816                 } else {
10817                         struct subsys_tbl_ent *p;
10818
10819                         /* No eeprom signature?  Try the hardcoded
10820                          * subsys device table.
10821                          */
10822                         p = lookup_by_subsys(tp);
10823                         if (!p)
10824                                 return -ENODEV;
10825
10826                         tp->phy_id = p->phy_id;
10827                         if (!tp->phy_id ||
10828                             tp->phy_id == PHY_ID_BCM8002)
10829                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10830                 }
10831         }
10832
10833         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
10834             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) &&
10835             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
10836                 u32 bmsr, adv_reg, tg3_ctrl, mask;
10837
10838                 tg3_readphy(tp, MII_BMSR, &bmsr);
10839                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
10840                     (bmsr & BMSR_LSTATUS))
10841                         goto skip_phy_reset;
10842
10843                 err = tg3_phy_reset(tp);
10844                 if (err)
10845                         return err;
10846
10847                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
10848                            ADVERTISE_100HALF | ADVERTISE_100FULL |
10849                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
10850                 tg3_ctrl = 0;
10851                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
10852                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
10853                                     MII_TG3_CTRL_ADV_1000_FULL);
10854                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
10855                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
10856                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
10857                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
10858                 }
10859
10860                 mask = (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
10861                         ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
10862                         ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
10863                 if (!tg3_copper_is_advertising_all(tp, mask)) {
10864                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
10865
10866                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
10867                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
10868
10869                         tg3_writephy(tp, MII_BMCR,
10870                                      BMCR_ANENABLE | BMCR_ANRESTART);
10871                 }
10872                 tg3_phy_set_wirespeed(tp);
10873
10874                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
10875                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
10876                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
10877         }
10878
10879 skip_phy_reset:
10880         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
10881                 err = tg3_init_5401phy_dsp(tp);
10882                 if (err)
10883                         return err;
10884         }
10885
10886         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
10887                 err = tg3_init_5401phy_dsp(tp);
10888         }
10889
10890         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
10891                 tp->link_config.advertising =
10892                         (ADVERTISED_1000baseT_Half |
10893                          ADVERTISED_1000baseT_Full |
10894                          ADVERTISED_Autoneg |
10895                          ADVERTISED_FIBRE);
10896         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
10897                 tp->link_config.advertising &=
10898                         ~(ADVERTISED_1000baseT_Half |
10899                           ADVERTISED_1000baseT_Full);
10900
10901         return err;
10902 }
10903
10904 static void __devinit tg3_read_partno(struct tg3 *tp)
10905 {
10906         unsigned char vpd_data[256];
10907         unsigned int i;
10908         u32 magic;
10909
10910         if (tg3_nvram_read_swab(tp, 0x0, &magic))
10911                 goto out_not_found;
10912
10913         if (magic == TG3_EEPROM_MAGIC) {
10914                 for (i = 0; i < 256; i += 4) {
10915                         u32 tmp;
10916
10917                         if (tg3_nvram_read(tp, 0x100 + i, &tmp))
10918                                 goto out_not_found;
10919
10920                         vpd_data[i + 0] = ((tmp >>  0) & 0xff);
10921                         vpd_data[i + 1] = ((tmp >>  8) & 0xff);
10922                         vpd_data[i + 2] = ((tmp >> 16) & 0xff);
10923                         vpd_data[i + 3] = ((tmp >> 24) & 0xff);
10924                 }
10925         } else {
10926                 int vpd_cap;
10927
10928                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
10929                 for (i = 0; i < 256; i += 4) {
10930                         u32 tmp, j = 0;
10931                         __le32 v;
10932                         u16 tmp16;
10933
10934                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
10935                                               i);
10936                         while (j++ < 100) {
10937                                 pci_read_config_word(tp->pdev, vpd_cap +
10938                                                      PCI_VPD_ADDR, &tmp16);
10939                                 if (tmp16 & 0x8000)
10940                                         break;
10941                                 msleep(1);
10942                         }
10943                         if (!(tmp16 & 0x8000))
10944                                 goto out_not_found;
10945
10946                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
10947                                               &tmp);
10948                         v = cpu_to_le32(tmp);
10949                         memcpy(&vpd_data[i], &v, 4);
10950                 }
10951         }
10952
10953         /* Now parse and find the part number. */
10954         for (i = 0; i < 254; ) {
10955                 unsigned char val = vpd_data[i];
10956                 unsigned int block_end;
10957
10958                 if (val == 0x82 || val == 0x91) {
10959                         i = (i + 3 +
10960                              (vpd_data[i + 1] +
10961                               (vpd_data[i + 2] << 8)));
10962                         continue;
10963                 }
10964
10965                 if (val != 0x90)
10966                         goto out_not_found;
10967
10968                 block_end = (i + 3 +
10969                              (vpd_data[i + 1] +
10970                               (vpd_data[i + 2] << 8)));
10971                 i += 3;
10972
10973                 if (block_end > 256)
10974                         goto out_not_found;
10975
10976                 while (i < (block_end - 2)) {
10977                         if (vpd_data[i + 0] == 'P' &&
10978                             vpd_data[i + 1] == 'N') {
10979                                 int partno_len = vpd_data[i + 2];
10980
10981                                 i += 3;
10982                                 if (partno_len > 24 || (partno_len + i) > 256)
10983                                         goto out_not_found;
10984
10985                                 memcpy(tp->board_part_number,
10986                                        &vpd_data[i], partno_len);
10987
10988                                 /* Success. */
10989                                 return;
10990                         }
10991                         i += 3 + vpd_data[i + 2];
10992                 }
10993
10994                 /* Part number not found. */
10995                 goto out_not_found;
10996         }
10997
10998 out_not_found:
10999         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11000                 strcpy(tp->board_part_number, "BCM95906");
11001         else
11002                 strcpy(tp->board_part_number, "none");
11003 }
11004
11005 static int __devinit tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
11006 {
11007         u32 val;
11008
11009         if (tg3_nvram_read_swab(tp, offset, &val) ||
11010             (val & 0xfc000000) != 0x0c000000 ||
11011             tg3_nvram_read_swab(tp, offset + 4, &val) ||
11012             val != 0)
11013                 return 0;
11014
11015         return 1;
11016 }
11017
11018 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
11019 {
11020         u32 val, offset, start;
11021         u32 ver_offset;
11022         int i, bcnt;
11023
11024         if (tg3_nvram_read_swab(tp, 0, &val))
11025                 return;
11026
11027         if (val != TG3_EEPROM_MAGIC)
11028                 return;
11029
11030         if (tg3_nvram_read_swab(tp, 0xc, &offset) ||
11031             tg3_nvram_read_swab(tp, 0x4, &start))
11032                 return;
11033
11034         offset = tg3_nvram_logical_addr(tp, offset);
11035
11036         if (!tg3_fw_img_is_valid(tp, offset) ||
11037             tg3_nvram_read_swab(tp, offset + 8, &ver_offset))
11038                 return;
11039
11040         offset = offset + ver_offset - start;
11041         for (i = 0; i < 16; i += 4) {
11042                 __le32 v;
11043                 if (tg3_nvram_read_le(tp, offset + i, &v))
11044                         return;
11045
11046                 memcpy(tp->fw_ver + i, &v, 4);
11047         }
11048
11049         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
11050              (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
11051                 return;
11052
11053         for (offset = TG3_NVM_DIR_START;
11054              offset < TG3_NVM_DIR_END;
11055              offset += TG3_NVM_DIRENT_SIZE) {
11056                 if (tg3_nvram_read_swab(tp, offset, &val))
11057                         return;
11058
11059                 if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
11060                         break;
11061         }
11062
11063         if (offset == TG3_NVM_DIR_END)
11064                 return;
11065
11066         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
11067                 start = 0x08000000;
11068         else if (tg3_nvram_read_swab(tp, offset - 4, &start))
11069                 return;
11070
11071         if (tg3_nvram_read_swab(tp, offset + 4, &offset) ||
11072             !tg3_fw_img_is_valid(tp, offset) ||
11073             tg3_nvram_read_swab(tp, offset + 8, &val))
11074                 return;
11075
11076         offset += val - start;
11077
11078         bcnt = strlen(tp->fw_ver);
11079
11080         tp->fw_ver[bcnt++] = ',';
11081         tp->fw_ver[bcnt++] = ' ';
11082
11083         for (i = 0; i < 4; i++) {
11084                 __le32 v;
11085                 if (tg3_nvram_read_le(tp, offset, &v))
11086                         return;
11087
11088                 offset += sizeof(v);
11089
11090                 if (bcnt > TG3_VER_SIZE - sizeof(v)) {
11091                         memcpy(&tp->fw_ver[bcnt], &v, TG3_VER_SIZE - bcnt);
11092                         break;
11093                 }
11094
11095                 memcpy(&tp->fw_ver[bcnt], &v, sizeof(v));
11096                 bcnt += sizeof(v);
11097         }
11098
11099         tp->fw_ver[TG3_VER_SIZE - 1] = 0;
11100 }
11101
11102 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *);
11103
11104 static int __devinit tg3_get_invariants(struct tg3 *tp)
11105 {
11106         static struct pci_device_id write_reorder_chipsets[] = {
11107                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
11108                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
11109                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
11110                              PCI_DEVICE_ID_AMD_8131_BRIDGE) },
11111                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
11112                              PCI_DEVICE_ID_VIA_8385_0) },
11113                 { },
11114         };
11115         u32 misc_ctrl_reg;
11116         u32 cacheline_sz_reg;
11117         u32 pci_state_reg, grc_misc_cfg;
11118         u32 val;
11119         u16 pci_cmd;
11120         int err, pcie_cap;
11121
11122         /* Force memory write invalidate off.  If we leave it on,
11123          * then on 5700_BX chips we have to enable a workaround.
11124          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
11125          * to match the cacheline size.  The Broadcom driver have this
11126          * workaround but turns MWI off all the times so never uses
11127          * it.  This seems to suggest that the workaround is insufficient.
11128          */
11129         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11130         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
11131         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11132
11133         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
11134          * has the register indirect write enable bit set before
11135          * we try to access any of the MMIO registers.  It is also
11136          * critical that the PCI-X hw workaround situation is decided
11137          * before that as well.
11138          */
11139         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11140                               &misc_ctrl_reg);
11141
11142         tp->pci_chip_rev_id = (misc_ctrl_reg >>
11143                                MISC_HOST_CTRL_CHIPREV_SHIFT);
11144         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_USE_PROD_ID_REG) {
11145                 u32 prod_id_asic_rev;
11146
11147                 pci_read_config_dword(tp->pdev, TG3PCI_PRODID_ASICREV,
11148                                       &prod_id_asic_rev);
11149                 tp->pci_chip_rev_id = prod_id_asic_rev & PROD_ID_ASIC_REV_MASK;
11150         }
11151
11152         /* Wrong chip ID in 5752 A0. This code can be removed later
11153          * as A0 is not in production.
11154          */
11155         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
11156                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
11157
11158         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
11159          * we need to disable memory and use config. cycles
11160          * only to access all registers. The 5702/03 chips
11161          * can mistakenly decode the special cycles from the
11162          * ICH chipsets as memory write cycles, causing corruption
11163          * of register and memory space. Only certain ICH bridges
11164          * will drive special cycles with non-zero data during the
11165          * address phase which can fall within the 5703's address
11166          * range. This is not an ICH bug as the PCI spec allows
11167          * non-zero address during special cycles. However, only
11168          * these ICH bridges are known to drive non-zero addresses
11169          * during special cycles.
11170          *
11171          * Since special cycles do not cross PCI bridges, we only
11172          * enable this workaround if the 5703 is on the secondary
11173          * bus of these ICH bridges.
11174          */
11175         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
11176             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
11177                 static struct tg3_dev_id {
11178                         u32     vendor;
11179                         u32     device;
11180                         u32     rev;
11181                 } ich_chipsets[] = {
11182                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
11183                           PCI_ANY_ID },
11184                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
11185                           PCI_ANY_ID },
11186                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
11187                           0xa },
11188                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
11189                           PCI_ANY_ID },
11190                         { },
11191                 };
11192                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
11193                 struct pci_dev *bridge = NULL;
11194
11195                 while (pci_id->vendor != 0) {
11196                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
11197                                                 bridge);
11198                         if (!bridge) {
11199                                 pci_id++;
11200                                 continue;
11201                         }
11202                         if (pci_id->rev != PCI_ANY_ID) {
11203                                 if (bridge->revision > pci_id->rev)
11204                                         continue;
11205                         }
11206                         if (bridge->subordinate &&
11207                             (bridge->subordinate->number ==
11208                              tp->pdev->bus->number)) {
11209
11210                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
11211                                 pci_dev_put(bridge);
11212                                 break;
11213                         }
11214                 }
11215         }
11216
11217         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
11218          * DMA addresses > 40-bit. This bridge may have other additional
11219          * 57xx devices behind it in some 4-port NIC designs for example.
11220          * Any tg3 device found behind the bridge will also need the 40-bit
11221          * DMA workaround.
11222          */
11223         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
11224             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
11225                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
11226                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11227                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
11228         }
11229         else {
11230                 struct pci_dev *bridge = NULL;
11231
11232                 do {
11233                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
11234                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
11235                                                 bridge);
11236                         if (bridge && bridge->subordinate &&
11237                             (bridge->subordinate->number <=
11238                              tp->pdev->bus->number) &&
11239                             (bridge->subordinate->subordinate >=
11240                              tp->pdev->bus->number)) {
11241                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11242                                 pci_dev_put(bridge);
11243                                 break;
11244                         }
11245                 } while (bridge);
11246         }
11247
11248         /* Initialize misc host control in PCI block. */
11249         tp->misc_host_ctrl |= (misc_ctrl_reg &
11250                                MISC_HOST_CTRL_CHIPREV);
11251         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11252                                tp->misc_host_ctrl);
11253
11254         pci_read_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11255                               &cacheline_sz_reg);
11256
11257         tp->pci_cacheline_sz = (cacheline_sz_reg >>  0) & 0xff;
11258         tp->pci_lat_timer    = (cacheline_sz_reg >>  8) & 0xff;
11259         tp->pci_hdr_type     = (cacheline_sz_reg >> 16) & 0xff;
11260         tp->pci_bist         = (cacheline_sz_reg >> 24) & 0xff;
11261
11262         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11263             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714))
11264                 tp->pdev_peer = tg3_find_peer(tp);
11265
11266         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11267             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11268             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11269             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11270             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11271             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11272             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
11273             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11274                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
11275
11276         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
11277             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
11278                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
11279
11280         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
11281                 tp->tg3_flags |= TG3_FLAG_SUPPORT_MSI;
11282                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX ||
11283                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX ||
11284                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 &&
11285                      tp->pci_chip_rev_id <= CHIPREV_ID_5714_A2 &&
11286                      tp->pdev_peer == tp->pdev))
11287                         tp->tg3_flags &= ~TG3_FLAG_SUPPORT_MSI;
11288
11289                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11290                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11291                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11292                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11293                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11294                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
11295                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
11296                 } else {
11297                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1 | TG3_FLG2_TSO_BUG;
11298                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
11299                                 ASIC_REV_5750 &&
11300                             tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
11301                                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_BUG;
11302                 }
11303         }
11304
11305         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
11306             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
11307             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
11308             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755 &&
11309             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787 &&
11310             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
11311             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761 &&
11312             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11313                 tp->tg3_flags2 |= TG3_FLG2_JUMBO_CAPABLE;
11314
11315         pcie_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_EXP);
11316         if (pcie_cap != 0) {
11317                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
11318
11319                 pcie_set_readrq(tp->pdev, 4096);
11320
11321                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11322                         u16 lnkctl;
11323
11324                         pci_read_config_word(tp->pdev,
11325                                              pcie_cap + PCI_EXP_LNKCTL,
11326                                              &lnkctl);
11327                         if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN)
11328                                 tp->tg3_flags2 &= ~TG3_FLG2_HW_TSO_2;
11329                 }
11330         }
11331
11332         /* If we have an AMD 762 or VIA K8T800 chipset, write
11333          * reordering to the mailbox registers done by the host
11334          * controller can cause major troubles.  We read back from
11335          * every mailbox register write to force the writes to be
11336          * posted to the chip in order.
11337          */
11338         if (pci_dev_present(write_reorder_chipsets) &&
11339             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
11340                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
11341
11342         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11343             tp->pci_lat_timer < 64) {
11344                 tp->pci_lat_timer = 64;
11345
11346                 cacheline_sz_reg  = ((tp->pci_cacheline_sz & 0xff) <<  0);
11347                 cacheline_sz_reg |= ((tp->pci_lat_timer    & 0xff) <<  8);
11348                 cacheline_sz_reg |= ((tp->pci_hdr_type     & 0xff) << 16);
11349                 cacheline_sz_reg |= ((tp->pci_bist         & 0xff) << 24);
11350
11351                 pci_write_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11352                                        cacheline_sz_reg);
11353         }
11354
11355         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
11356             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11357                 tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
11358                 if (!tp->pcix_cap) {
11359                         printk(KERN_ERR PFX "Cannot find PCI-X "
11360                                             "capability, aborting.\n");
11361                         return -EIO;
11362                 }
11363         }
11364
11365         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11366                               &pci_state_reg);
11367
11368         if (tp->pcix_cap && (pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0) {
11369                 tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
11370
11371                 /* If this is a 5700 BX chipset, and we are in PCI-X
11372                  * mode, enable register write workaround.
11373                  *
11374                  * The workaround is to use indirect register accesses
11375                  * for all chip writes not to mailbox registers.
11376                  */
11377                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
11378                         u32 pm_reg;
11379
11380                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11381
11382                         /* The chip can have it's power management PCI config
11383                          * space registers clobbered due to this bug.
11384                          * So explicitly force the chip into D0 here.
11385                          */
11386                         pci_read_config_dword(tp->pdev,
11387                                               tp->pm_cap + PCI_PM_CTRL,
11388                                               &pm_reg);
11389                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
11390                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
11391                         pci_write_config_dword(tp->pdev,
11392                                                tp->pm_cap + PCI_PM_CTRL,
11393                                                pm_reg);
11394
11395                         /* Also, force SERR#/PERR# in PCI command. */
11396                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11397                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
11398                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11399                 }
11400         }
11401
11402         /* 5700 BX chips need to have their TX producer index mailboxes
11403          * written twice to workaround a bug.
11404          */
11405         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX)
11406                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
11407
11408         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
11409                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
11410         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
11411                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
11412
11413         /* Chip-specific fixup from Broadcom driver */
11414         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
11415             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
11416                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
11417                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
11418         }
11419
11420         /* Default fast path register access methods */
11421         tp->read32 = tg3_read32;
11422         tp->write32 = tg3_write32;
11423         tp->read32_mbox = tg3_read32;
11424         tp->write32_mbox = tg3_write32;
11425         tp->write32_tx_mbox = tg3_write32;
11426         tp->write32_rx_mbox = tg3_write32;
11427
11428         /* Various workaround register access methods */
11429         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
11430                 tp->write32 = tg3_write_indirect_reg32;
11431         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
11432                  ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
11433                   tp->pci_chip_rev_id == CHIPREV_ID_5750_A0)) {
11434                 /*
11435                  * Back to back register writes can cause problems on these
11436                  * chips, the workaround is to read back all reg writes
11437                  * except those to mailbox regs.
11438                  *
11439                  * See tg3_write_indirect_reg32().
11440                  */
11441                 tp->write32 = tg3_write_flush_reg32;
11442         }
11443
11444
11445         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
11446             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
11447                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
11448                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
11449                         tp->write32_rx_mbox = tg3_write_flush_reg32;
11450         }
11451
11452         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
11453                 tp->read32 = tg3_read_indirect_reg32;
11454                 tp->write32 = tg3_write_indirect_reg32;
11455                 tp->read32_mbox = tg3_read_indirect_mbox;
11456                 tp->write32_mbox = tg3_write_indirect_mbox;
11457                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
11458                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
11459
11460                 iounmap(tp->regs);
11461                 tp->regs = NULL;
11462
11463                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11464                 pci_cmd &= ~PCI_COMMAND_MEMORY;
11465                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11466         }
11467         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11468                 tp->read32_mbox = tg3_read32_mbox_5906;
11469                 tp->write32_mbox = tg3_write32_mbox_5906;
11470                 tp->write32_tx_mbox = tg3_write32_mbox_5906;
11471                 tp->write32_rx_mbox = tg3_write32_mbox_5906;
11472         }
11473
11474         if (tp->write32 == tg3_write_indirect_reg32 ||
11475             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
11476              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11477               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
11478                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
11479
11480         /* Get eeprom hw config before calling tg3_set_power_state().
11481          * In particular, the TG3_FLG2_IS_NIC flag must be
11482          * determined before calling tg3_set_power_state() so that
11483          * we know whether or not to switch out of Vaux power.
11484          * When the flag is set, it means that GPIO1 is used for eeprom
11485          * write protect and also implies that it is a LOM where GPIOs
11486          * are not used to switch power.
11487          */
11488         tg3_get_eeprom_hw_cfg(tp);
11489
11490         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
11491                 /* Allow reads and writes to the
11492                  * APE register and memory space.
11493                  */
11494                 pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
11495                                  PCISTATE_ALLOW_APE_SHMEM_WR;
11496                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
11497                                        pci_state_reg);
11498         }
11499
11500         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11501             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
11502                 tp->tg3_flags |= TG3_FLAG_CPMU_PRESENT;
11503
11504                 if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
11505                     tp->pci_chip_rev_id == CHIPREV_ID_5784_A1 ||
11506                     tp->pci_chip_rev_id == CHIPREV_ID_5761_A0 ||
11507                     tp->pci_chip_rev_id == CHIPREV_ID_5761_A1)
11508                         tp->tg3_flags3 |= TG3_FLG3_5761_5784_AX_FIXES;
11509         }
11510
11511         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
11512          * GPIO1 driven high will bring 5700's external PHY out of reset.
11513          * It is also used as eeprom write protect on LOMs.
11514          */
11515         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
11516         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11517             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
11518                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
11519                                        GRC_LCLCTRL_GPIO_OUTPUT1);
11520         /* Unused GPIO3 must be driven as output on 5752 because there
11521          * are no pull-up resistors on unused GPIO pins.
11522          */
11523         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
11524                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
11525
11526         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11527                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
11528
11529         /* Force the chip into D0. */
11530         err = tg3_set_power_state(tp, PCI_D0);
11531         if (err) {
11532                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
11533                        pci_name(tp->pdev));
11534                 return err;
11535         }
11536
11537         /* 5700 B0 chips do not support checksumming correctly due
11538          * to hardware bugs.
11539          */
11540         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
11541                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
11542
11543         /* Derive initial jumbo mode from MTU assigned in
11544          * ether_setup() via the alloc_etherdev() call
11545          */
11546         if (tp->dev->mtu > ETH_DATA_LEN &&
11547             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11548                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
11549
11550         /* Determine WakeOnLan speed to use. */
11551         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11552             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
11553             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
11554             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
11555                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
11556         } else {
11557                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
11558         }
11559
11560         /* A few boards don't want Ethernet@WireSpeed phy feature */
11561         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11562             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
11563              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
11564              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
11565             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) ||
11566             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
11567                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
11568
11569         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
11570             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
11571                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
11572         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
11573                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
11574
11575         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11576                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11577                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11578                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11579                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
11580                         if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
11581                             tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
11582                                 tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
11583                         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
11584                                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADJUST_TRIM;
11585                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11586                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
11587         }
11588
11589         tp->coalesce_mode = 0;
11590         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
11591             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
11592                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
11593
11594         /* Initialize MAC MI mode, polling disabled. */
11595         tw32_f(MAC_MI_MODE, tp->mi_mode);
11596         udelay(80);
11597
11598         /* Initialize data/descriptor byte/word swapping. */
11599         val = tr32(GRC_MODE);
11600         val &= GRC_MODE_HOST_STACKUP;
11601         tw32(GRC_MODE, val | tp->grc_mode);
11602
11603         tg3_switch_clocks(tp);
11604
11605         /* Clear this out for sanity. */
11606         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
11607
11608         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11609                               &pci_state_reg);
11610         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
11611             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
11612                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
11613
11614                 if (chiprevid == CHIPREV_ID_5701_A0 ||
11615                     chiprevid == CHIPREV_ID_5701_B0 ||
11616                     chiprevid == CHIPREV_ID_5701_B2 ||
11617                     chiprevid == CHIPREV_ID_5701_B5) {
11618                         void __iomem *sram_base;
11619
11620                         /* Write some dummy words into the SRAM status block
11621                          * area, see if it reads back correctly.  If the return
11622                          * value is bad, force enable the PCIX workaround.
11623                          */
11624                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
11625
11626                         writel(0x00000000, sram_base);
11627                         writel(0x00000000, sram_base + 4);
11628                         writel(0xffffffff, sram_base + 4);
11629                         if (readl(sram_base) != 0x00000000)
11630                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11631                 }
11632         }
11633
11634         udelay(50);
11635         tg3_nvram_init(tp);
11636
11637         grc_misc_cfg = tr32(GRC_MISC_CFG);
11638         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
11639
11640         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11641             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
11642              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
11643                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
11644
11645         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
11646             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
11647                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
11648         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
11649                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
11650                                       HOSTCC_MODE_CLRTICK_TXBD);
11651
11652                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
11653                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11654                                        tp->misc_host_ctrl);
11655         }
11656
11657         /* these are limited to 10/100 only */
11658         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11659              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
11660             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11661              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11662              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
11663               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
11664               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
11665             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11666              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
11667               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F ||
11668               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5787F)) ||
11669             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11670                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
11671
11672         err = tg3_phy_probe(tp);
11673         if (err) {
11674                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
11675                        pci_name(tp->pdev), err);
11676                 /* ... but do not return immediately ... */
11677         }
11678
11679         tg3_read_partno(tp);
11680         tg3_read_fw_ver(tp);
11681
11682         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
11683                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11684         } else {
11685                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11686                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
11687                 else
11688                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11689         }
11690
11691         /* 5700 {AX,BX} chips have a broken status block link
11692          * change bit implementation, so we must use the
11693          * status register in those cases.
11694          */
11695         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11696                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
11697         else
11698                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
11699
11700         /* The led_ctrl is set during tg3_phy_probe, here we might
11701          * have to force the link status polling mechanism based
11702          * upon subsystem IDs.
11703          */
11704         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
11705             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11706             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
11707                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
11708                                   TG3_FLAG_USE_LINKCHG_REG);
11709         }
11710
11711         /* For all SERDES we poll the MAC status register. */
11712         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
11713                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
11714         else
11715                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
11716
11717         /* All chips before 5787 can get confused if TX buffers
11718          * straddle the 4GB address boundary in some cases.
11719          */
11720         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11721             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11722             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11723             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11724             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11725                 tp->dev->hard_start_xmit = tg3_start_xmit;
11726         else
11727                 tp->dev->hard_start_xmit = tg3_start_xmit_dma_bug;
11728
11729         tp->rx_offset = 2;
11730         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11731             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
11732                 tp->rx_offset = 0;
11733
11734         tp->rx_std_max_post = TG3_RX_RING_SIZE;
11735
11736         /* Increment the rx prod index on the rx std ring by at most
11737          * 8 for these chips to workaround hw errata.
11738          */
11739         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11740             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11741             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11742                 tp->rx_std_max_post = 8;
11743
11744         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND)
11745                 tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
11746                                      PCIE_PWR_MGMT_L1_THRESH_MSK;
11747
11748         return err;
11749 }
11750
11751 #ifdef CONFIG_SPARC
11752 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
11753 {
11754         struct net_device *dev = tp->dev;
11755         struct pci_dev *pdev = tp->pdev;
11756         struct device_node *dp = pci_device_to_OF_node(pdev);
11757         const unsigned char *addr;
11758         int len;
11759
11760         addr = of_get_property(dp, "local-mac-address", &len);
11761         if (addr && len == 6) {
11762                 memcpy(dev->dev_addr, addr, 6);
11763                 memcpy(dev->perm_addr, dev->dev_addr, 6);
11764                 return 0;
11765         }
11766         return -ENODEV;
11767 }
11768
11769 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
11770 {
11771         struct net_device *dev = tp->dev;
11772
11773         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
11774         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
11775         return 0;
11776 }
11777 #endif
11778
11779 static int __devinit tg3_get_device_address(struct tg3 *tp)
11780 {
11781         struct net_device *dev = tp->dev;
11782         u32 hi, lo, mac_offset;
11783         int addr_ok = 0;
11784
11785 #ifdef CONFIG_SPARC
11786         if (!tg3_get_macaddr_sparc(tp))
11787                 return 0;
11788 #endif
11789
11790         mac_offset = 0x7c;
11791         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11792             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11793                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
11794                         mac_offset = 0xcc;
11795                 if (tg3_nvram_lock(tp))
11796                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
11797                 else
11798                         tg3_nvram_unlock(tp);
11799         }
11800         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11801                 mac_offset = 0x10;
11802
11803         /* First try to get it from MAC address mailbox. */
11804         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
11805         if ((hi >> 16) == 0x484b) {
11806                 dev->dev_addr[0] = (hi >>  8) & 0xff;
11807                 dev->dev_addr[1] = (hi >>  0) & 0xff;
11808
11809                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
11810                 dev->dev_addr[2] = (lo >> 24) & 0xff;
11811                 dev->dev_addr[3] = (lo >> 16) & 0xff;
11812                 dev->dev_addr[4] = (lo >>  8) & 0xff;
11813                 dev->dev_addr[5] = (lo >>  0) & 0xff;
11814
11815                 /* Some old bootcode may report a 0 MAC address in SRAM */
11816                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
11817         }
11818         if (!addr_ok) {
11819                 /* Next, try NVRAM. */
11820                 if (!tg3_nvram_read(tp, mac_offset + 0, &hi) &&
11821                     !tg3_nvram_read(tp, mac_offset + 4, &lo)) {
11822                         dev->dev_addr[0] = ((hi >> 16) & 0xff);
11823                         dev->dev_addr[1] = ((hi >> 24) & 0xff);
11824                         dev->dev_addr[2] = ((lo >>  0) & 0xff);
11825                         dev->dev_addr[3] = ((lo >>  8) & 0xff);
11826                         dev->dev_addr[4] = ((lo >> 16) & 0xff);
11827                         dev->dev_addr[5] = ((lo >> 24) & 0xff);
11828                 }
11829                 /* Finally just fetch it out of the MAC control regs. */
11830                 else {
11831                         hi = tr32(MAC_ADDR_0_HIGH);
11832                         lo = tr32(MAC_ADDR_0_LOW);
11833
11834                         dev->dev_addr[5] = lo & 0xff;
11835                         dev->dev_addr[4] = (lo >> 8) & 0xff;
11836                         dev->dev_addr[3] = (lo >> 16) & 0xff;
11837                         dev->dev_addr[2] = (lo >> 24) & 0xff;
11838                         dev->dev_addr[1] = hi & 0xff;
11839                         dev->dev_addr[0] = (hi >> 8) & 0xff;
11840                 }
11841         }
11842
11843         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
11844 #ifdef CONFIG_SPARC64
11845                 if (!tg3_get_default_macaddr_sparc(tp))
11846                         return 0;
11847 #endif
11848                 return -EINVAL;
11849         }
11850         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
11851         return 0;
11852 }
11853
11854 #define BOUNDARY_SINGLE_CACHELINE       1
11855 #define BOUNDARY_MULTI_CACHELINE        2
11856
11857 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
11858 {
11859         int cacheline_size;
11860         u8 byte;
11861         int goal;
11862
11863         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
11864         if (byte == 0)
11865                 cacheline_size = 1024;
11866         else
11867                 cacheline_size = (int) byte * 4;
11868
11869         /* On 5703 and later chips, the boundary bits have no
11870          * effect.
11871          */
11872         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
11873             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
11874             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
11875                 goto out;
11876
11877 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
11878         goal = BOUNDARY_MULTI_CACHELINE;
11879 #else
11880 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
11881         goal = BOUNDARY_SINGLE_CACHELINE;
11882 #else
11883         goal = 0;
11884 #endif
11885 #endif
11886
11887         if (!goal)
11888                 goto out;
11889
11890         /* PCI controllers on most RISC systems tend to disconnect
11891          * when a device tries to burst across a cache-line boundary.
11892          * Therefore, letting tg3 do so just wastes PCI bandwidth.
11893          *
11894          * Unfortunately, for PCI-E there are only limited
11895          * write-side controls for this, and thus for reads
11896          * we will still get the disconnects.  We'll also waste
11897          * these PCI cycles for both read and write for chips
11898          * other than 5700 and 5701 which do not implement the
11899          * boundary bits.
11900          */
11901         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
11902             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
11903                 switch (cacheline_size) {
11904                 case 16:
11905                 case 32:
11906                 case 64:
11907                 case 128:
11908                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11909                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
11910                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
11911                         } else {
11912                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
11913                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
11914                         }
11915                         break;
11916
11917                 case 256:
11918                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
11919                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
11920                         break;
11921
11922                 default:
11923                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
11924                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
11925                         break;
11926                 };
11927         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11928                 switch (cacheline_size) {
11929                 case 16:
11930                 case 32:
11931                 case 64:
11932                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11933                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
11934                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
11935                                 break;
11936                         }
11937                         /* fallthrough */
11938                 case 128:
11939                 default:
11940                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
11941                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
11942                         break;
11943                 };
11944         } else {
11945                 switch (cacheline_size) {
11946                 case 16:
11947                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11948                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
11949                                         DMA_RWCTRL_WRITE_BNDRY_16);
11950                                 break;
11951                         }
11952                         /* fallthrough */
11953                 case 32:
11954                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11955                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
11956                                         DMA_RWCTRL_WRITE_BNDRY_32);
11957                                 break;
11958                         }
11959                         /* fallthrough */
11960                 case 64:
11961                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11962                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
11963                                         DMA_RWCTRL_WRITE_BNDRY_64);
11964                                 break;
11965                         }
11966                         /* fallthrough */
11967                 case 128:
11968                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
11969                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
11970                                         DMA_RWCTRL_WRITE_BNDRY_128);
11971                                 break;
11972                         }
11973                         /* fallthrough */
11974                 case 256:
11975                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
11976                                 DMA_RWCTRL_WRITE_BNDRY_256);
11977                         break;
11978                 case 512:
11979                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
11980                                 DMA_RWCTRL_WRITE_BNDRY_512);
11981                         break;
11982                 case 1024:
11983                 default:
11984                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
11985                                 DMA_RWCTRL_WRITE_BNDRY_1024);
11986                         break;
11987                 };
11988         }
11989
11990 out:
11991         return val;
11992 }
11993
11994 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
11995 {
11996         struct tg3_internal_buffer_desc test_desc;
11997         u32 sram_dma_descs;
11998         int i, ret;
11999
12000         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
12001
12002         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
12003         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
12004         tw32(RDMAC_STATUS, 0);
12005         tw32(WDMAC_STATUS, 0);
12006
12007         tw32(BUFMGR_MODE, 0);
12008         tw32(FTQ_RESET, 0);
12009
12010         test_desc.addr_hi = ((u64) buf_dma) >> 32;
12011         test_desc.addr_lo = buf_dma & 0xffffffff;
12012         test_desc.nic_mbuf = 0x00002100;
12013         test_desc.len = size;
12014
12015         /*
12016          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
12017          * the *second* time the tg3 driver was getting loaded after an
12018          * initial scan.
12019          *
12020          * Broadcom tells me:
12021          *   ...the DMA engine is connected to the GRC block and a DMA
12022          *   reset may affect the GRC block in some unpredictable way...
12023          *   The behavior of resets to individual blocks has not been tested.
12024          *
12025          * Broadcom noted the GRC reset will also reset all sub-components.
12026          */
12027         if (to_device) {
12028                 test_desc.cqid_sqid = (13 << 8) | 2;
12029
12030                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
12031                 udelay(40);
12032         } else {
12033                 test_desc.cqid_sqid = (16 << 8) | 7;
12034
12035                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
12036                 udelay(40);
12037         }
12038         test_desc.flags = 0x00000005;
12039
12040         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
12041                 u32 val;
12042
12043                 val = *(((u32 *)&test_desc) + i);
12044                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
12045                                        sram_dma_descs + (i * sizeof(u32)));
12046                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
12047         }
12048         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
12049
12050         if (to_device) {
12051                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
12052         } else {
12053                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
12054         }
12055
12056         ret = -ENODEV;
12057         for (i = 0; i < 40; i++) {
12058                 u32 val;
12059
12060                 if (to_device)
12061                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
12062                 else
12063                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
12064                 if ((val & 0xffff) == sram_dma_descs) {
12065                         ret = 0;
12066                         break;
12067                 }
12068
12069                 udelay(100);
12070         }
12071
12072         return ret;
12073 }
12074
12075 #define TEST_BUFFER_SIZE        0x2000
12076
12077 static int __devinit tg3_test_dma(struct tg3 *tp)
12078 {
12079         dma_addr_t buf_dma;
12080         u32 *buf, saved_dma_rwctrl;
12081         int ret;
12082
12083         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
12084         if (!buf) {
12085                 ret = -ENOMEM;
12086                 goto out_nofree;
12087         }
12088
12089         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
12090                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
12091
12092         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
12093
12094         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12095                 /* DMA read watermark not used on PCIE */
12096                 tp->dma_rwctrl |= 0x00180000;
12097         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
12098                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
12099                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
12100                         tp->dma_rwctrl |= 0x003f0000;
12101                 else
12102                         tp->dma_rwctrl |= 0x003f000f;
12103         } else {
12104                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
12105                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
12106                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
12107                         u32 read_water = 0x7;
12108
12109                         /* If the 5704 is behind the EPB bridge, we can
12110                          * do the less restrictive ONE_DMA workaround for
12111                          * better performance.
12112                          */
12113                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
12114                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
12115                                 tp->dma_rwctrl |= 0x8000;
12116                         else if (ccval == 0x6 || ccval == 0x7)
12117                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
12118
12119                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703)
12120                                 read_water = 4;
12121                         /* Set bit 23 to enable PCIX hw bug fix */
12122                         tp->dma_rwctrl |=
12123                                 (read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
12124                                 (0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
12125                                 (1 << 23);
12126                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
12127                         /* 5780 always in PCIX mode */
12128                         tp->dma_rwctrl |= 0x00144000;
12129                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
12130                         /* 5714 always in PCIX mode */
12131                         tp->dma_rwctrl |= 0x00148000;
12132                 } else {
12133                         tp->dma_rwctrl |= 0x001b000f;
12134                 }
12135         }
12136
12137         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
12138             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
12139                 tp->dma_rwctrl &= 0xfffffff0;
12140
12141         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12142             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
12143                 /* Remove this if it causes problems for some boards. */
12144                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
12145
12146                 /* On 5700/5701 chips, we need to set this bit.
12147                  * Otherwise the chip will issue cacheline transactions
12148                  * to streamable DMA memory with not all the byte
12149                  * enables turned on.  This is an error on several
12150                  * RISC PCI controllers, in particular sparc64.
12151                  *
12152                  * On 5703/5704 chips, this bit has been reassigned
12153                  * a different meaning.  In particular, it is used
12154                  * on those chips to enable a PCI-X workaround.
12155                  */
12156                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
12157         }
12158
12159         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12160
12161 #if 0
12162         /* Unneeded, already done by tg3_get_invariants.  */
12163         tg3_switch_clocks(tp);
12164 #endif
12165
12166         ret = 0;
12167         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
12168             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
12169                 goto out;
12170
12171         /* It is best to perform DMA test with maximum write burst size
12172          * to expose the 5700/5701 write DMA bug.
12173          */
12174         saved_dma_rwctrl = tp->dma_rwctrl;
12175         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12176         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12177
12178         while (1) {
12179                 u32 *p = buf, i;
12180
12181                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
12182                         p[i] = i;
12183
12184                 /* Send the buffer to the chip. */
12185                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
12186                 if (ret) {
12187                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
12188                         break;
12189                 }
12190
12191 #if 0
12192                 /* validate data reached card RAM correctly. */
12193                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12194                         u32 val;
12195                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
12196                         if (le32_to_cpu(val) != p[i]) {
12197                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
12198                                 /* ret = -ENODEV here? */
12199                         }
12200                         p[i] = 0;
12201                 }
12202 #endif
12203                 /* Now read it back. */
12204                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
12205                 if (ret) {
12206                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
12207
12208                         break;
12209                 }
12210
12211                 /* Verify it. */
12212                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12213                         if (p[i] == i)
12214                                 continue;
12215
12216                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12217                             DMA_RWCTRL_WRITE_BNDRY_16) {
12218                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12219                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12220                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12221                                 break;
12222                         } else {
12223                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
12224                                 ret = -ENODEV;
12225                                 goto out;
12226                         }
12227                 }
12228
12229                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
12230                         /* Success. */
12231                         ret = 0;
12232                         break;
12233                 }
12234         }
12235         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12236             DMA_RWCTRL_WRITE_BNDRY_16) {
12237                 static struct pci_device_id dma_wait_state_chipsets[] = {
12238                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
12239                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
12240                         { },
12241                 };
12242
12243                 /* DMA test passed without adjusting DMA boundary,
12244                  * now look for chipsets that are known to expose the
12245                  * DMA bug without failing the test.
12246                  */
12247                 if (pci_dev_present(dma_wait_state_chipsets)) {
12248                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12249                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12250                 }
12251                 else
12252                         /* Safe to use the calculated DMA boundary. */
12253                         tp->dma_rwctrl = saved_dma_rwctrl;
12254
12255                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12256         }
12257
12258 out:
12259         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
12260 out_nofree:
12261         return ret;
12262 }
12263
12264 static void __devinit tg3_init_link_config(struct tg3 *tp)
12265 {
12266         tp->link_config.advertising =
12267                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
12268                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
12269                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
12270                  ADVERTISED_Autoneg | ADVERTISED_MII);
12271         tp->link_config.speed = SPEED_INVALID;
12272         tp->link_config.duplex = DUPLEX_INVALID;
12273         tp->link_config.autoneg = AUTONEG_ENABLE;
12274         tp->link_config.active_speed = SPEED_INVALID;
12275         tp->link_config.active_duplex = DUPLEX_INVALID;
12276         tp->link_config.phy_is_low_power = 0;
12277         tp->link_config.orig_speed = SPEED_INVALID;
12278         tp->link_config.orig_duplex = DUPLEX_INVALID;
12279         tp->link_config.orig_autoneg = AUTONEG_INVALID;
12280 }
12281
12282 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
12283 {
12284         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12285                 tp->bufmgr_config.mbuf_read_dma_low_water =
12286                         DEFAULT_MB_RDMA_LOW_WATER_5705;
12287                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12288                         DEFAULT_MB_MACRX_LOW_WATER_5705;
12289                 tp->bufmgr_config.mbuf_high_water =
12290                         DEFAULT_MB_HIGH_WATER_5705;
12291                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12292                         tp->bufmgr_config.mbuf_mac_rx_low_water =
12293                                 DEFAULT_MB_MACRX_LOW_WATER_5906;
12294                         tp->bufmgr_config.mbuf_high_water =
12295                                 DEFAULT_MB_HIGH_WATER_5906;
12296                 }
12297
12298                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12299                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
12300                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12301                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
12302                 tp->bufmgr_config.mbuf_high_water_jumbo =
12303                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
12304         } else {
12305                 tp->bufmgr_config.mbuf_read_dma_low_water =
12306                         DEFAULT_MB_RDMA_LOW_WATER;
12307                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12308                         DEFAULT_MB_MACRX_LOW_WATER;
12309                 tp->bufmgr_config.mbuf_high_water =
12310                         DEFAULT_MB_HIGH_WATER;
12311
12312                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12313                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
12314                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12315                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
12316                 tp->bufmgr_config.mbuf_high_water_jumbo =
12317                         DEFAULT_MB_HIGH_WATER_JUMBO;
12318         }
12319
12320         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
12321         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
12322 }
12323
12324 static char * __devinit tg3_phy_string(struct tg3 *tp)
12325 {
12326         switch (tp->phy_id & PHY_ID_MASK) {
12327         case PHY_ID_BCM5400:    return "5400";
12328         case PHY_ID_BCM5401:    return "5401";
12329         case PHY_ID_BCM5411:    return "5411";
12330         case PHY_ID_BCM5701:    return "5701";
12331         case PHY_ID_BCM5703:    return "5703";
12332         case PHY_ID_BCM5704:    return "5704";
12333         case PHY_ID_BCM5705:    return "5705";
12334         case PHY_ID_BCM5750:    return "5750";
12335         case PHY_ID_BCM5752:    return "5752";
12336         case PHY_ID_BCM5714:    return "5714";
12337         case PHY_ID_BCM5780:    return "5780";
12338         case PHY_ID_BCM5755:    return "5755";
12339         case PHY_ID_BCM5787:    return "5787";
12340         case PHY_ID_BCM5784:    return "5784";
12341         case PHY_ID_BCM5756:    return "5722/5756";
12342         case PHY_ID_BCM5906:    return "5906";
12343         case PHY_ID_BCM5761:    return "5761";
12344         case PHY_ID_BCM8002:    return "8002/serdes";
12345         case 0:                 return "serdes";
12346         default:                return "unknown";
12347         };
12348 }
12349
12350 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
12351 {
12352         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12353                 strcpy(str, "PCI Express");
12354                 return str;
12355         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
12356                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
12357
12358                 strcpy(str, "PCIX:");
12359
12360                 if ((clock_ctrl == 7) ||
12361                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
12362                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
12363                         strcat(str, "133MHz");
12364                 else if (clock_ctrl == 0)
12365                         strcat(str, "33MHz");
12366                 else if (clock_ctrl == 2)
12367                         strcat(str, "50MHz");
12368                 else if (clock_ctrl == 4)
12369                         strcat(str, "66MHz");
12370                 else if (clock_ctrl == 6)
12371                         strcat(str, "100MHz");
12372         } else {
12373                 strcpy(str, "PCI:");
12374                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
12375                         strcat(str, "66MHz");
12376                 else
12377                         strcat(str, "33MHz");
12378         }
12379         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
12380                 strcat(str, ":32-bit");
12381         else
12382                 strcat(str, ":64-bit");
12383         return str;
12384 }
12385
12386 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
12387 {
12388         struct pci_dev *peer;
12389         unsigned int func, devnr = tp->pdev->devfn & ~7;
12390
12391         for (func = 0; func < 8; func++) {
12392                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
12393                 if (peer && peer != tp->pdev)
12394                         break;
12395                 pci_dev_put(peer);
12396         }
12397         /* 5704 can be configured in single-port mode, set peer to
12398          * tp->pdev in that case.
12399          */
12400         if (!peer) {
12401                 peer = tp->pdev;
12402                 return peer;
12403         }
12404
12405         /*
12406          * We don't need to keep the refcount elevated; there's no way
12407          * to remove one half of this device without removing the other
12408          */
12409         pci_dev_put(peer);
12410
12411         return peer;
12412 }
12413
12414 static void __devinit tg3_init_coal(struct tg3 *tp)
12415 {
12416         struct ethtool_coalesce *ec = &tp->coal;
12417
12418         memset(ec, 0, sizeof(*ec));
12419         ec->cmd = ETHTOOL_GCOALESCE;
12420         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
12421         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
12422         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
12423         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
12424         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
12425         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
12426         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
12427         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
12428         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
12429
12430         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
12431                                  HOSTCC_MODE_CLRTICK_TXBD)) {
12432                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
12433                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
12434                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
12435                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
12436         }
12437
12438         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12439                 ec->rx_coalesce_usecs_irq = 0;
12440                 ec->tx_coalesce_usecs_irq = 0;
12441                 ec->stats_block_coalesce_usecs = 0;
12442         }
12443 }
12444
12445 static int __devinit tg3_init_one(struct pci_dev *pdev,
12446                                   const struct pci_device_id *ent)
12447 {
12448         static int tg3_version_printed = 0;
12449         unsigned long tg3reg_base, tg3reg_len;
12450         struct net_device *dev;
12451         struct tg3 *tp;
12452         int err, pm_cap;
12453         char str[40];
12454         u64 dma_mask, persist_dma_mask;
12455         DECLARE_MAC_BUF(mac);
12456
12457         if (tg3_version_printed++ == 0)
12458                 printk(KERN_INFO "%s", version);
12459
12460         err = pci_enable_device(pdev);
12461         if (err) {
12462                 printk(KERN_ERR PFX "Cannot enable PCI device, "
12463                        "aborting.\n");
12464                 return err;
12465         }
12466
12467         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12468                 printk(KERN_ERR PFX "Cannot find proper PCI device "
12469                        "base address, aborting.\n");
12470                 err = -ENODEV;
12471                 goto err_out_disable_pdev;
12472         }
12473
12474         err = pci_request_regions(pdev, DRV_MODULE_NAME);
12475         if (err) {
12476                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
12477                        "aborting.\n");
12478                 goto err_out_disable_pdev;
12479         }
12480
12481         pci_set_master(pdev);
12482
12483         /* Find power-management capability. */
12484         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
12485         if (pm_cap == 0) {
12486                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
12487                        "aborting.\n");
12488                 err = -EIO;
12489                 goto err_out_free_res;
12490         }
12491
12492         tg3reg_base = pci_resource_start(pdev, 0);
12493         tg3reg_len = pci_resource_len(pdev, 0);
12494
12495         dev = alloc_etherdev(sizeof(*tp));
12496         if (!dev) {
12497                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
12498                 err = -ENOMEM;
12499                 goto err_out_free_res;
12500         }
12501
12502         SET_NETDEV_DEV(dev, &pdev->dev);
12503
12504 #if TG3_VLAN_TAG_USED
12505         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
12506         dev->vlan_rx_register = tg3_vlan_rx_register;
12507 #endif
12508
12509         tp = netdev_priv(dev);
12510         tp->pdev = pdev;
12511         tp->dev = dev;
12512         tp->pm_cap = pm_cap;
12513         tp->mac_mode = TG3_DEF_MAC_MODE;
12514         tp->rx_mode = TG3_DEF_RX_MODE;
12515         tp->tx_mode = TG3_DEF_TX_MODE;
12516         tp->mi_mode = MAC_MI_MODE_BASE;
12517         if (tg3_debug > 0)
12518                 tp->msg_enable = tg3_debug;
12519         else
12520                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
12521
12522         /* The word/byte swap controls here control register access byte
12523          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
12524          * setting below.
12525          */
12526         tp->misc_host_ctrl =
12527                 MISC_HOST_CTRL_MASK_PCI_INT |
12528                 MISC_HOST_CTRL_WORD_SWAP |
12529                 MISC_HOST_CTRL_INDIR_ACCESS |
12530                 MISC_HOST_CTRL_PCISTATE_RW;
12531
12532         /* The NONFRM (non-frame) byte/word swap controls take effect
12533          * on descriptor entries, anything which isn't packet data.
12534          *
12535          * The StrongARM chips on the board (one for tx, one for rx)
12536          * are running in big-endian mode.
12537          */
12538         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
12539                         GRC_MODE_WSWAP_NONFRM_DATA);
12540 #ifdef __BIG_ENDIAN
12541         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
12542 #endif
12543         spin_lock_init(&tp->lock);
12544         spin_lock_init(&tp->indirect_lock);
12545         INIT_WORK(&tp->reset_task, tg3_reset_task);
12546
12547         tp->regs = ioremap_nocache(tg3reg_base, tg3reg_len);
12548         if (!tp->regs) {
12549                 printk(KERN_ERR PFX "Cannot map device registers, "
12550                        "aborting.\n");
12551                 err = -ENOMEM;
12552                 goto err_out_free_dev;
12553         }
12554
12555         tg3_init_link_config(tp);
12556
12557         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
12558         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
12559         tp->tx_pending = TG3_DEF_TX_RING_PENDING;
12560
12561         dev->open = tg3_open;
12562         dev->stop = tg3_close;
12563         dev->get_stats = tg3_get_stats;
12564         dev->set_multicast_list = tg3_set_rx_mode;
12565         dev->set_mac_address = tg3_set_mac_addr;
12566         dev->do_ioctl = tg3_ioctl;
12567         dev->tx_timeout = tg3_tx_timeout;
12568         netif_napi_add(dev, &tp->napi, tg3_poll, 64);
12569         dev->ethtool_ops = &tg3_ethtool_ops;
12570         dev->watchdog_timeo = TG3_TX_TIMEOUT;
12571         dev->change_mtu = tg3_change_mtu;
12572         dev->irq = pdev->irq;
12573 #ifdef CONFIG_NET_POLL_CONTROLLER
12574         dev->poll_controller = tg3_poll_controller;
12575 #endif
12576
12577         err = tg3_get_invariants(tp);
12578         if (err) {
12579                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
12580                        "aborting.\n");
12581                 goto err_out_iounmap;
12582         }
12583
12584         /* The EPB bridge inside 5714, 5715, and 5780 and any
12585          * device behind the EPB cannot support DMA addresses > 40-bit.
12586          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
12587          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
12588          * do DMA address check in tg3_start_xmit().
12589          */
12590         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
12591                 persist_dma_mask = dma_mask = DMA_32BIT_MASK;
12592         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
12593                 persist_dma_mask = dma_mask = DMA_40BIT_MASK;
12594 #ifdef CONFIG_HIGHMEM
12595                 dma_mask = DMA_64BIT_MASK;
12596 #endif
12597         } else
12598                 persist_dma_mask = dma_mask = DMA_64BIT_MASK;
12599
12600         /* Configure DMA attributes. */
12601         if (dma_mask > DMA_32BIT_MASK) {
12602                 err = pci_set_dma_mask(pdev, dma_mask);
12603                 if (!err) {
12604                         dev->features |= NETIF_F_HIGHDMA;
12605                         err = pci_set_consistent_dma_mask(pdev,
12606                                                           persist_dma_mask);
12607                         if (err < 0) {
12608                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
12609                                        "DMA for consistent allocations\n");
12610                                 goto err_out_iounmap;
12611                         }
12612                 }
12613         }
12614         if (err || dma_mask == DMA_32BIT_MASK) {
12615                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
12616                 if (err) {
12617                         printk(KERN_ERR PFX "No usable DMA configuration, "
12618                                "aborting.\n");
12619                         goto err_out_iounmap;
12620                 }
12621         }
12622
12623         tg3_init_bufmgr_config(tp);
12624
12625         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12626                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
12627         }
12628         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12629             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
12630             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
12631             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
12632             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
12633                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
12634         } else {
12635                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE | TG3_FLG2_TSO_BUG;
12636         }
12637
12638         /* TSO is on by default on chips that support hardware TSO.
12639          * Firmware TSO on older chips gives lower performance, so it
12640          * is off by default, but can be enabled using ethtool.
12641          */
12642         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12643                 dev->features |= NETIF_F_TSO;
12644                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
12645                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906))
12646                         dev->features |= NETIF_F_TSO6;
12647                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12648                         dev->features |= NETIF_F_TSO_ECN;
12649         }
12650
12651
12652         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
12653             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
12654             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
12655                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
12656                 tp->rx_pending = 63;
12657         }
12658
12659         err = tg3_get_device_address(tp);
12660         if (err) {
12661                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
12662                        "aborting.\n");
12663                 goto err_out_iounmap;
12664         }
12665
12666         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
12667                 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
12668                         printk(KERN_ERR PFX "Cannot find proper PCI device "
12669                                "base address for APE, aborting.\n");
12670                         err = -ENODEV;
12671                         goto err_out_iounmap;
12672                 }
12673
12674                 tg3reg_base = pci_resource_start(pdev, 2);
12675                 tg3reg_len = pci_resource_len(pdev, 2);
12676
12677                 tp->aperegs = ioremap_nocache(tg3reg_base, tg3reg_len);
12678                 if (!tp->aperegs) {
12679                         printk(KERN_ERR PFX "Cannot map APE registers, "
12680                                "aborting.\n");
12681                         err = -ENOMEM;
12682                         goto err_out_iounmap;
12683                 }
12684
12685                 tg3_ape_lock_init(tp);
12686         }
12687
12688         /*
12689          * Reset chip in case UNDI or EFI driver did not shutdown
12690          * DMA self test will enable WDMAC and we'll see (spurious)
12691          * pending DMA on the PCI bus at that point.
12692          */
12693         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
12694             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
12695                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
12696                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12697         }
12698
12699         err = tg3_test_dma(tp);
12700         if (err) {
12701                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
12702                 goto err_out_apeunmap;
12703         }
12704
12705         /* Tigon3 can do ipv4 only... and some chips have buggy
12706          * checksumming.
12707          */
12708         if ((tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) == 0) {
12709                 dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
12710                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12711                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12712                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12713                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12714                         dev->features |= NETIF_F_IPV6_CSUM;
12715
12716                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
12717         } else
12718                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
12719
12720         /* flow control autonegotiation is default behavior */
12721         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
12722         tp->link_config.flowctrl = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
12723
12724         tg3_init_coal(tp);
12725
12726         pci_set_drvdata(pdev, dev);
12727
12728         err = register_netdev(dev);
12729         if (err) {
12730                 printk(KERN_ERR PFX "Cannot register net device, "
12731                        "aborting.\n");
12732                 goto err_out_apeunmap;
12733         }
12734
12735         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x PHY(%s)] "
12736                "(%s) %s Ethernet %s\n",
12737                dev->name,
12738                tp->board_part_number,
12739                tp->pci_chip_rev_id,
12740                tg3_phy_string(tp),
12741                tg3_bus_string(tp, str),
12742                ((tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100Base-TX" :
12743                 ((tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) ? "1000Base-SX" :
12744                  "10/100/1000Base-T")),
12745                print_mac(mac, dev->dev_addr));
12746
12747         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] "
12748                "MIirq[%d] ASF[%d] WireSpeed[%d] TSOcap[%d]\n",
12749                dev->name,
12750                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
12751                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
12752                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
12753                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
12754                (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0,
12755                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
12756         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
12757                dev->name, tp->dma_rwctrl,
12758                (pdev->dma_mask == DMA_32BIT_MASK) ? 32 :
12759                 (((u64) pdev->dma_mask == DMA_40BIT_MASK) ? 40 : 64));
12760
12761         return 0;
12762
12763 err_out_apeunmap:
12764         if (tp->aperegs) {
12765                 iounmap(tp->aperegs);
12766                 tp->aperegs = NULL;
12767         }
12768
12769 err_out_iounmap:
12770         if (tp->regs) {
12771                 iounmap(tp->regs);
12772                 tp->regs = NULL;
12773         }
12774
12775 err_out_free_dev:
12776         free_netdev(dev);
12777
12778 err_out_free_res:
12779         pci_release_regions(pdev);
12780
12781 err_out_disable_pdev:
12782         pci_disable_device(pdev);
12783         pci_set_drvdata(pdev, NULL);
12784         return err;
12785 }
12786
12787 static void __devexit tg3_remove_one(struct pci_dev *pdev)
12788 {
12789         struct net_device *dev = pci_get_drvdata(pdev);
12790
12791         if (dev) {
12792                 struct tg3 *tp = netdev_priv(dev);
12793
12794                 flush_scheduled_work();
12795                 unregister_netdev(dev);
12796                 if (tp->aperegs) {
12797                         iounmap(tp->aperegs);
12798                         tp->aperegs = NULL;
12799                 }
12800                 if (tp->regs) {
12801                         iounmap(tp->regs);
12802                         tp->regs = NULL;
12803                 }
12804                 free_netdev(dev);
12805                 pci_release_regions(pdev);
12806                 pci_disable_device(pdev);
12807                 pci_set_drvdata(pdev, NULL);
12808         }
12809 }
12810
12811 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
12812 {
12813         struct net_device *dev = pci_get_drvdata(pdev);
12814         struct tg3 *tp = netdev_priv(dev);
12815         int err;
12816
12817         /* PCI register 4 needs to be saved whether netif_running() or not.
12818          * MSI address and data need to be saved if using MSI and
12819          * netif_running().
12820          */
12821         pci_save_state(pdev);
12822
12823         if (!netif_running(dev))
12824                 return 0;
12825
12826         flush_scheduled_work();
12827         tg3_netif_stop(tp);
12828
12829         del_timer_sync(&tp->timer);
12830
12831         tg3_full_lock(tp, 1);
12832         tg3_disable_ints(tp);
12833         tg3_full_unlock(tp);
12834
12835         netif_device_detach(dev);
12836
12837         tg3_full_lock(tp, 0);
12838         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12839         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
12840         tg3_full_unlock(tp);
12841
12842         err = tg3_set_power_state(tp, pci_choose_state(pdev, state));
12843         if (err) {
12844                 tg3_full_lock(tp, 0);
12845
12846                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
12847                 if (tg3_restart_hw(tp, 1))
12848                         goto out;
12849
12850                 tp->timer.expires = jiffies + tp->timer_offset;
12851                 add_timer(&tp->timer);
12852
12853                 netif_device_attach(dev);
12854                 tg3_netif_start(tp);
12855
12856 out:
12857                 tg3_full_unlock(tp);
12858         }
12859
12860         return err;
12861 }
12862
12863 static int tg3_resume(struct pci_dev *pdev)
12864 {
12865         struct net_device *dev = pci_get_drvdata(pdev);
12866         struct tg3 *tp = netdev_priv(dev);
12867         int err;
12868
12869         pci_restore_state(tp->pdev);
12870
12871         if (!netif_running(dev))
12872                 return 0;
12873
12874         err = tg3_set_power_state(tp, PCI_D0);
12875         if (err)
12876                 return err;
12877
12878         netif_device_attach(dev);
12879
12880         tg3_full_lock(tp, 0);
12881
12882         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
12883         err = tg3_restart_hw(tp, 1);
12884         if (err)
12885                 goto out;
12886
12887         tp->timer.expires = jiffies + tp->timer_offset;
12888         add_timer(&tp->timer);
12889
12890         tg3_netif_start(tp);
12891
12892 out:
12893         tg3_full_unlock(tp);
12894
12895         return err;
12896 }
12897
12898 static struct pci_driver tg3_driver = {
12899         .name           = DRV_MODULE_NAME,
12900         .id_table       = tg3_pci_tbl,
12901         .probe          = tg3_init_one,
12902         .remove         = __devexit_p(tg3_remove_one),
12903         .suspend        = tg3_suspend,
12904         .resume         = tg3_resume
12905 };
12906
12907 static int __init tg3_init(void)
12908 {
12909         return pci_register_driver(&tg3_driver);
12910 }
12911
12912 static void __exit tg3_cleanup(void)
12913 {
12914         pci_unregister_driver(&tg3_driver);
12915 }
12916
12917 module_init(tg3_init);
12918 module_exit(tg3_cleanup);