drivers/net/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2009 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/ipv6.h>
  35 #include <net/checksum.h>
  36 #include <net/ip6_checksum.h>
  37 #include <linux/net_tstamp.h>
  38 #include <linux/mii.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/pci.h>
  42 #include <linux/pci-aspm.h>
  43 #include <linux/delay.h>
  44 #include <linux/interrupt.h>
  45 #include <linux/if_ether.h>
  46 #include <linux/aer.h>
  47 #ifdef CONFIG_IGB_DCA
  48 #include <linux/dca.h>
  49 #endif
  50 #include "igb.h"
  51
  52 #define DRV_VERSION "2.1.0-k2"
  53 char igb_driver_name[] = "igb";
  54 char igb_driver_version[] = DRV_VERSION;
  55 static const char igb_driver_string[] =
  56                                 "Intel(R) Gigabit Ethernet Network Driver";
  57 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
  58
  59 static const struct e1000_info *igb_info_tbl[] = {
  60         [board_82575] = &e1000_82575_info,
  61 };
  62
  63 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  64         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  79         /* required last entry */
  80         {0, }
  81 };
  82
  83 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  84
  85 void igb_reset(struct igb_adapter *);
  86 static int igb_setup_all_tx_resources(struct igb_adapter *);
  87 static int igb_setup_all_rx_resources(struct igb_adapter *);
  88 static void igb_free_all_tx_resources(struct igb_adapter *);
  89 static void igb_free_all_rx_resources(struct igb_adapter *);
  90 static void igb_setup_mrqc(struct igb_adapter *);
  91 void igb_update_stats(struct igb_adapter *);
  92 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  93 static void __devexit igb_remove(struct pci_dev *pdev);
  94 static int igb_sw_init(struct igb_adapter *);
  95 static int igb_open(struct net_device *);
  96 static int igb_close(struct net_device *);
  97 static void igb_configure_tx(struct igb_adapter *);
  98 static void igb_configure_rx(struct igb_adapter *);
  99 static void igb_clean_all_tx_rings(struct igb_adapter *);
 100 static void igb_clean_all_rx_rings(struct igb_adapter *);
 101 static void igb_clean_tx_ring(struct igb_ring *);
 102 static void igb_clean_rx_ring(struct igb_ring *);
 103 static void igb_set_rx_mode(struct net_device *);
 104 static void igb_update_phy_info(unsigned long);
 105 static void igb_watchdog(unsigned long);
 106 static void igb_watchdog_task(struct work_struct *);
 107 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
 108 static struct net_device_stats *igb_get_stats(struct net_device *);
 109 static int igb_change_mtu(struct net_device *, int);
 110 static int igb_set_mac(struct net_device *, void *);
 111 static void igb_set_uta(struct igb_adapter *adapter);
 112 static irqreturn_t igb_intr(int irq, void *);
 113 static irqreturn_t igb_intr_msi(int irq, void *);
 114 static irqreturn_t igb_msix_other(int irq, void *);
 115 static irqreturn_t igb_msix_ring(int irq, void *);
 116 #ifdef CONFIG_IGB_DCA
 117 static void igb_update_dca(struct igb_q_vector *);
 118 static void igb_setup_dca(struct igb_adapter *);
 119 #endif /* CONFIG_IGB_DCA */
 120 static bool igb_clean_tx_irq(struct igb_q_vector *);
 121 static int igb_poll(struct napi_struct *, int);
 122 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
 123 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 124 static void igb_tx_timeout(struct net_device *);
 125 static void igb_reset_task(struct work_struct *);
 126 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
 127 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 128 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 129 static void igb_restore_vlan(struct igb_adapter *);
 130 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 131 static void igb_ping_all_vfs(struct igb_adapter *);
 132 static void igb_msg_task(struct igb_adapter *);
 133 static void igb_vmm_control(struct igb_adapter *);
 134 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 135 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 136
 137 #ifdef CONFIG_PM
 138 static int igb_suspend(struct pci_dev *, pm_message_t);
 139 static int igb_resume(struct pci_dev *);
 140 #endif
 141 static void igb_shutdown(struct pci_dev *);
 142 #ifdef CONFIG_IGB_DCA
 143 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 144 static struct notifier_block dca_notifier = {
 145         .notifier_call  = igb_notify_dca,
 146         .next           = NULL,
 147         .priority       = 0
 148 };
 149 #endif
 150 #ifdef CONFIG_NET_POLL_CONTROLLER
 151 /* for netdump / net console */
 152 static void igb_netpoll(struct net_device *);
 153 #endif
 154 #ifdef CONFIG_PCI_IOV
 155 static unsigned int max_vfs = 0;
 156 module_param(max_vfs, uint, 0);
 157 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 158                  "per physical function");
 159 #endif /* CONFIG_PCI_IOV */
 160
 161 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 162                      pci_channel_state_t);
 163 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 164 static void igb_io_resume(struct pci_dev *);
 165
 166 static struct pci_error_handlers igb_err_handler = {
 167         .error_detected = igb_io_error_detected,
 168         .slot_reset = igb_io_slot_reset,
 169         .resume = igb_io_resume,
 170 };
 171
 172
 173 static struct pci_driver igb_driver = {
 174         .name     = igb_driver_name,
 175         .id_table = igb_pci_tbl,
 176         .probe    = igb_probe,
 177         .remove   = __devexit_p(igb_remove),
 178 #ifdef CONFIG_PM
 179         /* Power Managment Hooks */
 180         .suspend  = igb_suspend,
 181         .resume   = igb_resume,
 182 #endif
 183         .shutdown = igb_shutdown,
 184         .err_handler = &igb_err_handler
 185 };
 186
 187 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 188 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 189 MODULE_LICENSE("GPL");
 190 MODULE_VERSION(DRV_VERSION);
 191
 192 /**
 193  * igb_read_clock - read raw cycle counter (to be used by time counter)
 194  */
 195 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 196 {
 197         struct igb_adapter *adapter =
 198                 container_of(tc, struct igb_adapter, cycles);
 199         struct e1000_hw *hw = &adapter->hw;
 200         u64 stamp = 0;
 201         int shift = 0;
 202
 203         /*
 204          * The timestamp latches on lowest register read. For the 82580
 205          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 206          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 207          */
 208         if (hw->mac.type == e1000_82580) {
 209                 stamp = rd32(E1000_SYSTIMR) >> 8;
 210                 shift = IGB_82580_TSYNC_SHIFT;
 211         }
 212
 213         stamp |= (u64)rd32(E1000_SYSTIML) << shift;
 214         stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
 215         return stamp;
 216 }
 217
 218 #ifdef DEBUG
 219 /**
 220  * igb_get_hw_dev_name - return device name string
 221  * used by hardware layer to print debugging information
 222  **/
 223 char *igb_get_hw_dev_name(struct e1000_hw *hw)
 224 {
 225         struct igb_adapter *adapter = hw->back;
 226         return adapter->netdev->name;
 227 }
 228
 229 /**
 230  * igb_get_time_str - format current NIC and system time as string
 231  */
 232 static char *igb_get_time_str(struct igb_adapter *adapter,
 233                               char buffer[160])
 234 {
 235         cycle_t hw = adapter->cycles.read(&adapter->cycles);
 236         struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
 237         struct timespec sys;
 238         struct timespec delta;
 239         getnstimeofday(&sys);
 240
 241         delta = timespec_sub(nic, sys);
 242
 243         sprintf(buffer,
 244                 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
 245                 hw,
 246                 (long)nic.tv_sec, nic.tv_nsec,
 247                 (long)sys.tv_sec, sys.tv_nsec,
 248                 (long)delta.tv_sec, delta.tv_nsec);
 249
 250         return buffer;
 251 }
 252 #endif
 253
 254 /**
 255  * igb_init_module - Driver Registration Routine
 256  *
 257  * igb_init_module is the first routine called when the driver is
 258  * loaded. All it does is register with the PCI subsystem.
 259  **/
 260 static int __init igb_init_module(void)
 261 {
 262         int ret;
 263         printk(KERN_INFO "%s - version %s\n",
 264                igb_driver_string, igb_driver_version);
 265
 266         printk(KERN_INFO "%s\n", igb_copyright);
 267
 268 #ifdef CONFIG_IGB_DCA
 269         dca_register_notify(&dca_notifier);
 270 #endif
 271         ret = pci_register_driver(&igb_driver);
 272         return ret;
 273 }
 274
 275 module_init(igb_init_module);
 276
 277 /**
 278  * igb_exit_module - Driver Exit Cleanup Routine
 279  *
 280  * igb_exit_module is called just before the driver is removed
 281  * from memory.
 282  **/
 283 static void __exit igb_exit_module(void)
 284 {
 285 #ifdef CONFIG_IGB_DCA
 286         dca_unregister_notify(&dca_notifier);
 287 #endif
 288         pci_unregister_driver(&igb_driver);
 289 }
 290
 291 module_exit(igb_exit_module);
 292
 293 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 294 /**
 295  * igb_cache_ring_register - Descriptor ring to register mapping
 296  * @adapter: board private structure to initialize
 297  *
 298  * Once we know the feature-set enabled for the device, we'll cache
 299  * the register offset the descriptor ring is assigned to.
 300  **/
 301 static void igb_cache_ring_register(struct igb_adapter *adapter)
 302 {
 303         int i = 0, j = 0;
 304         u32 rbase_offset = adapter->vfs_allocated_count;
 305
 306         switch (adapter->hw.mac.type) {
 307         case e1000_82576:
 308                 /* The queues are allocated for virtualization such that VF 0
 309                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 310                  * In order to avoid collision we start at the first free queue
 311                  * and continue consuming queues in the same sequence
 312                  */
 313                 if (adapter->vfs_allocated_count) {
 314                         for (; i < adapter->rss_queues; i++)
 315                                 adapter->rx_ring[i].reg_idx = rbase_offset +
 316                                                               Q_IDX_82576(i);
 317                         for (; j < adapter->rss_queues; j++)
 318                                 adapter->tx_ring[j].reg_idx = rbase_offset +
 319                                                               Q_IDX_82576(j);
 320                 }
 321         case e1000_82575:
 322         case e1000_82580:
 323         default:
 324                 for (; i < adapter->num_rx_queues; i++)
 325                         adapter->rx_ring[i].reg_idx = rbase_offset + i;
 326                 for (; j < adapter->num_tx_queues; j++)
 327                         adapter->tx_ring[j].reg_idx = rbase_offset + j;
 328                 break;
 329         }
 330 }
 331
 332 static void igb_free_queues(struct igb_adapter *adapter)
 333 {
 334         kfree(adapter->tx_ring);
 335         kfree(adapter->rx_ring);
 336
 337         adapter->tx_ring = NULL;
 338         adapter->rx_ring = NULL;
 339
 340         adapter->num_rx_queues = 0;
 341         adapter->num_tx_queues = 0;
 342 }
 343
 344 /**
 345  * igb_alloc_queues - Allocate memory for all rings
 346  * @adapter: board private structure to initialize
 347  *
 348  * We allocate one ring per queue at run-time since we don't know the
 349  * number of queues at compile-time.
 350  **/
 351 static int igb_alloc_queues(struct igb_adapter *adapter)
 352 {
 353         int i;
 354
 355         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
 356                                    sizeof(struct igb_ring), GFP_KERNEL);
 357         if (!adapter->tx_ring)
 358                 goto err;
 359
 360         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
 361                                    sizeof(struct igb_ring), GFP_KERNEL);
 362         if (!adapter->rx_ring)
 363                 goto err;
 364
 365         for (i = 0; i < adapter->num_tx_queues; i++) {
 366                 struct igb_ring *ring = &(adapter->tx_ring[i]);
 367                 ring->count = adapter->tx_ring_count;
 368                 ring->queue_index = i;
 369                 ring->pdev = adapter->pdev;
 370                 ring->netdev = adapter->netdev;
 371                 /* For 82575, context index must be unique per ring. */
 372                 if (adapter->hw.mac.type == e1000_82575)
 373                         ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
 374         }
 375
 376         for (i = 0; i < adapter->num_rx_queues; i++) {
 377                 struct igb_ring *ring = &(adapter->rx_ring[i]);
 378                 ring->count = adapter->rx_ring_count;
 379                 ring->queue_index = i;
 380                 ring->pdev = adapter->pdev;
 381                 ring->netdev = adapter->netdev;
 382                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 383                 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
 384                 /* set flag indicating ring supports SCTP checksum offload */
 385                 if (adapter->hw.mac.type >= e1000_82576)
 386                         ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
 387         }
 388
 389         igb_cache_ring_register(adapter);
 390
 391         return 0;
 392
 393 err:
 394         igb_free_queues(adapter);
 395
 396         return -ENOMEM;
 397 }
 398
 399 #define IGB_N0_QUEUE -1
 400 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 401 {
 402         u32 msixbm = 0;
 403         struct igb_adapter *adapter = q_vector->adapter;
 404         struct e1000_hw *hw = &adapter->hw;
 405         u32 ivar, index;
 406         int rx_queue = IGB_N0_QUEUE;
 407         int tx_queue = IGB_N0_QUEUE;
 408
 409         if (q_vector->rx_ring)
 410                 rx_queue = q_vector->rx_ring->reg_idx;
 411         if (q_vector->tx_ring)
 412                 tx_queue = q_vector->tx_ring->reg_idx;
 413
 414         switch (hw->mac.type) {
 415         case e1000_82575:
 416                 /* The 82575 assigns vectors using a bitmask, which matches the
 417                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 418                    or more queues to a vector, we write the appropriate bits
 419                    into the MSIXBM register for that vector. */
 420                 if (rx_queue > IGB_N0_QUEUE)
 421                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 422                 if (tx_queue > IGB_N0_QUEUE)
 423                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 424                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 425                 q_vector->eims_value = msixbm;
 426                 break;
 427         case e1000_82576:
 428                 /* 82576 uses a table-based method for assigning vectors.
 429                    Each queue has a single entry in the table to which we write
 430                    a vector number along with a "valid" bit.  Sadly, the layout
 431                    of the table is somewhat counterintuitive. */
 432                 if (rx_queue > IGB_N0_QUEUE) {
 433                         index = (rx_queue & 0x7);
 434                         ivar = array_rd32(E1000_IVAR0, index);
 435                         if (rx_queue < 8) {
 436                                 /* vector goes into low byte of register */
 437                                 ivar = ivar & 0xFFFFFF00;
 438                                 ivar |= msix_vector | E1000_IVAR_VALID;
 439                         } else {
 440                                 /* vector goes into third byte of register */
 441                                 ivar = ivar & 0xFF00FFFF;
 442                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 443                         }
 444                         array_wr32(E1000_IVAR0, index, ivar);
 445                 }
 446                 if (tx_queue > IGB_N0_QUEUE) {
 447                         index = (tx_queue & 0x7);
 448                         ivar = array_rd32(E1000_IVAR0, index);
 449                         if (tx_queue < 8) {
 450                                 /* vector goes into second byte of register */
 451                                 ivar = ivar & 0xFFFF00FF;
 452                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 453                         } else {
 454                                 /* vector goes into high byte of register */
 455                                 ivar = ivar & 0x00FFFFFF;
 456                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 457                         }
 458                         array_wr32(E1000_IVAR0, index, ivar);
 459                 }
 460                 q_vector->eims_value = 1 << msix_vector;
 461                 break;
 462         case e1000_82580:
 463                 /* 82580 uses the same table-based approach as 82576 but has fewer
 464                    entries as a result we carry over for queues greater than 4. */
 465                 if (rx_queue > IGB_N0_QUEUE) {
 466                         index = (rx_queue >> 1);
 467                         ivar = array_rd32(E1000_IVAR0, index);
 468                         if (rx_queue & 0x1) {
 469                                 /* vector goes into third byte of register */
 470                                 ivar = ivar & 0xFF00FFFF;
 471                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 472                         } else {
 473                                 /* vector goes into low byte of register */
 474                                 ivar = ivar & 0xFFFFFF00;
 475                                 ivar |= msix_vector | E1000_IVAR_VALID;
 476                         }
 477                         array_wr32(E1000_IVAR0, index, ivar);
 478                 }
 479                 if (tx_queue > IGB_N0_QUEUE) {
 480                         index = (tx_queue >> 1);
 481                         ivar = array_rd32(E1000_IVAR0, index);
 482                         if (tx_queue & 0x1) {
 483                                 /* vector goes into high byte of register */
 484                                 ivar = ivar & 0x00FFFFFF;
 485                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 486                         } else {
 487                                 /* vector goes into second byte of register */
 488                                 ivar = ivar & 0xFFFF00FF;
 489                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 490                         }
 491                         array_wr32(E1000_IVAR0, index, ivar);
 492                 }
 493                 q_vector->eims_value = 1 << msix_vector;
 494                 break;
 495         default:
 496                 BUG();
 497                 break;
 498         }
 499 }
 500
 501 /**
 502  * igb_configure_msix - Configure MSI-X hardware
 503  *
 504  * igb_configure_msix sets up the hardware to properly
 505  * generate MSI-X interrupts.
 506  **/
 507 static void igb_configure_msix(struct igb_adapter *adapter)
 508 {
 509         u32 tmp;
 510         int i, vector = 0;
 511         struct e1000_hw *hw = &adapter->hw;
 512
 513         adapter->eims_enable_mask = 0;
 514
 515         /* set vector for other causes, i.e. link changes */
 516         switch (hw->mac.type) {
 517         case e1000_82575:
 518                 tmp = rd32(E1000_CTRL_EXT);
 519                 /* enable MSI-X PBA support*/
 520                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 521
 522                 /* Auto-Mask interrupts upon ICR read. */
 523                 tmp |= E1000_CTRL_EXT_EIAME;
 524                 tmp |= E1000_CTRL_EXT_IRCA;
 525
 526                 wr32(E1000_CTRL_EXT, tmp);
 527
 528                 /* enable msix_other interrupt */
 529                 array_wr32(E1000_MSIXBM(0), vector++,
 530                                       E1000_EIMS_OTHER);
 531                 adapter->eims_other = E1000_EIMS_OTHER;
 532
 533                 break;
 534
 535         case e1000_82576:
 536         case e1000_82580:
 537                 /* Turn on MSI-X capability first, or our settings
 538                  * won't stick.  And it will take days to debug. */
 539                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 540                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 541                                 E1000_GPIE_NSICR);
 542
 543                 /* enable msix_other interrupt */
 544                 adapter->eims_other = 1 << vector;
 545                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 546
 547                 wr32(E1000_IVAR_MISC, tmp);
 548                 break;
 549         default:
 550                 /* do nothing, since nothing else supports MSI-X */
 551                 break;
 552         } /* switch (hw->mac.type) */
 553
 554         adapter->eims_enable_mask |= adapter->eims_other;
 555
 556         for (i = 0; i < adapter->num_q_vectors; i++) {
 557                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 558                 igb_assign_vector(q_vector, vector++);
 559                 adapter->eims_enable_mask |= q_vector->eims_value;
 560         }
 561
 562         wrfl();
 563 }
 564
 565 /**
 566  * igb_request_msix - Initialize MSI-X interrupts
 567  *
 568  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 569  * kernel.
 570  **/
 571 static int igb_request_msix(struct igb_adapter *adapter)
 572 {
 573         struct net_device *netdev = adapter->netdev;
 574         struct e1000_hw *hw = &adapter->hw;
 575         int i, err = 0, vector = 0;
 576
 577         err = request_irq(adapter->msix_entries[vector].vector,
 578                           igb_msix_other, 0, netdev->name, adapter);
 579         if (err)
 580                 goto out;
 581         vector++;
 582
 583         for (i = 0; i < adapter->num_q_vectors; i++) {
 584                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 585
 586                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 587
 588                 if (q_vector->rx_ring && q_vector->tx_ring)
 589                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 590                                 q_vector->rx_ring->queue_index);
 591                 else if (q_vector->tx_ring)
 592                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 593                                 q_vector->tx_ring->queue_index);
 594                 else if (q_vector->rx_ring)
 595                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 596                                 q_vector->rx_ring->queue_index);
 597                 else
 598                         sprintf(q_vector->name, "%s-unused", netdev->name);
 599
 600                 err = request_irq(adapter->msix_entries[vector].vector,
 601                                   igb_msix_ring, 0, q_vector->name,
 602                                   q_vector);
 603                 if (err)
 604                         goto out;
 605                 vector++;
 606         }
 607
 608         igb_configure_msix(adapter);
 609         return 0;
 610 out:
 611         return err;
 612 }
 613
 614 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 615 {
 616         if (adapter->msix_entries) {
 617                 pci_disable_msix(adapter->pdev);
 618                 kfree(adapter->msix_entries);
 619                 adapter->msix_entries = NULL;
 620         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 621                 pci_disable_msi(adapter->pdev);
 622         }
 623 }
 624
 625 /**
 626  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 627  * @adapter: board private structure to initialize
 628  *
 629  * This function frees the memory allocated to the q_vectors.  In addition if
 630  * NAPI is enabled it will delete any references to the NAPI struct prior
 631  * to freeing the q_vector.
 632  **/
 633 static void igb_free_q_vectors(struct igb_adapter *adapter)
 634 {
 635         int v_idx;
 636
 637         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 638                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 639                 adapter->q_vector[v_idx] = NULL;
 640                 netif_napi_del(&q_vector->napi);
 641                 kfree(q_vector);
 642         }
 643         adapter->num_q_vectors = 0;
 644 }
 645
 646 /**
 647  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 648  *
 649  * This function resets the device so that it has 0 rx queues, tx queues, and
 650  * MSI-X interrupts allocated.
 651  */
 652 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 653 {
 654         igb_free_queues(adapter);
 655         igb_free_q_vectors(adapter);
 656         igb_reset_interrupt_capability(adapter);
 657 }
 658
 659 /**
 660  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 661  *
 662  * Attempt to configure interrupts using the best available
 663  * capabilities of the hardware and kernel.
 664  **/
 665 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
 666 {
 667         int err;
 668         int numvecs, i;
 669
 670         /* Number of supported queues. */
 671         adapter->num_rx_queues = adapter->rss_queues;
 672         adapter->num_tx_queues = adapter->rss_queues;
 673
 674         /* start with one vector for every rx queue */
 675         numvecs = adapter->num_rx_queues;
 676
 677         /* if tx handler is seperate add 1 for every tx queue */
 678         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
 679                 numvecs += adapter->num_tx_queues;
 680
 681         /* store the number of vectors reserved for queues */
 682         adapter->num_q_vectors = numvecs;
 683
 684         /* add 1 vector for link status interrupts */
 685         numvecs++;
 686         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
 687                                         GFP_KERNEL);
 688         if (!adapter->msix_entries)
 689                 goto msi_only;
 690
 691         for (i = 0; i < numvecs; i++)
 692                 adapter->msix_entries[i].entry = i;
 693
 694         err = pci_enable_msix(adapter->pdev,
 695                               adapter->msix_entries,
 696                               numvecs);
 697         if (err == 0)
 698                 goto out;
 699
 700         igb_reset_interrupt_capability(adapter);
 701
 702         /* If we can't do MSI-X, try MSI */
 703 msi_only:
 704 #ifdef CONFIG_PCI_IOV
 705         /* disable SR-IOV for non MSI-X configurations */
 706         if (adapter->vf_data) {
 707                 struct e1000_hw *hw = &adapter->hw;
 708                 /* disable iov and allow time for transactions to clear */
 709                 pci_disable_sriov(adapter->pdev);
 710                 msleep(500);
 711
 712                 kfree(adapter->vf_data);
 713                 adapter->vf_data = NULL;
 714                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
 715                 msleep(100);
 716                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
 717         }
 718 #endif
 719         adapter->vfs_allocated_count = 0;
 720         adapter->rss_queues = 1;
 721         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
 722         adapter->num_rx_queues = 1;
 723         adapter->num_tx_queues = 1;
 724         adapter->num_q_vectors = 1;
 725         if (!pci_enable_msi(adapter->pdev))
 726                 adapter->flags |= IGB_FLAG_HAS_MSI;
 727 out:
 728         /* Notify the stack of the (possibly) reduced Tx Queue count. */
 729         adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
 730         return;
 731 }
 732
 733 /**
 734  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
 735  * @adapter: board private structure to initialize
 736  *
 737  * We allocate one q_vector per queue interrupt.  If allocation fails we
 738  * return -ENOMEM.
 739  **/
 740 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
 741 {
 742         struct igb_q_vector *q_vector;
 743         struct e1000_hw *hw = &adapter->hw;
 744         int v_idx;
 745
 746         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 747                 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
 748                 if (!q_vector)
 749                         goto err_out;
 750                 q_vector->adapter = adapter;
 751                 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
 752                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
 753                 q_vector->itr_val = IGB_START_ITR;
 754                 q_vector->set_itr = 1;
 755                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
 756                 adapter->q_vector[v_idx] = q_vector;
 757         }
 758         return 0;
 759
 760 err_out:
 761         while (v_idx) {
 762                 v_idx--;
 763                 q_vector = adapter->q_vector[v_idx];
 764                 netif_napi_del(&q_vector->napi);
 765                 kfree(q_vector);
 766                 adapter->q_vector[v_idx] = NULL;
 767         }
 768         return -ENOMEM;
 769 }
 770
 771 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
 772                                       int ring_idx, int v_idx)
 773 {
 774         struct igb_q_vector *q_vector;
 775
 776         q_vector = adapter->q_vector[v_idx];
 777         q_vector->rx_ring = &adapter->rx_ring[ring_idx];
 778         q_vector->rx_ring->q_vector = q_vector;
 779         q_vector->itr_val = adapter->rx_itr_setting;
 780         if (q_vector->itr_val && q_vector->itr_val <= 3)
 781                 q_vector->itr_val = IGB_START_ITR;
 782 }
 783
 784 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
 785                                       int ring_idx, int v_idx)
 786 {
 787         struct igb_q_vector *q_vector;
 788
 789         q_vector = adapter->q_vector[v_idx];
 790         q_vector->tx_ring = &adapter->tx_ring[ring_idx];
 791         q_vector->tx_ring->q_vector = q_vector;
 792         q_vector->itr_val = adapter->tx_itr_setting;
 793         if (q_vector->itr_val && q_vector->itr_val <= 3)
 794                 q_vector->itr_val = IGB_START_ITR;
 795 }
 796
 797 /**
 798  * igb_map_ring_to_vector - maps allocated queues to vectors
 799  *
 800  * This function maps the recently allocated queues to vectors.
 801  **/
 802 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
 803 {
 804         int i;
 805         int v_idx = 0;
 806
 807         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
 808             (adapter->num_q_vectors < adapter->num_tx_queues))
 809                 return -ENOMEM;
 810
 811         if (adapter->num_q_vectors >=
 812             (adapter->num_rx_queues + adapter->num_tx_queues)) {
 813                 for (i = 0; i < adapter->num_rx_queues; i++)
 814                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 815                 for (i = 0; i < adapter->num_tx_queues; i++)
 816                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 817         } else {
 818                 for (i = 0; i < adapter->num_rx_queues; i++) {
 819                         if (i < adapter->num_tx_queues)
 820                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
 821                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 822                 }
 823                 for (; i < adapter->num_tx_queues; i++)
 824                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 825         }
 826         return 0;
 827 }
 828
 829 /**
 830  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
 831  *
 832  * This function initializes the interrupts and allocates all of the queues.
 833  **/
 834 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
 835 {
 836         struct pci_dev *pdev = adapter->pdev;
 837         int err;
 838
 839         igb_set_interrupt_capability(adapter);
 840
 841         err = igb_alloc_q_vectors(adapter);
 842         if (err) {
 843                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
 844                 goto err_alloc_q_vectors;
 845         }
 846
 847         err = igb_alloc_queues(adapter);
 848         if (err) {
 849                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
 850                 goto err_alloc_queues;
 851         }
 852
 853         err = igb_map_ring_to_vector(adapter);
 854         if (err) {
 855                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
 856                 goto err_map_queues;
 857         }
 858
 859
 860         return 0;
 861 err_map_queues:
 862         igb_free_queues(adapter);
 863 err_alloc_queues:
 864         igb_free_q_vectors(adapter);
 865 err_alloc_q_vectors:
 866         igb_reset_interrupt_capability(adapter);
 867         return err;
 868 }
 869
 870 /**
 871  * igb_request_irq - initialize interrupts
 872  *
 873  * Attempts to configure interrupts using the best available
 874  * capabilities of the hardware and kernel.
 875  **/
 876 static int igb_request_irq(struct igb_adapter *adapter)
 877 {
 878         struct net_device *netdev = adapter->netdev;
 879         struct pci_dev *pdev = adapter->pdev;
 880         struct e1000_hw *hw = &adapter->hw;
 881         int err = 0;
 882
 883         if (adapter->msix_entries) {
 884                 err = igb_request_msix(adapter);
 885                 if (!err)
 886                         goto request_done;
 887                 /* fall back to MSI */
 888                 igb_clear_interrupt_scheme(adapter);
 889                 if (!pci_enable_msi(adapter->pdev))
 890                         adapter->flags |= IGB_FLAG_HAS_MSI;
 891                 igb_free_all_tx_resources(adapter);
 892                 igb_free_all_rx_resources(adapter);
 893                 adapter->num_tx_queues = 1;
 894                 adapter->num_rx_queues = 1;
 895                 adapter->num_q_vectors = 1;
 896                 err = igb_alloc_q_vectors(adapter);
 897                 if (err) {
 898                         dev_err(&pdev->dev,
 899                                 "Unable to allocate memory for vectors\n");
 900                         goto request_done;
 901                 }
 902                 err = igb_alloc_queues(adapter);
 903                 if (err) {
 904                         dev_err(&pdev->dev,
 905                                 "Unable to allocate memory for queues\n");
 906                         igb_free_q_vectors(adapter);
 907                         goto request_done;
 908                 }
 909                 igb_setup_all_tx_resources(adapter);
 910                 igb_setup_all_rx_resources(adapter);
 911         } else {
 912                 switch (hw->mac.type) {
 913                 case e1000_82575:
 914                         wr32(E1000_MSIXBM(0),
 915                              (E1000_EICR_RX_QUEUE0 |
 916                               E1000_EICR_TX_QUEUE0 |
 917                               E1000_EIMS_OTHER));
 918                         break;
 919                 case e1000_82580:
 920                 case e1000_82576:
 921                         wr32(E1000_IVAR0, E1000_IVAR_VALID);
 922                         break;
 923                 default:
 924                         break;
 925                 }
 926         }
 927
 928         if (adapter->flags & IGB_FLAG_HAS_MSI) {
 929                 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
 930                                   netdev->name, adapter);
 931                 if (!err)
 932                         goto request_done;
 933
 934                 /* fall back to legacy interrupts */
 935                 igb_reset_interrupt_capability(adapter);
 936                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
 937         }
 938
 939         err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
 940                           netdev->name, adapter);
 941
 942         if (err)
 943                 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
 944                         err);
 945
 946 request_done:
 947         return err;
 948 }
 949
 950 static void igb_free_irq(struct igb_adapter *adapter)
 951 {
 952         if (adapter->msix_entries) {
 953                 int vector = 0, i;
 954
 955                 free_irq(adapter->msix_entries[vector++].vector, adapter);
 956
 957                 for (i = 0; i < adapter->num_q_vectors; i++) {
 958                         struct igb_q_vector *q_vector = adapter->q_vector[i];
 959                         free_irq(adapter->msix_entries[vector++].vector,
 960                                  q_vector);
 961                 }
 962         } else {
 963                 free_irq(adapter->pdev->irq, adapter);
 964         }
 965 }
 966
 967 /**
 968  * igb_irq_disable - Mask off interrupt generation on the NIC
 969  * @adapter: board private structure
 970  **/
 971 static void igb_irq_disable(struct igb_adapter *adapter)
 972 {
 973         struct e1000_hw *hw = &adapter->hw;
 974
 975         /*
 976          * we need to be careful when disabling interrupts.  The VFs are also
 977          * mapped into these registers and so clearing the bits can cause
 978          * issues on the VF drivers so we only need to clear what we set
 979          */
 980         if (adapter->msix_entries) {
 981                 u32 regval = rd32(E1000_EIAM);
 982                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
 983                 wr32(E1000_EIMC, adapter->eims_enable_mask);
 984                 regval = rd32(E1000_EIAC);
 985                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
 986         }
 987
 988         wr32(E1000_IAM, 0);
 989         wr32(E1000_IMC, ~0);
 990         wrfl();
 991         synchronize_irq(adapter->pdev->irq);
 992 }
 993
 994 /**
 995  * igb_irq_enable - Enable default interrupt generation settings
 996  * @adapter: board private structure
 997  **/
 998 static void igb_irq_enable(struct igb_adapter *adapter)
 999 {
1000         struct e1000_hw *hw = &adapter->hw;
1001
1002         if (adapter->msix_entries) {
1003                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1004                 u32 regval = rd32(E1000_EIAC);
1005                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1006                 regval = rd32(E1000_EIAM);
1007                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1008                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1009                 if (adapter->vfs_allocated_count) {
1010                         wr32(E1000_MBVFIMR, 0xFF);
1011                         ims |= E1000_IMS_VMMB;
1012                 }
1013                 if (adapter->hw.mac.type == e1000_82580)
1014                         ims |= E1000_IMS_DRSTA;
1015
1016                 wr32(E1000_IMS, ims);
1017         } else {
1018                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1019                                 E1000_IMS_DRSTA);
1020                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1021                                 E1000_IMS_DRSTA);
1022         }
1023 }
1024
1025 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1026 {
1027         struct e1000_hw *hw = &adapter->hw;
1028         u16 vid = adapter->hw.mng_cookie.vlan_id;
1029         u16 old_vid = adapter->mng_vlan_id;
1030
1031         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1032                 /* add VID to filter table */
1033                 igb_vfta_set(hw, vid, true);
1034                 adapter->mng_vlan_id = vid;
1035         } else {
1036                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1037         }
1038
1039         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1040             (vid != old_vid) &&
1041             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1042                 /* remove VID from filter table */
1043                 igb_vfta_set(hw, old_vid, false);
1044         }
1045 }
1046
1047 /**
1048  * igb_release_hw_control - release control of the h/w to f/w
1049  * @adapter: address of board private structure
1050  *
1051  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1052  * For ASF and Pass Through versions of f/w this means that the
1053  * driver is no longer loaded.
1054  *
1055  **/
1056 static void igb_release_hw_control(struct igb_adapter *adapter)
1057 {
1058         struct e1000_hw *hw = &adapter->hw;
1059         u32 ctrl_ext;
1060
1061         /* Let firmware take over control of h/w */
1062         ctrl_ext = rd32(E1000_CTRL_EXT);
1063         wr32(E1000_CTRL_EXT,
1064                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1065 }
1066
1067 /**
1068  * igb_get_hw_control - get control of the h/w from f/w
1069  * @adapter: address of board private structure
1070  *
1071  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1072  * For ASF and Pass Through versions of f/w this means that
1073  * the driver is loaded.
1074  *
1075  **/
1076 static void igb_get_hw_control(struct igb_adapter *adapter)
1077 {
1078         struct e1000_hw *hw = &adapter->hw;
1079         u32 ctrl_ext;
1080
1081         /* Let firmware know the driver has taken over */
1082         ctrl_ext = rd32(E1000_CTRL_EXT);
1083         wr32(E1000_CTRL_EXT,
1084                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1085 }
1086
1087 /**
1088  * igb_configure - configure the hardware for RX and TX
1089  * @adapter: private board structure
1090  **/
1091 static void igb_configure(struct igb_adapter *adapter)
1092 {
1093         struct net_device *netdev = adapter->netdev;
1094         int i;
1095
1096         igb_get_hw_control(adapter);
1097         igb_set_rx_mode(netdev);
1098
1099         igb_restore_vlan(adapter);
1100
1101         igb_setup_tctl(adapter);
1102         igb_setup_mrqc(adapter);
1103         igb_setup_rctl(adapter);
1104
1105         igb_configure_tx(adapter);
1106         igb_configure_rx(adapter);
1107
1108         igb_rx_fifo_flush_82575(&adapter->hw);
1109
1110         /* call igb_desc_unused which always leaves
1111          * at least 1 descriptor unused to make sure
1112          * next_to_use != next_to_clean */
1113         for (i = 0; i < adapter->num_rx_queues; i++) {
1114                 struct igb_ring *ring = &adapter->rx_ring[i];
1115                 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1116         }
1117
1118
1119         adapter->tx_queue_len = netdev->tx_queue_len;
1120 }
1121
1122
1123 /**
1124  * igb_up - Open the interface and prepare it to handle traffic
1125  * @adapter: board private structure
1126  **/
1127 int igb_up(struct igb_adapter *adapter)
1128 {
1129         struct e1000_hw *hw = &adapter->hw;
1130         int i;
1131
1132         /* hardware has been reset, we need to reload some things */
1133         igb_configure(adapter);
1134
1135         clear_bit(__IGB_DOWN, &adapter->state);
1136
1137         for (i = 0; i < adapter->num_q_vectors; i++) {
1138                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1139                 napi_enable(&q_vector->napi);
1140         }
1141         if (adapter->msix_entries)
1142                 igb_configure_msix(adapter);
1143
1144         /* Clear any pending interrupts. */
1145         rd32(E1000_ICR);
1146         igb_irq_enable(adapter);
1147
1148         /* notify VFs that reset has been completed */
1149         if (adapter->vfs_allocated_count) {
1150                 u32 reg_data = rd32(E1000_CTRL_EXT);
1151                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1152                 wr32(E1000_CTRL_EXT, reg_data);
1153         }
1154
1155         netif_tx_start_all_queues(adapter->netdev);
1156
1157         /* start the watchdog. */
1158         hw->mac.get_link_status = 1;
1159         schedule_work(&adapter->watchdog_task);
1160
1161         return 0;
1162 }
1163
1164 void igb_down(struct igb_adapter *adapter)
1165 {
1166         struct net_device *netdev = adapter->netdev;
1167         struct e1000_hw *hw = &adapter->hw;
1168         u32 tctl, rctl;
1169         int i;
1170
1171         /* signal that we're down so the interrupt handler does not
1172          * reschedule our watchdog timer */
1173         set_bit(__IGB_DOWN, &adapter->state);
1174
1175         /* disable receives in the hardware */
1176         rctl = rd32(E1000_RCTL);
1177         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1178         /* flush and sleep below */
1179
1180         netif_tx_stop_all_queues(netdev);
1181
1182         /* disable transmits in the hardware */
1183         tctl = rd32(E1000_TCTL);
1184         tctl &= ~E1000_TCTL_EN;
1185         wr32(E1000_TCTL, tctl);
1186         /* flush both disables and wait for them to finish */
1187         wrfl();
1188         msleep(10);
1189
1190         for (i = 0; i < adapter->num_q_vectors; i++) {
1191                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1192                 napi_disable(&q_vector->napi);
1193         }
1194
1195         igb_irq_disable(adapter);
1196
1197         del_timer_sync(&adapter->watchdog_timer);
1198         del_timer_sync(&adapter->phy_info_timer);
1199
1200         netdev->tx_queue_len = adapter->tx_queue_len;
1201         netif_carrier_off(netdev);
1202
1203         /* record the stats before reset*/
1204         igb_update_stats(adapter);
1205
1206         adapter->link_speed = 0;
1207         adapter->link_duplex = 0;
1208
1209         if (!pci_channel_offline(adapter->pdev))
1210                 igb_reset(adapter);
1211         igb_clean_all_tx_rings(adapter);
1212         igb_clean_all_rx_rings(adapter);
1213 #ifdef CONFIG_IGB_DCA
1214
1215         /* since we reset the hardware DCA settings were cleared */
1216         igb_setup_dca(adapter);
1217 #endif
1218 }
1219
1220 void igb_reinit_locked(struct igb_adapter *adapter)
1221 {
1222         WARN_ON(in_interrupt());
1223         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1224                 msleep(1);
1225         igb_down(adapter);
1226         igb_up(adapter);
1227         clear_bit(__IGB_RESETTING, &adapter->state);
1228 }
1229
1230 void igb_reset(struct igb_adapter *adapter)
1231 {
1232         struct pci_dev *pdev = adapter->pdev;
1233         struct e1000_hw *hw = &adapter->hw;
1234         struct e1000_mac_info *mac = &hw->mac;
1235         struct e1000_fc_info *fc = &hw->fc;
1236         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1237         u16 hwm;
1238
1239         /* Repartition Pba for greater than 9k mtu
1240          * To take effect CTRL.RST is required.
1241          */
1242         switch (mac->type) {
1243         case e1000_82580:
1244                 pba = rd32(E1000_RXPBS);
1245                 pba = igb_rxpbs_adjust_82580(pba);
1246                 break;
1247         case e1000_82576:
1248                 pba = rd32(E1000_RXPBS);
1249                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1250                 break;
1251         case e1000_82575:
1252         default:
1253                 pba = E1000_PBA_34K;
1254                 break;
1255         }
1256
1257         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1258             (mac->type < e1000_82576)) {
1259                 /* adjust PBA for jumbo frames */
1260                 wr32(E1000_PBA, pba);
1261
1262                 /* To maintain wire speed transmits, the Tx FIFO should be
1263                  * large enough to accommodate two full transmit packets,
1264                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1265                  * the Rx FIFO should be large enough to accommodate at least
1266                  * one full receive packet and is similarly rounded up and
1267                  * expressed in KB. */
1268                 pba = rd32(E1000_PBA);
1269                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1270                 tx_space = pba >> 16;
1271                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1272                 pba &= 0xffff;
1273                 /* the tx fifo also stores 16 bytes of information about the tx
1274                  * but don't include ethernet FCS because hardware appends it */
1275                 min_tx_space = (adapter->max_frame_size +
1276                                 sizeof(union e1000_adv_tx_desc) -
1277                                 ETH_FCS_LEN) * 2;
1278                 min_tx_space = ALIGN(min_tx_space, 1024);
1279                 min_tx_space >>= 10;
1280                 /* software strips receive CRC, so leave room for it */
1281                 min_rx_space = adapter->max_frame_size;
1282                 min_rx_space = ALIGN(min_rx_space, 1024);
1283                 min_rx_space >>= 10;
1284
1285                 /* If current Tx allocation is less than the min Tx FIFO size,
1286                  * and the min Tx FIFO size is less than the current Rx FIFO
1287                  * allocation, take space away from current Rx allocation */
1288                 if (tx_space < min_tx_space &&
1289                     ((min_tx_space - tx_space) < pba)) {
1290                         pba = pba - (min_tx_space - tx_space);
1291
1292                         /* if short on rx space, rx wins and must trump tx
1293                          * adjustment */
1294                         if (pba < min_rx_space)
1295                                 pba = min_rx_space;
1296                 }
1297                 wr32(E1000_PBA, pba);
1298         }
1299
1300         /* flow control settings */
1301         /* The high water mark must be low enough to fit one full frame
1302          * (or the size used for early receive) above it in the Rx FIFO.
1303          * Set it to the lower of:
1304          * - 90% of the Rx FIFO size, or
1305          * - the full Rx FIFO size minus one full frame */
1306         hwm = min(((pba << 10) * 9 / 10),
1307                         ((pba << 10) - 2 * adapter->max_frame_size));
1308
1309         if (mac->type < e1000_82576) {
1310                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
1311                 fc->low_water = fc->high_water - 8;
1312         } else {
1313                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1314                 fc->low_water = fc->high_water - 16;
1315         }
1316         fc->pause_time = 0xFFFF;
1317         fc->send_xon = 1;
1318         fc->current_mode = fc->requested_mode;
1319
1320         /* disable receive for all VFs and wait one second */
1321         if (adapter->vfs_allocated_count) {
1322                 int i;
1323                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1324                         adapter->vf_data[i].flags = 0;
1325
1326                 /* ping all the active vfs to let them know we are going down */
1327                 igb_ping_all_vfs(adapter);
1328
1329                 /* disable transmits and receives */
1330                 wr32(E1000_VFRE, 0);
1331                 wr32(E1000_VFTE, 0);
1332         }
1333
1334         /* Allow time for pending master requests to run */
1335         hw->mac.ops.reset_hw(hw);
1336         wr32(E1000_WUC, 0);
1337
1338         if (hw->mac.ops.init_hw(hw))
1339                 dev_err(&pdev->dev, "Hardware Error\n");
1340
1341         if (hw->mac.type == e1000_82580) {
1342                 u32 reg = rd32(E1000_PCIEMISC);
1343                 wr32(E1000_PCIEMISC,
1344                                 reg & ~E1000_PCIEMISC_LX_DECISION);
1345         }
1346         igb_update_mng_vlan(adapter);
1347
1348         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1349         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1350
1351         igb_reset_adaptive(hw);
1352         igb_get_phy_info(hw);
1353 }
1354
1355 static const struct net_device_ops igb_netdev_ops = {
1356         .ndo_open               = igb_open,
1357         .ndo_stop               = igb_close,
1358         .ndo_start_xmit         = igb_xmit_frame_adv,
1359         .ndo_get_stats          = igb_get_stats,
1360         .ndo_set_rx_mode        = igb_set_rx_mode,
1361         .ndo_set_multicast_list = igb_set_rx_mode,
1362         .ndo_set_mac_address    = igb_set_mac,
1363         .ndo_change_mtu         = igb_change_mtu,
1364         .ndo_do_ioctl           = igb_ioctl,
1365         .ndo_tx_timeout         = igb_tx_timeout,
1366         .ndo_validate_addr      = eth_validate_addr,
1367         .ndo_vlan_rx_register   = igb_vlan_rx_register,
1368         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1369         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1370 #ifdef CONFIG_NET_POLL_CONTROLLER
1371         .ndo_poll_controller    = igb_netpoll,
1372 #endif
1373 };
1374
1375 /**
1376  * igb_probe - Device Initialization Routine
1377  * @pdev: PCI device information struct
1378  * @ent: entry in igb_pci_tbl
1379  *
1380  * Returns 0 on success, negative on failure
1381  *
1382  * igb_probe initializes an adapter identified by a pci_dev structure.
1383  * The OS initialization, configuring of the adapter private structure,
1384  * and a hardware reset occur.
1385  **/
1386 static int __devinit igb_probe(struct pci_dev *pdev,
1387                                const struct pci_device_id *ent)
1388 {
1389         struct net_device *netdev;
1390         struct igb_adapter *adapter;
1391         struct e1000_hw *hw;
1392         u16 eeprom_data = 0;
1393         static int global_quad_port_a; /* global quad port a indication */
1394         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1395         unsigned long mmio_start, mmio_len;
1396         int err, pci_using_dac;
1397         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1398         u32 part_num;
1399
1400         err = pci_enable_device_mem(pdev);
1401         if (err)
1402                 return err;
1403
1404         pci_using_dac = 0;
1405         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1406         if (!err) {
1407                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1408                 if (!err)
1409                         pci_using_dac = 1;
1410         } else {
1411                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1412                 if (err) {
1413                         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1414                         if (err) {
1415                                 dev_err(&pdev->dev, "No usable DMA "
1416                                         "configuration, aborting\n");
1417                                 goto err_dma;
1418                         }
1419                 }
1420         }
1421
1422         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1423                                            IORESOURCE_MEM),
1424                                            igb_driver_name);
1425         if (err)
1426                 goto err_pci_reg;
1427
1428         pci_enable_pcie_error_reporting(pdev);
1429
1430         pci_set_master(pdev);
1431         pci_save_state(pdev);
1432
1433         err = -ENOMEM;
1434         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1435                                    IGB_ABS_MAX_TX_QUEUES);
1436         if (!netdev)
1437                 goto err_alloc_etherdev;
1438
1439         SET_NETDEV_DEV(netdev, &pdev->dev);
1440
1441         pci_set_drvdata(pdev, netdev);
1442         adapter = netdev_priv(netdev);
1443         adapter->netdev = netdev;
1444         adapter->pdev = pdev;
1445         hw = &adapter->hw;
1446         hw->back = adapter;
1447         adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1448
1449         mmio_start = pci_resource_start(pdev, 0);
1450         mmio_len = pci_resource_len(pdev, 0);
1451
1452         err = -EIO;
1453         hw->hw_addr = ioremap(mmio_start, mmio_len);
1454         if (!hw->hw_addr)
1455                 goto err_ioremap;
1456
1457         netdev->netdev_ops = &igb_netdev_ops;
1458         igb_set_ethtool_ops(netdev);
1459         netdev->watchdog_timeo = 5 * HZ;
1460
1461         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1462
1463         netdev->mem_start = mmio_start;
1464         netdev->mem_end = mmio_start + mmio_len;
1465
1466         /* PCI config space info */
1467         hw->vendor_id = pdev->vendor;
1468         hw->device_id = pdev->device;
1469         hw->revision_id = pdev->revision;
1470         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1471         hw->subsystem_device_id = pdev->subsystem_device;
1472
1473         /* Copy the default MAC, PHY and NVM function pointers */
1474         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1475         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1476         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1477         /* Initialize skew-specific constants */
1478         err = ei->get_invariants(hw);
1479         if (err)
1480                 goto err_sw_init;
1481
1482         /* setup the private structure */
1483         err = igb_sw_init(adapter);
1484         if (err)
1485                 goto err_sw_init;
1486
1487         igb_get_bus_info_pcie(hw);
1488
1489         hw->phy.autoneg_wait_to_complete = false;
1490         hw->mac.adaptive_ifs = true;
1491
1492         /* Copper options */
1493         if (hw->phy.media_type == e1000_media_type_copper) {
1494                 hw->phy.mdix = AUTO_ALL_MODES;
1495                 hw->phy.disable_polarity_correction = false;
1496                 hw->phy.ms_type = e1000_ms_hw_default;
1497         }
1498
1499         if (igb_check_reset_block(hw))
1500                 dev_info(&pdev->dev,
1501                         "PHY reset is blocked due to SOL/IDER session.\n");
1502
1503         netdev->features = NETIF_F_SG |
1504                            NETIF_F_IP_CSUM |
1505                            NETIF_F_HW_VLAN_TX |
1506                            NETIF_F_HW_VLAN_RX |
1507                            NETIF_F_HW_VLAN_FILTER;
1508
1509         netdev->features |= NETIF_F_IPV6_CSUM;
1510         netdev->features |= NETIF_F_TSO;
1511         netdev->features |= NETIF_F_TSO6;
1512         netdev->features |= NETIF_F_GRO;
1513
1514         netdev->vlan_features |= NETIF_F_TSO;
1515         netdev->vlan_features |= NETIF_F_TSO6;
1516         netdev->vlan_features |= NETIF_F_IP_CSUM;
1517         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1518         netdev->vlan_features |= NETIF_F_SG;
1519
1520         if (pci_using_dac)
1521                 netdev->features |= NETIF_F_HIGHDMA;
1522
1523         if (hw->mac.type >= e1000_82576)
1524                 netdev->features |= NETIF_F_SCTP_CSUM;
1525
1526         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1527
1528         /* before reading the NVM, reset the controller to put the device in a
1529          * known good starting state */
1530         hw->mac.ops.reset_hw(hw);
1531
1532         /* make sure the NVM is good */
1533         if (igb_validate_nvm_checksum(hw) < 0) {
1534                 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1535                 err = -EIO;
1536                 goto err_eeprom;
1537         }
1538
1539         /* copy the MAC address out of the NVM */
1540         if (hw->mac.ops.read_mac_addr(hw))
1541                 dev_err(&pdev->dev, "NVM Read Error\n");
1542
1543         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1544         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1545
1546         if (!is_valid_ether_addr(netdev->perm_addr)) {
1547                 dev_err(&pdev->dev, "Invalid MAC Address\n");
1548                 err = -EIO;
1549                 goto err_eeprom;
1550         }
1551
1552         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1553                     (unsigned long) adapter);
1554         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1555                     (unsigned long) adapter);
1556
1557         INIT_WORK(&adapter->reset_task, igb_reset_task);
1558         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1559
1560         /* Initialize link properties that are user-changeable */
1561         adapter->fc_autoneg = true;
1562         hw->mac.autoneg = true;
1563         hw->phy.autoneg_advertised = 0x2f;
1564
1565         hw->fc.requested_mode = e1000_fc_default;
1566         hw->fc.current_mode = e1000_fc_default;
1567
1568         igb_validate_mdi_setting(hw);
1569
1570         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1571          * enable the ACPI Magic Packet filter
1572          */
1573
1574         if (hw->bus.func == 0)
1575                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1576         else if (hw->mac.type == e1000_82580)
1577                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1578                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1579                                  &eeprom_data);
1580         else if (hw->bus.func == 1)
1581                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1582
1583         if (eeprom_data & eeprom_apme_mask)
1584                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1585
1586         /* now that we have the eeprom settings, apply the special cases where
1587          * the eeprom may be wrong or the board simply won't support wake on
1588          * lan on a particular port */
1589         switch (pdev->device) {
1590         case E1000_DEV_ID_82575GB_QUAD_COPPER:
1591                 adapter->eeprom_wol = 0;
1592                 break;
1593         case E1000_DEV_ID_82575EB_FIBER_SERDES:
1594         case E1000_DEV_ID_82576_FIBER:
1595         case E1000_DEV_ID_82576_SERDES:
1596                 /* Wake events only supported on port A for dual fiber
1597                  * regardless of eeprom setting */
1598                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1599                         adapter->eeprom_wol = 0;
1600                 break;
1601         case E1000_DEV_ID_82576_QUAD_COPPER:
1602                 /* if quad port adapter, disable WoL on all but port A */
1603                 if (global_quad_port_a != 0)
1604                         adapter->eeprom_wol = 0;
1605                 else
1606                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1607                 /* Reset for multiple quad port adapters */
1608                 if (++global_quad_port_a == 4)
1609                         global_quad_port_a = 0;
1610                 break;
1611         }
1612
1613         /* initialize the wol settings based on the eeprom settings */
1614         adapter->wol = adapter->eeprom_wol;
1615         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1616
1617         /* reset the hardware with the new settings */
1618         igb_reset(adapter);
1619
1620         /* let the f/w know that the h/w is now under the control of the
1621          * driver. */
1622         igb_get_hw_control(adapter);
1623
1624         strcpy(netdev->name, "eth%d");
1625         err = register_netdev(netdev);
1626         if (err)
1627                 goto err_register;
1628
1629         /* carrier off reporting is important to ethtool even BEFORE open */
1630         netif_carrier_off(netdev);
1631
1632 #ifdef CONFIG_IGB_DCA
1633         if (dca_add_requester(&pdev->dev) == 0) {
1634                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1635                 dev_info(&pdev->dev, "DCA enabled\n");
1636                 igb_setup_dca(adapter);
1637         }
1638
1639 #endif
1640         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1641         /* print bus type/speed/width info */
1642         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1643                  netdev->name,
1644                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1645                                                             "unknown"),
1646                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1647                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1648                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1649                    "unknown"),
1650                  netdev->dev_addr);
1651
1652         igb_read_part_num(hw, &part_num);
1653         dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1654                 (part_num >> 8), (part_num & 0xff));
1655
1656         dev_info(&pdev->dev,
1657                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1658                 adapter->msix_entries ? "MSI-X" :
1659                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1660                 adapter->num_rx_queues, adapter->num_tx_queues);
1661
1662         return 0;
1663
1664 err_register:
1665         igb_release_hw_control(adapter);
1666 err_eeprom:
1667         if (!igb_check_reset_block(hw))
1668                 igb_reset_phy(hw);
1669
1670         if (hw->flash_address)
1671                 iounmap(hw->flash_address);
1672 err_sw_init:
1673         igb_clear_interrupt_scheme(adapter);
1674         iounmap(hw->hw_addr);
1675 err_ioremap:
1676         free_netdev(netdev);
1677 err_alloc_etherdev:
1678         pci_release_selected_regions(pdev,
1679                                      pci_select_bars(pdev, IORESOURCE_MEM));
1680 err_pci_reg:
1681 err_dma:
1682         pci_disable_device(pdev);
1683         return err;
1684 }
1685
1686 /**
1687  * igb_remove - Device Removal Routine
1688  * @pdev: PCI device information struct
1689  *
1690  * igb_remove is called by the PCI subsystem to alert the driver
1691  * that it should release a PCI device.  The could be caused by a
1692  * Hot-Plug event, or because the driver is going to be removed from
1693  * memory.
1694  **/
1695 static void __devexit igb_remove(struct pci_dev *pdev)
1696 {
1697         struct net_device *netdev = pci_get_drvdata(pdev);
1698         struct igb_adapter *adapter = netdev_priv(netdev);
1699         struct e1000_hw *hw = &adapter->hw;
1700
1701         /* flush_scheduled work may reschedule our watchdog task, so
1702          * explicitly disable watchdog tasks from being rescheduled  */
1703         set_bit(__IGB_DOWN, &adapter->state);
1704         del_timer_sync(&adapter->watchdog_timer);
1705         del_timer_sync(&adapter->phy_info_timer);
1706
1707         flush_scheduled_work();
1708
1709 #ifdef CONFIG_IGB_DCA
1710         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
1711                 dev_info(&pdev->dev, "DCA disabled\n");
1712                 dca_remove_requester(&pdev->dev);
1713                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
1714                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
1715         }
1716 #endif
1717
1718         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
1719          * would have already happened in close and is redundant. */
1720         igb_release_hw_control(adapter);
1721
1722         unregister_netdev(netdev);
1723
1724         if (!igb_check_reset_block(hw))
1725                 igb_reset_phy(hw);
1726
1727         igb_clear_interrupt_scheme(adapter);
1728
1729 #ifdef CONFIG_PCI_IOV
1730         /* reclaim resources allocated to VFs */
1731         if (adapter->vf_data) {
1732                 /* disable iov and allow time for transactions to clear */
1733                 pci_disable_sriov(pdev);
1734                 msleep(500);
1735
1736                 kfree(adapter->vf_data);
1737                 adapter->vf_data = NULL;
1738                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1739                 msleep(100);
1740                 dev_info(&pdev->dev, "IOV Disabled\n");
1741         }
1742 #endif
1743
1744         iounmap(hw->hw_addr);
1745         if (hw->flash_address)
1746                 iounmap(hw->flash_address);
1747         pci_release_selected_regions(pdev,
1748                                      pci_select_bars(pdev, IORESOURCE_MEM));
1749
1750         free_netdev(netdev);
1751
1752         pci_disable_pcie_error_reporting(pdev);
1753
1754         pci_disable_device(pdev);
1755 }
1756
1757 /**
1758  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1759  * @adapter: board private structure to initialize
1760  *
1761  * This function initializes the vf specific data storage and then attempts to
1762  * allocate the VFs.  The reason for ordering it this way is because it is much
1763  * mor expensive time wise to disable SR-IOV than it is to allocate and free
1764  * the memory for the VFs.
1765  **/
1766 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1767 {
1768 #ifdef CONFIG_PCI_IOV
1769         struct pci_dev *pdev = adapter->pdev;
1770
1771         if (adapter->vfs_allocated_count > 7)
1772                 adapter->vfs_allocated_count = 7;
1773
1774         if (adapter->vfs_allocated_count) {
1775                 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1776                                            sizeof(struct vf_data_storage),
1777                                            GFP_KERNEL);
1778                 /* if allocation failed then we do not support SR-IOV */
1779                 if (!adapter->vf_data) {
1780                         adapter->vfs_allocated_count = 0;
1781                         dev_err(&pdev->dev, "Unable to allocate memory for VF "
1782                                 "Data Storage\n");
1783                 }
1784         }
1785
1786         if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1787                 kfree(adapter->vf_data);
1788                 adapter->vf_data = NULL;
1789 #endif /* CONFIG_PCI_IOV */
1790                 adapter->vfs_allocated_count = 0;
1791 #ifdef CONFIG_PCI_IOV
1792         } else {
1793                 unsigned char mac_addr[ETH_ALEN];
1794                 int i;
1795                 dev_info(&pdev->dev, "%d vfs allocated\n",
1796                          adapter->vfs_allocated_count);
1797                 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1798                         random_ether_addr(mac_addr);
1799                         igb_set_vf_mac(adapter, i, mac_addr);
1800                 }
1801         }
1802 #endif /* CONFIG_PCI_IOV */
1803 }
1804
1805
1806 /**
1807  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1808  * @adapter: board private structure to initialize
1809  *
1810  * igb_init_hw_timer initializes the function pointer and values for the hw
1811  * timer found in hardware.
1812  **/
1813 static void igb_init_hw_timer(struct igb_adapter *adapter)
1814 {
1815         struct e1000_hw *hw = &adapter->hw;
1816
1817         switch (hw->mac.type) {
1818         case e1000_82580:
1819                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1820                 adapter->cycles.read = igb_read_clock;
1821                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1822                 adapter->cycles.mult = 1;
1823                 /*
1824                  * The 82580 timesync updates the system timer every 8ns by 8ns
1825                  * and the value cannot be shifted.  Instead we need to shift
1826                  * the registers to generate a 64bit timer value.  As a result
1827                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1828                  * 24 in order to generate a larger value for synchronization.
1829                  */
1830                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1831                 /* disable system timer temporarily by setting bit 31 */
1832                 wr32(E1000_TSAUXC, 0x80000000);
1833                 wrfl();
1834
1835                 /* Set registers so that rollover occurs soon to test this. */
1836                 wr32(E1000_SYSTIMR, 0x00000000);
1837                 wr32(E1000_SYSTIML, 0x80000000);
1838                 wr32(E1000_SYSTIMH, 0x000000FF);
1839                 wrfl();
1840
1841                 /* enable system timer by clearing bit 31 */
1842                 wr32(E1000_TSAUXC, 0x0);
1843                 wrfl();
1844
1845                 timecounter_init(&adapter->clock,
1846                                  &adapter->cycles,
1847                                  ktime_to_ns(ktime_get_real()));
1848                 /*
1849                  * Synchronize our NIC clock against system wall clock. NIC
1850                  * time stamp reading requires ~3us per sample, each sample
1851                  * was pretty stable even under load => only require 10
1852                  * samples for each offset comparison.
1853                  */
1854                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1855                 adapter->compare.source = &adapter->clock;
1856                 adapter->compare.target = ktime_get_real;
1857                 adapter->compare.num_samples = 10;
1858                 timecompare_update(&adapter->compare, 0);
1859                 break;
1860         case e1000_82576:
1861                 /*
1862                  * Initialize hardware timer: we keep it running just in case
1863                  * that some program needs it later on.
1864                  */
1865                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1866                 adapter->cycles.read = igb_read_clock;
1867                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1868                 adapter->cycles.mult = 1;
1869                 /**
1870                  * Scale the NIC clock cycle by a large factor so that
1871                  * relatively small clock corrections can be added or
1872                  * substracted at each clock tick. The drawbacks of a large
1873                  * factor are a) that the clock register overflows more quickly
1874                  * (not such a big deal) and b) that the increment per tick has
1875                  * to fit into 24 bits.  As a result we need to use a shift of
1876                  * 19 so we can fit a value of 16 into the TIMINCA register.
1877                  */
1878                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1879                 wr32(E1000_TIMINCA,
1880                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
1881                                 (16 << IGB_82576_TSYNC_SHIFT));
1882
1883                 /* Set registers so that rollover occurs soon to test this. */
1884                 wr32(E1000_SYSTIML, 0x00000000);
1885                 wr32(E1000_SYSTIMH, 0xFF800000);
1886                 wrfl();
1887
1888                 timecounter_init(&adapter->clock,
1889                                  &adapter->cycles,
1890                                  ktime_to_ns(ktime_get_real()));
1891                 /*
1892                  * Synchronize our NIC clock against system wall clock. NIC
1893                  * time stamp reading requires ~3us per sample, each sample
1894                  * was pretty stable even under load => only require 10
1895                  * samples for each offset comparison.
1896                  */
1897                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1898                 adapter->compare.source = &adapter->clock;
1899                 adapter->compare.target = ktime_get_real;
1900                 adapter->compare.num_samples = 10;
1901                 timecompare_update(&adapter->compare, 0);
1902                 break;
1903         case e1000_82575:
1904                 /* 82575 does not support timesync */
1905         default:
1906                 break;
1907         }
1908
1909 }
1910
1911 /**
1912  * igb_sw_init - Initialize general software structures (struct igb_adapter)
1913  * @adapter: board private structure to initialize
1914  *
1915  * igb_sw_init initializes the Adapter private data structure.
1916  * Fields are initialized based on PCI device information and
1917  * OS network device settings (MTU size).
1918  **/
1919 static int __devinit igb_sw_init(struct igb_adapter *adapter)
1920 {
1921         struct e1000_hw *hw = &adapter->hw;
1922         struct net_device *netdev = adapter->netdev;
1923         struct pci_dev *pdev = adapter->pdev;
1924
1925         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1926
1927         adapter->tx_ring_count = IGB_DEFAULT_TXD;
1928         adapter->rx_ring_count = IGB_DEFAULT_RXD;
1929         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1930         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1931
1932         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1933         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1934
1935 #ifdef CONFIG_PCI_IOV
1936         if (hw->mac.type == e1000_82576)
1937                 adapter->vfs_allocated_count = max_vfs;
1938
1939 #endif /* CONFIG_PCI_IOV */
1940         adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1941
1942         /*
1943          * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1944          * then we should combine the queues into a queue pair in order to
1945          * conserve interrupts due to limited supply
1946          */
1947         if ((adapter->rss_queues > 4) ||
1948             ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1949                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1950
1951         /* This call may decrease the number of queues */
1952         if (igb_init_interrupt_scheme(adapter)) {
1953                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1954                 return -ENOMEM;
1955         }
1956
1957         igb_init_hw_timer(adapter);
1958         igb_probe_vfs(adapter);
1959
1960         /* Explicitly disable IRQ since the NIC can be in any state. */
1961         igb_irq_disable(adapter);
1962
1963         set_bit(__IGB_DOWN, &adapter->state);
1964         return 0;
1965 }
1966
1967 /**
1968  * igb_open - Called when a network interface is made active
1969  * @netdev: network interface device structure
1970  *
1971  * Returns 0 on success, negative value on failure
1972  *
1973  * The open entry point is called when a network interface is made
1974  * active by the system (IFF_UP).  At this point all resources needed
1975  * for transmit and receive operations are allocated, the interrupt
1976  * handler is registered with the OS, the watchdog timer is started,
1977  * and the stack is notified that the interface is ready.
1978  **/
1979 static int igb_open(struct net_device *netdev)
1980 {
1981         struct igb_adapter *adapter = netdev_priv(netdev);
1982         struct e1000_hw *hw = &adapter->hw;
1983         int err;
1984         int i;
1985
1986         /* disallow open during test */
1987         if (test_bit(__IGB_TESTING, &adapter->state))
1988                 return -EBUSY;
1989
1990         netif_carrier_off(netdev);
1991
1992         /* allocate transmit descriptors */
1993         err = igb_setup_all_tx_resources(adapter);
1994         if (err)
1995                 goto err_setup_tx;
1996
1997         /* allocate receive descriptors */
1998         err = igb_setup_all_rx_resources(adapter);
1999         if (err)
2000                 goto err_setup_rx;
2001
2002         /* e1000_power_up_phy(adapter); */
2003
2004         /* before we allocate an interrupt, we must be ready to handle it.
2005          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2006          * as soon as we call pci_request_irq, so we have to setup our
2007          * clean_rx handler before we do so.  */
2008         igb_configure(adapter);
2009
2010         err = igb_request_irq(adapter);
2011         if (err)
2012                 goto err_req_irq;
2013
2014         /* From here on the code is the same as igb_up() */
2015         clear_bit(__IGB_DOWN, &adapter->state);
2016
2017         for (i = 0; i < adapter->num_q_vectors; i++) {
2018                 struct igb_q_vector *q_vector = adapter->q_vector[i];
2019                 napi_enable(&q_vector->napi);
2020         }
2021
2022         /* Clear any pending interrupts. */
2023         rd32(E1000_ICR);
2024
2025         igb_irq_enable(adapter);
2026
2027         /* notify VFs that reset has been completed */
2028         if (adapter->vfs_allocated_count) {
2029                 u32 reg_data = rd32(E1000_CTRL_EXT);
2030                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2031                 wr32(E1000_CTRL_EXT, reg_data);
2032         }
2033
2034         netif_tx_start_all_queues(netdev);
2035
2036         /* start the watchdog. */
2037         hw->mac.get_link_status = 1;
2038         schedule_work(&adapter->watchdog_task);
2039
2040         return 0;
2041
2042 err_req_irq:
2043         igb_release_hw_control(adapter);
2044         /* e1000_power_down_phy(adapter); */
2045         igb_free_all_rx_resources(adapter);
2046 err_setup_rx:
2047         igb_free_all_tx_resources(adapter);
2048 err_setup_tx:
2049         igb_reset(adapter);
2050
2051         return err;
2052 }
2053
2054 /**
2055  * igb_close - Disables a network interface
2056  * @netdev: network interface device structure
2057  *
2058  * Returns 0, this is not allowed to fail
2059  *
2060  * The close entry point is called when an interface is de-activated
2061  * by the OS.  The hardware is still under the driver's control, but
2062  * needs to be disabled.  A global MAC reset is issued to stop the
2063  * hardware, and all transmit and receive resources are freed.
2064  **/
2065 static int igb_close(struct net_device *netdev)
2066 {
2067         struct igb_adapter *adapter = netdev_priv(netdev);
2068
2069         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2070         igb_down(adapter);
2071
2072         igb_free_irq(adapter);
2073
2074         igb_free_all_tx_resources(adapter);
2075         igb_free_all_rx_resources(adapter);
2076
2077         return 0;
2078 }
2079
2080 /**
2081  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2082  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2083  *
2084  * Return 0 on success, negative on failure
2085  **/
2086 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2087 {
2088         struct pci_dev *pdev = tx_ring->pdev;
2089         int size;
2090
2091         size = sizeof(struct igb_buffer) * tx_ring->count;
2092         tx_ring->buffer_info = vmalloc(size);
2093         if (!tx_ring->buffer_info)
2094                 goto err;
2095         memset(tx_ring->buffer_info, 0, size);
2096
2097         /* round up to nearest 4K */
2098         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2099         tx_ring->size = ALIGN(tx_ring->size, 4096);
2100
2101         tx_ring->desc = pci_alloc_consistent(pdev,
2102                                              tx_ring->size,
2103                                              &tx_ring->dma);
2104
2105         if (!tx_ring->desc)
2106                 goto err;
2107
2108         tx_ring->next_to_use = 0;
2109         tx_ring->next_to_clean = 0;
2110         return 0;
2111
2112 err:
2113         vfree(tx_ring->buffer_info);
2114         dev_err(&pdev->dev,
2115                 "Unable to allocate memory for the transmit descriptor ring\n");
2116         return -ENOMEM;
2117 }
2118
2119 /**
2120  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2121  *                                (Descriptors) for all queues
2122  * @adapter: board private structure
2123  *
2124  * Return 0 on success, negative on failure
2125  **/
2126 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2127 {
2128         struct pci_dev *pdev = adapter->pdev;
2129         int i, err = 0;
2130
2131         for (i = 0; i < adapter->num_tx_queues; i++) {
2132                 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
2133                 if (err) {
2134                         dev_err(&pdev->dev,
2135                                 "Allocation for Tx Queue %u failed\n", i);
2136                         for (i--; i >= 0; i--)
2137                                 igb_free_tx_resources(&adapter->tx_ring[i]);
2138                         break;
2139                 }
2140         }
2141
2142         for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2143                 int r_idx = i % adapter->num_tx_queues;
2144                 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
2145         }
2146         return err;
2147 }
2148
2149 /**
2150  * igb_setup_tctl - configure the transmit control registers
2151  * @adapter: Board private structure
2152  **/
2153 void igb_setup_tctl(struct igb_adapter *adapter)
2154 {
2155         struct e1000_hw *hw = &adapter->hw;
2156         u32 tctl;
2157
2158         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2159         wr32(E1000_TXDCTL(0), 0);
2160
2161         /* Program the Transmit Control Register */
2162         tctl = rd32(E1000_TCTL);
2163         tctl &= ~E1000_TCTL_CT;
2164         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2165                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2166
2167         igb_config_collision_dist(hw);
2168
2169         /* Enable transmits */
2170         tctl |= E1000_TCTL_EN;
2171
2172         wr32(E1000_TCTL, tctl);
2173 }
2174
2175 /**
2176  * igb_configure_tx_ring - Configure transmit ring after Reset
2177  * @adapter: board private structure
2178  * @ring: tx ring to configure
2179  *
2180  * Configure a transmit ring after a reset.
2181  **/
2182 void igb_configure_tx_ring(struct igb_adapter *adapter,
2183                            struct igb_ring *ring)
2184 {
2185         struct e1000_hw *hw = &adapter->hw;
2186         u32 txdctl;
2187         u64 tdba = ring->dma;
2188         int reg_idx = ring->reg_idx;
2189
2190         /* disable the queue */
2191         txdctl = rd32(E1000_TXDCTL(reg_idx));
2192         wr32(E1000_TXDCTL(reg_idx),
2193                         txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2194         wrfl();
2195         mdelay(10);
2196
2197         wr32(E1000_TDLEN(reg_idx),
2198                         ring->count * sizeof(union e1000_adv_tx_desc));
2199         wr32(E1000_TDBAL(reg_idx),
2200                         tdba & 0x00000000ffffffffULL);
2201         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2202
2203         ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2204         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2205         writel(0, ring->head);
2206         writel(0, ring->tail);
2207
2208         txdctl |= IGB_TX_PTHRESH;
2209         txdctl |= IGB_TX_HTHRESH << 8;
2210         txdctl |= IGB_TX_WTHRESH << 16;
2211
2212         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2213         wr32(E1000_TXDCTL(reg_idx), txdctl);
2214 }
2215
2216 /**
2217  * igb_configure_tx - Configure transmit Unit after Reset
2218  * @adapter: board private structure
2219  *
2220  * Configure the Tx unit of the MAC after a reset.
2221  **/
2222 static void igb_configure_tx(struct igb_adapter *adapter)
2223 {
2224         int i;
2225
2226         for (i = 0; i < adapter->num_tx_queues; i++)
2227                 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
2228 }
2229
2230 /**
2231  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2232  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2233  *
2234  * Returns 0 on success, negative on failure
2235  **/
2236 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2237 {
2238         struct pci_dev *pdev = rx_ring->pdev;
2239         int size, desc_len;
2240
2241         size = sizeof(struct igb_buffer) * rx_ring->count;
2242         rx_ring->buffer_info = vmalloc(size);
2243         if (!rx_ring->buffer_info)
2244                 goto err;
2245         memset(rx_ring->buffer_info, 0, size);
2246
2247         desc_len = sizeof(union e1000_adv_rx_desc);
2248
2249         /* Round up to nearest 4K */
2250         rx_ring->size = rx_ring->count * desc_len;
2251         rx_ring->size = ALIGN(rx_ring->size, 4096);
2252
2253         rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2254                                              &rx_ring->dma);
2255
2256         if (!rx_ring->desc)
2257                 goto err;
2258
2259         rx_ring->next_to_clean = 0;
2260         rx_ring->next_to_use = 0;
2261
2262         return 0;
2263
2264 err:
2265         vfree(rx_ring->buffer_info);
2266         rx_ring->buffer_info = NULL;
2267         dev_err(&pdev->dev, "Unable to allocate memory for "
2268                 "the receive descriptor ring\n");
2269         return -ENOMEM;
2270 }
2271
2272 /**
2273  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2274  *                                (Descriptors) for all queues
2275  * @adapter: board private structure
2276  *
2277  * Return 0 on success, negative on failure
2278  **/
2279 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2280 {
2281         struct pci_dev *pdev = adapter->pdev;
2282         int i, err = 0;
2283
2284         for (i = 0; i < adapter->num_rx_queues; i++) {
2285                 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
2286                 if (err) {
2287                         dev_err(&pdev->dev,
2288                                 "Allocation for Rx Queue %u failed\n", i);
2289                         for (i--; i >= 0; i--)
2290                                 igb_free_rx_resources(&adapter->rx_ring[i]);
2291                         break;
2292                 }
2293         }
2294
2295         return err;
2296 }
2297
2298 /**
2299  * igb_setup_mrqc - configure the multiple receive queue control registers
2300  * @adapter: Board private structure
2301  **/
2302 static void igb_setup_mrqc(struct igb_adapter *adapter)
2303 {
2304         struct e1000_hw *hw = &adapter->hw;
2305         u32 mrqc, rxcsum;
2306         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2307         union e1000_reta {
2308                 u32 dword;
2309                 u8  bytes[4];
2310         } reta;
2311         static const u8 rsshash[40] = {
2312                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2313                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2314                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2315                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2316
2317         /* Fill out hash function seeds */
2318         for (j = 0; j < 10; j++) {
2319                 u32 rsskey = rsshash[(j * 4)];
2320                 rsskey |= rsshash[(j * 4) + 1] << 8;
2321                 rsskey |= rsshash[(j * 4) + 2] << 16;
2322                 rsskey |= rsshash[(j * 4) + 3] << 24;
2323                 array_wr32(E1000_RSSRK(0), j, rsskey);
2324         }
2325
2326         num_rx_queues = adapter->rss_queues;
2327
2328         if (adapter->vfs_allocated_count) {
2329                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2330                 switch (hw->mac.type) {
2331                 case e1000_82580:
2332                         num_rx_queues = 1;
2333                         shift = 0;
2334                         break;
2335                 case e1000_82576:
2336                         shift = 3;
2337                         num_rx_queues = 2;
2338                         break;
2339                 case e1000_82575:
2340                         shift = 2;
2341                         shift2 = 6;
2342                 default:
2343                         break;
2344                 }
2345         } else {
2346                 if (hw->mac.type == e1000_82575)
2347                         shift = 6;
2348         }
2349
2350         for (j = 0; j < (32 * 4); j++) {
2351                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2352                 if (shift2)
2353                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2354                 if ((j & 3) == 3)
2355                         wr32(E1000_RETA(j >> 2), reta.dword);
2356         }
2357
2358         /*
2359          * Disable raw packet checksumming so that RSS hash is placed in
2360          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2361          * offloads as they are enabled by default
2362          */
2363         rxcsum = rd32(E1000_RXCSUM);
2364         rxcsum |= E1000_RXCSUM_PCSD;
2365
2366         if (adapter->hw.mac.type >= e1000_82576)
2367                 /* Enable Receive Checksum Offload for SCTP */
2368                 rxcsum |= E1000_RXCSUM_CRCOFL;
2369
2370         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2371         wr32(E1000_RXCSUM, rxcsum);
2372
2373         /* If VMDq is enabled then we set the appropriate mode for that, else
2374          * we default to RSS so that an RSS hash is calculated per packet even
2375          * if we are only using one queue */
2376         if (adapter->vfs_allocated_count) {
2377                 if (hw->mac.type > e1000_82575) {
2378                         /* Set the default pool for the PF's first queue */
2379                         u32 vtctl = rd32(E1000_VT_CTL);
2380                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2381                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2382                         vtctl |= adapter->vfs_allocated_count <<
2383                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2384                         wr32(E1000_VT_CTL, vtctl);
2385                 }
2386                 if (adapter->rss_queues > 1)
2387                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2388                 else
2389                         mrqc = E1000_MRQC_ENABLE_VMDQ;
2390         } else {
2391                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2392         }
2393         igb_vmm_control(adapter);
2394
2395         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2396                  E1000_MRQC_RSS_FIELD_IPV4_TCP);
2397         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2398                  E1000_MRQC_RSS_FIELD_IPV6_TCP);
2399         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2400                  E1000_MRQC_RSS_FIELD_IPV6_UDP);
2401         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2402                  E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2403
2404         wr32(E1000_MRQC, mrqc);
2405 }
2406
2407 /**
2408  * igb_setup_rctl - configure the receive control registers
2409  * @adapter: Board private structure
2410  **/
2411 void igb_setup_rctl(struct igb_adapter *adapter)
2412 {
2413         struct e1000_hw *hw = &adapter->hw;
2414         u32 rctl;
2415
2416         rctl = rd32(E1000_RCTL);
2417
2418         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2419         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2420
2421         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2422                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2423
2424         /*
2425          * enable stripping of CRC. It's unlikely this will break BMC
2426          * redirection as it did with e1000. Newer features require
2427          * that the HW strips the CRC.
2428          */
2429         rctl |= E1000_RCTL_SECRC;
2430
2431         /* disable store bad packets and clear size bits. */
2432         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2433
2434         /* enable LPE to prevent packets larger than max_frame_size */
2435         rctl |= E1000_RCTL_LPE;
2436
2437         /* disable queue 0 to prevent tail write w/o re-config */
2438         wr32(E1000_RXDCTL(0), 0);
2439
2440         /* Attention!!!  For SR-IOV PF driver operations you must enable
2441          * queue drop for all VF and PF queues to prevent head of line blocking
2442          * if an un-trusted VF does not provide descriptors to hardware.
2443          */
2444         if (adapter->vfs_allocated_count) {
2445                 /* set all queue drop enable bits */
2446                 wr32(E1000_QDE, ALL_QUEUES);
2447         }
2448
2449         wr32(E1000_RCTL, rctl);
2450 }
2451
2452 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2453                                    int vfn)
2454 {
2455         struct e1000_hw *hw = &adapter->hw;
2456         u32 vmolr;
2457
2458         /* if it isn't the PF check to see if VFs are enabled and
2459          * increase the size to support vlan tags */
2460         if (vfn < adapter->vfs_allocated_count &&
2461             adapter->vf_data[vfn].vlans_enabled)
2462                 size += VLAN_TAG_SIZE;
2463
2464         vmolr = rd32(E1000_VMOLR(vfn));
2465         vmolr &= ~E1000_VMOLR_RLPML_MASK;
2466         vmolr |= size | E1000_VMOLR_LPE;
2467         wr32(E1000_VMOLR(vfn), vmolr);
2468
2469         return 0;
2470 }
2471
2472 /**
2473  * igb_rlpml_set - set maximum receive packet size
2474  * @adapter: board private structure
2475  *
2476  * Configure maximum receivable packet size.
2477  **/
2478 static void igb_rlpml_set(struct igb_adapter *adapter)
2479 {
2480         u32 max_frame_size = adapter->max_frame_size;
2481         struct e1000_hw *hw = &adapter->hw;
2482         u16 pf_id = adapter->vfs_allocated_count;
2483
2484         if (adapter->vlgrp)
2485                 max_frame_size += VLAN_TAG_SIZE;
2486
2487         /* if vfs are enabled we set RLPML to the largest possible request
2488          * size and set the VMOLR RLPML to the size we need */
2489         if (pf_id) {
2490                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2491                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2492         }
2493
2494         wr32(E1000_RLPML, max_frame_size);
2495 }
2496
2497 static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2498 {
2499         struct e1000_hw *hw = &adapter->hw;
2500         u32 vmolr;
2501
2502         /*
2503          * This register exists only on 82576 and newer so if we are older then
2504          * we should exit and do nothing
2505          */
2506         if (hw->mac.type < e1000_82576)
2507                 return;
2508
2509         vmolr = rd32(E1000_VMOLR(vfn));
2510         vmolr |= E1000_VMOLR_AUPE |        /* Accept untagged packets */
2511                  E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
2512
2513         /* clear all bits that might not be set */
2514         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2515
2516         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2517                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2518         /*
2519          * for VMDq only allow the VFs and pool 0 to accept broadcast and
2520          * multicast packets
2521          */
2522         if (vfn <= adapter->vfs_allocated_count)
2523                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
2524
2525         wr32(E1000_VMOLR(vfn), vmolr);
2526 }
2527
2528 /**
2529  * igb_configure_rx_ring - Configure a receive ring after Reset
2530  * @adapter: board private structure
2531  * @ring: receive ring to be configured
2532  *
2533  * Configure the Rx unit of the MAC after a reset.
2534  **/
2535 void igb_configure_rx_ring(struct igb_adapter *adapter,
2536                            struct igb_ring *ring)
2537 {
2538         struct e1000_hw *hw = &adapter->hw;
2539         u64 rdba = ring->dma;
2540         int reg_idx = ring->reg_idx;
2541         u32 srrctl, rxdctl;
2542
2543         /* disable the queue */
2544         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2545         wr32(E1000_RXDCTL(reg_idx),
2546                         rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2547
2548         /* Set DMA base address registers */
2549         wr32(E1000_RDBAL(reg_idx),
2550              rdba & 0x00000000ffffffffULL);
2551         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2552         wr32(E1000_RDLEN(reg_idx),
2553                        ring->count * sizeof(union e1000_adv_rx_desc));
2554
2555         /* initialize head and tail */
2556         ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2557         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2558         writel(0, ring->head);
2559         writel(0, ring->tail);
2560
2561         /* set descriptor configuration */
2562         if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2563                 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2564                          E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2565 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2566                 srrctl |= IGB_RXBUFFER_16384 >>
2567                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2568 #else
2569                 srrctl |= (PAGE_SIZE / 2) >>
2570                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2571 #endif
2572                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2573         } else {
2574                 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2575                          E1000_SRRCTL_BSIZEPKT_SHIFT;
2576                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2577         }
2578
2579         wr32(E1000_SRRCTL(reg_idx), srrctl);
2580
2581         /* set filtering for VMDQ pools */
2582         igb_set_vmolr(adapter, reg_idx & 0x7);
2583
2584         /* enable receive descriptor fetching */
2585         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2586         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2587         rxdctl &= 0xFFF00000;
2588         rxdctl |= IGB_RX_PTHRESH;
2589         rxdctl |= IGB_RX_HTHRESH << 8;
2590         rxdctl |= IGB_RX_WTHRESH << 16;
2591         wr32(E1000_RXDCTL(reg_idx), rxdctl);
2592 }
2593
2594 /**
2595  * igb_configure_rx - Configure receive Unit after Reset
2596  * @adapter: board private structure
2597  *
2598  * Configure the Rx unit of the MAC after a reset.
2599  **/
2600 static void igb_configure_rx(struct igb_adapter *adapter)
2601 {
2602         int i;
2603
2604         /* set UTA to appropriate mode */
2605         igb_set_uta(adapter);
2606
2607         /* set the correct pool for the PF default MAC address in entry 0 */
2608         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2609                          adapter->vfs_allocated_count);
2610
2611         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2612          * the Base and Length of the Rx Descriptor Ring */
2613         for (i = 0; i < adapter->num_rx_queues; i++)
2614                 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
2615 }
2616
2617 /**
2618  * igb_free_tx_resources - Free Tx Resources per Queue
2619  * @tx_ring: Tx descriptor ring for a specific queue
2620  *
2621  * Free all transmit software resources
2622  **/
2623 void igb_free_tx_resources(struct igb_ring *tx_ring)
2624 {
2625         igb_clean_tx_ring(tx_ring);
2626
2627         vfree(tx_ring->buffer_info);
2628         tx_ring->buffer_info = NULL;
2629
2630         /* if not set, then don't free */
2631         if (!tx_ring->desc)
2632                 return;
2633
2634         pci_free_consistent(tx_ring->pdev, tx_ring->size,
2635                             tx_ring->desc, tx_ring->dma);
2636
2637         tx_ring->desc = NULL;
2638 }
2639
2640 /**
2641  * igb_free_all_tx_resources - Free Tx Resources for All Queues
2642  * @adapter: board private structure
2643  *
2644  * Free all transmit software resources
2645  **/
2646 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2647 {
2648         int i;
2649
2650         for (i = 0; i < adapter->num_tx_queues; i++)
2651                 igb_free_tx_resources(&adapter->tx_ring[i]);
2652 }
2653
2654 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2655                                     struct igb_buffer *buffer_info)
2656 {
2657         if (buffer_info->dma) {
2658                 if (buffer_info->mapped_as_page)
2659                         pci_unmap_page(tx_ring->pdev,
2660                                         buffer_info->dma,
2661                                         buffer_info->length,
2662                                         PCI_DMA_TODEVICE);
2663                 else
2664                         pci_unmap_single(tx_ring->pdev,
2665                                         buffer_info->dma,
2666                                         buffer_info->length,
2667                                         PCI_DMA_TODEVICE);
2668                 buffer_info->dma = 0;
2669         }
2670         if (buffer_info->skb) {
2671                 dev_kfree_skb_any(buffer_info->skb);
2672                 buffer_info->skb = NULL;
2673         }
2674         buffer_info->time_stamp = 0;
2675         buffer_info->length = 0;
2676         buffer_info->next_to_watch = 0;
2677         buffer_info->mapped_as_page = false;
2678 }
2679
2680 /**
2681  * igb_clean_tx_ring - Free Tx Buffers
2682  * @tx_ring: ring to be cleaned
2683  **/
2684 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
2685 {
2686         struct igb_buffer *buffer_info;
2687         unsigned long size;
2688         unsigned int i;
2689
2690         if (!tx_ring->buffer_info)
2691                 return;
2692         /* Free all the Tx ring sk_buffs */
2693
2694         for (i = 0; i < tx_ring->count; i++) {
2695                 buffer_info = &tx_ring->buffer_info[i];
2696                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
2697         }
2698
2699         size = sizeof(struct igb_buffer) * tx_ring->count;
2700         memset(tx_ring->buffer_info, 0, size);
2701
2702         /* Zero out the descriptor ring */
2703         memset(tx_ring->desc, 0, tx_ring->size);
2704
2705         tx_ring->next_to_use = 0;
2706         tx_ring->next_to_clean = 0;
2707 }
2708
2709 /**
2710  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2711  * @adapter: board private structure
2712  **/
2713 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2714 {
2715         int i;
2716
2717         for (i = 0; i < adapter->num_tx_queues; i++)
2718                 igb_clean_tx_ring(&adapter->tx_ring[i]);
2719 }
2720
2721 /**
2722  * igb_free_rx_resources - Free Rx Resources
2723  * @rx_ring: ring to clean the resources from
2724  *
2725  * Free all receive software resources
2726  **/
2727 void igb_free_rx_resources(struct igb_ring *rx_ring)
2728 {
2729         igb_clean_rx_ring(rx_ring);
2730
2731         vfree(rx_ring->buffer_info);
2732         rx_ring->buffer_info = NULL;
2733
2734         /* if not set, then don't free */
2735         if (!rx_ring->desc)
2736                 return;
2737
2738         pci_free_consistent(rx_ring->pdev, rx_ring->size,
2739                             rx_ring->desc, rx_ring->dma);
2740
2741         rx_ring->desc = NULL;
2742 }
2743
2744 /**
2745  * igb_free_all_rx_resources - Free Rx Resources for All Queues
2746  * @adapter: board private structure
2747  *
2748  * Free all receive software resources
2749  **/
2750 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2751 {
2752         int i;
2753
2754         for (i = 0; i < adapter->num_rx_queues; i++)
2755                 igb_free_rx_resources(&adapter->rx_ring[i]);
2756 }
2757
2758 /**
2759  * igb_clean_rx_ring - Free Rx Buffers per Queue
2760  * @rx_ring: ring to free buffers from
2761  **/
2762 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
2763 {
2764         struct igb_buffer *buffer_info;
2765         unsigned long size;
2766         unsigned int i;
2767
2768         if (!rx_ring->buffer_info)
2769                 return;
2770
2771         /* Free all the Rx ring sk_buffs */
2772         for (i = 0; i < rx_ring->count; i++) {
2773                 buffer_info = &rx_ring->buffer_info[i];
2774                 if (buffer_info->dma) {
2775                         pci_unmap_single(rx_ring->pdev,
2776                                          buffer_info->dma,
2777                                          rx_ring->rx_buffer_len,
2778                                          PCI_DMA_FROMDEVICE);
2779                         buffer_info->dma = 0;
2780                 }
2781
2782                 if (buffer_info->skb) {
2783                         dev_kfree_skb(buffer_info->skb);
2784                         buffer_info->skb = NULL;
2785                 }
2786                 if (buffer_info->page_dma) {
2787                         pci_unmap_page(rx_ring->pdev,
2788                                        buffer_info->page_dma,
2789                                        PAGE_SIZE / 2,
2790                                        PCI_DMA_FROMDEVICE);
2791                         buffer_info->page_dma = 0;
2792                 }
2793                 if (buffer_info->page) {
2794                         put_page(buffer_info->page);
2795                         buffer_info->page = NULL;
2796                         buffer_info->page_offset = 0;
2797                 }
2798         }
2799
2800         size = sizeof(struct igb_buffer) * rx_ring->count;
2801         memset(rx_ring->buffer_info, 0, size);
2802
2803         /* Zero out the descriptor ring */
2804         memset(rx_ring->desc, 0, rx_ring->size);
2805
2806         rx_ring->next_to_clean = 0;
2807         rx_ring->next_to_use = 0;
2808 }
2809
2810 /**
2811  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2812  * @adapter: board private structure
2813  **/
2814 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2815 {
2816         int i;
2817
2818         for (i = 0; i < adapter->num_rx_queues; i++)
2819                 igb_clean_rx_ring(&adapter->rx_ring[i]);
2820 }
2821
2822 /**
2823  * igb_set_mac - Change the Ethernet Address of the NIC
2824  * @netdev: network interface device structure
2825  * @p: pointer to an address structure
2826  *
2827  * Returns 0 on success, negative on failure
2828  **/
2829 static int igb_set_mac(struct net_device *netdev, void *p)
2830 {
2831         struct igb_adapter *adapter = netdev_priv(netdev);
2832         struct e1000_hw *hw = &adapter->hw;
2833         struct sockaddr *addr = p;
2834
2835         if (!is_valid_ether_addr(addr->sa_data))
2836                 return -EADDRNOTAVAIL;
2837
2838         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2839         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2840
2841         /* set the correct pool for the new PF MAC address in entry 0 */
2842         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2843                          adapter->vfs_allocated_count);
2844
2845         return 0;
2846 }
2847
2848 /**
2849  * igb_write_mc_addr_list - write multicast addresses to MTA
2850  * @netdev: network interface device structure
2851  *
2852  * Writes multicast address list to the MTA hash table.
2853  * Returns: -ENOMEM on failure
2854  *                0 on no addresses written
2855  *                X on writing X addresses to MTA
2856  **/
2857 static int igb_write_mc_addr_list(struct net_device *netdev)
2858 {
2859         struct igb_adapter *adapter = netdev_priv(netdev);
2860         struct e1000_hw *hw = &adapter->hw;
2861         struct dev_mc_list *mc_ptr = netdev->mc_list;
2862         u8  *mta_list;
2863         u32 vmolr = 0;
2864         int i;
2865
2866         if (!netdev->mc_count) {
2867                 /* nothing to program, so clear mc list */
2868                 igb_update_mc_addr_list(hw, NULL, 0);
2869                 igb_restore_vf_multicasts(adapter);
2870                 return 0;
2871         }
2872
2873         mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2874         if (!mta_list)
2875                 return -ENOMEM;
2876
2877         /* set vmolr receive overflow multicast bit */
2878         vmolr |= E1000_VMOLR_ROMPE;
2879
2880         /* The shared function expects a packed array of only addresses. */
2881         mc_ptr = netdev->mc_list;
2882
2883         for (i = 0; i < netdev->mc_count; i++) {
2884                 if (!mc_ptr)
2885                         break;
2886                 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2887                 mc_ptr = mc_ptr->next;
2888         }
2889         igb_update_mc_addr_list(hw, mta_list, i);
2890         kfree(mta_list);
2891
2892         return netdev->mc_count;
2893 }
2894
2895 /**
2896  * igb_write_uc_addr_list - write unicast addresses to RAR table
2897  * @netdev: network interface device structure
2898  *
2899  * Writes unicast address list to the RAR table.
2900  * Returns: -ENOMEM on failure/insufficient address space
2901  *                0 on no addresses written
2902  *                X on writing X addresses to the RAR table
2903  **/
2904 static int igb_write_uc_addr_list(struct net_device *netdev)
2905 {
2906         struct igb_adapter *adapter = netdev_priv(netdev);
2907         struct e1000_hw *hw = &adapter->hw;
2908         unsigned int vfn = adapter->vfs_allocated_count;
2909         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2910         int count = 0;
2911
2912         /* return ENOMEM indicating insufficient memory for addresses */
2913         if (netdev->uc.count > rar_entries)
2914                 return -ENOMEM;
2915
2916         if (netdev->uc.count && rar_entries) {
2917                 struct netdev_hw_addr *ha;
2918                 list_for_each_entry(ha, &netdev->uc.list, list) {
2919                         if (!rar_entries)
2920                                 break;
2921                         igb_rar_set_qsel(adapter, ha->addr,
2922                                          rar_entries--,
2923                                          vfn);
2924                         count++;
2925                 }
2926         }
2927         /* write the addresses in reverse order to avoid write combining */
2928         for (; rar_entries > 0 ; rar_entries--) {
2929                 wr32(E1000_RAH(rar_entries), 0);
2930                 wr32(E1000_RAL(rar_entries), 0);
2931         }
2932         wrfl();
2933
2934         return count;
2935 }
2936
2937 /**
2938  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2939  * @netdev: network interface device structure
2940  *
2941  * The set_rx_mode entry point is called whenever the unicast or multicast
2942  * address lists or the network interface flags are updated.  This routine is
2943  * responsible for configuring the hardware for proper unicast, multicast,
2944  * promiscuous mode, and all-multi behavior.
2945  **/
2946 static void igb_set_rx_mode(struct net_device *netdev)
2947 {
2948         struct igb_adapter *adapter = netdev_priv(netdev);
2949         struct e1000_hw *hw = &adapter->hw;
2950         unsigned int vfn = adapter->vfs_allocated_count;
2951         u32 rctl, vmolr = 0;
2952         int count;
2953
2954         /* Check for Promiscuous and All Multicast modes */
2955         rctl = rd32(E1000_RCTL);
2956
2957         /* clear the effected bits */
2958         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2959
2960         if (netdev->flags & IFF_PROMISC) {
2961                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2962                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2963         } else {
2964                 if (netdev->flags & IFF_ALLMULTI) {
2965                         rctl |= E1000_RCTL_MPE;
2966                         vmolr |= E1000_VMOLR_MPME;
2967                 } else {
2968                         /*
2969                          * Write addresses to the MTA, if the attempt fails
2970                          * then we should just turn on promiscous mode so
2971                          * that we can at least receive multicast traffic
2972                          */
2973                         count = igb_write_mc_addr_list(netdev);
2974                         if (count < 0) {
2975                                 rctl |= E1000_RCTL_MPE;
2976                                 vmolr |= E1000_VMOLR_MPME;
2977                         } else if (count) {
2978                                 vmolr |= E1000_VMOLR_ROMPE;
2979                         }
2980                 }
2981                 /*
2982                  * Write addresses to available RAR registers, if there is not
2983                  * sufficient space to store all the addresses then enable
2984                  * unicast promiscous mode
2985                  */
2986                 count = igb_write_uc_addr_list(netdev);
2987                 if (count < 0) {
2988                         rctl |= E1000_RCTL_UPE;
2989                         vmolr |= E1000_VMOLR_ROPE;
2990                 }
2991                 rctl |= E1000_RCTL_VFE;
2992         }
2993         wr32(E1000_RCTL, rctl);
2994
2995         /*
2996          * In order to support SR-IOV and eventually VMDq it is necessary to set
2997          * the VMOLR to enable the appropriate modes.  Without this workaround
2998          * we will have issues with VLAN tag stripping not being done for frames
2999          * that are only arriving because we are the default pool
3000          */
3001         if (hw->mac.type < e1000_82576)
3002                 return;
3003
3004         vmolr |= rd32(E1000_VMOLR(vfn)) &
3005                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3006         wr32(E1000_VMOLR(vfn), vmolr);
3007         igb_restore_vf_multicasts(adapter);
3008 }
3009
3010 /* Need to wait a few seconds after link up to get diagnostic information from
3011  * the phy */
3012 static void igb_update_phy_info(unsigned long data)
3013 {
3014         struct igb_adapter *adapter = (struct igb_adapter *) data;
3015         igb_get_phy_info(&adapter->hw);
3016 }
3017
3018 /**
3019  * igb_has_link - check shared code for link and determine up/down
3020  * @adapter: pointer to driver private info
3021  **/
3022 static bool igb_has_link(struct igb_adapter *adapter)
3023 {
3024         struct e1000_hw *hw = &adapter->hw;
3025         bool link_active = false;
3026         s32 ret_val = 0;
3027
3028         /* get_link_status is set on LSC (link status) interrupt or
3029          * rx sequence error interrupt.  get_link_status will stay
3030          * false until the e1000_check_for_link establishes link
3031          * for copper adapters ONLY
3032          */
3033         switch (hw->phy.media_type) {
3034         case e1000_media_type_copper:
3035                 if (hw->mac.get_link_status) {
3036                         ret_val = hw->mac.ops.check_for_link(hw);
3037                         link_active = !hw->mac.get_link_status;
3038                 } else {
3039                         link_active = true;
3040                 }
3041                 break;
3042         case e1000_media_type_internal_serdes:
3043                 ret_val = hw->mac.ops.check_for_link(hw);
3044                 link_active = hw->mac.serdes_has_link;
3045                 break;
3046         default:
3047         case e1000_media_type_unknown:
3048                 break;
3049         }
3050
3051         return link_active;
3052 }
3053
3054 /**
3055  * igb_watchdog - Timer Call-back
3056  * @data: pointer to adapter cast into an unsigned long
3057  **/
3058 static void igb_watchdog(unsigned long data)
3059 {
3060         struct igb_adapter *adapter = (struct igb_adapter *)data;
3061         /* Do the rest outside of interrupt context */
3062         schedule_work(&adapter->watchdog_task);
3063 }
3064
3065 static void igb_watchdog_task(struct work_struct *work)
3066 {
3067         struct igb_adapter *adapter = container_of(work,
3068                                                    struct igb_adapter,
3069                                                    watchdog_task);
3070         struct e1000_hw *hw = &adapter->hw;
3071         struct net_device *netdev = adapter->netdev;
3072         u32 link;
3073         int i;
3074
3075         link = igb_has_link(adapter);
3076         if (link) {
3077                 if (!netif_carrier_ok(netdev)) {
3078                         u32 ctrl;
3079                         hw->mac.ops.get_speed_and_duplex(hw,
3080                                                          &adapter->link_speed,
3081                                                          &adapter->link_duplex);
3082
3083                         ctrl = rd32(E1000_CTRL);
3084                         /* Links status message must follow this format */
3085                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3086                                  "Flow Control: %s\n",
3087                                netdev->name,
3088                                adapter->link_speed,
3089                                adapter->link_duplex == FULL_DUPLEX ?
3090                                  "Full Duplex" : "Half Duplex",
3091                                ((ctrl & E1000_CTRL_TFCE) &&
3092                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3093                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
3094                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
3095
3096                         /* tweak tx_queue_len according to speed/duplex and
3097                          * adjust the timeout factor */
3098                         netdev->tx_queue_len = adapter->tx_queue_len;
3099                         adapter->tx_timeout_factor = 1;
3100                         switch (adapter->link_speed) {
3101                         case SPEED_10:
3102                                 netdev->tx_queue_len = 10;
3103                                 adapter->tx_timeout_factor = 14;
3104                                 break;
3105                         case SPEED_100:
3106                                 netdev->tx_queue_len = 100;
3107                                 /* maybe add some timeout factor ? */
3108                                 break;
3109                         }
3110
3111                         netif_carrier_on(netdev);
3112
3113                         igb_ping_all_vfs(adapter);
3114
3115                         /* link state has changed, schedule phy info update */
3116                         if (!test_bit(__IGB_DOWN, &adapter->state))
3117                                 mod_timer(&adapter->phy_info_timer,
3118                                           round_jiffies(jiffies + 2 * HZ));
3119                 }
3120         } else {
3121                 if (netif_carrier_ok(netdev)) {
3122                         adapter->link_speed = 0;
3123                         adapter->link_duplex = 0;
3124                         /* Links status message must follow this format */
3125                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3126                                netdev->name);
3127                         netif_carrier_off(netdev);
3128
3129                         igb_ping_all_vfs(adapter);
3130
3131                         /* link state has changed, schedule phy info update */
3132                         if (!test_bit(__IGB_DOWN, &adapter->state))
3133                                 mod_timer(&adapter->phy_info_timer,
3134                                           round_jiffies(jiffies + 2 * HZ));
3135                 }
3136         }
3137
3138         igb_update_stats(adapter);
3139         igb_update_adaptive(hw);
3140
3141         for (i = 0; i < adapter->num_tx_queues; i++) {
3142                 struct igb_ring *tx_ring = &adapter->tx_ring[i];
3143                 if (!netif_carrier_ok(netdev)) {
3144                         /* We've lost link, so the controller stops DMA,
3145                          * but we've got queued Tx work that's never going
3146                          * to get done, so reset controller to flush Tx.
3147                          * (Do the reset outside of interrupt context). */
3148                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3149                                 adapter->tx_timeout_count++;
3150                                 schedule_work(&adapter->reset_task);
3151                                 /* return immediately since reset is imminent */
3152                                 return;
3153                         }
3154                 }
3155
3156                 /* Force detection of hung controller every watchdog period */
3157                 tx_ring->detect_tx_hung = true;
3158         }
3159
3160         /* Cause software interrupt to ensure rx ring is cleaned */
3161         if (adapter->msix_entries) {
3162                 u32 eics = 0;
3163                 for (i = 0; i < adapter->num_q_vectors; i++) {
3164                         struct igb_q_vector *q_vector = adapter->q_vector[i];
3165                         eics |= q_vector->eims_value;
3166                 }
3167                 wr32(E1000_EICS, eics);
3168         } else {
3169                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3170         }
3171
3172         /* Reset the timer */
3173         if (!test_bit(__IGB_DOWN, &adapter->state))
3174                 mod_timer(&adapter->watchdog_timer,
3175                           round_jiffies(jiffies + 2 * HZ));
3176 }
3177
3178 enum latency_range {
3179         lowest_latency = 0,
3180         low_latency = 1,
3181         bulk_latency = 2,
3182         latency_invalid = 255
3183 };
3184
3185 /**
3186  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3187  *
3188  *      Stores a new ITR value based on strictly on packet size.  This
3189  *      algorithm is less sophisticated than that used in igb_update_itr,
3190  *      due to the difficulty of synchronizing statistics across multiple
3191  *      receive rings.  The divisors and thresholds used by this fuction
3192  *      were determined based on theoretical maximum wire speed and testing
3193  *      data, in order to minimize response time while increasing bulk
3194  *      throughput.
3195  *      This functionality is controlled by the InterruptThrottleRate module
3196  *      parameter (see igb_param.c)
3197  *      NOTE:  This function is called only when operating in a multiqueue
3198  *             receive environment.
3199  * @q_vector: pointer to q_vector
3200  **/
3201 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3202 {
3203         int new_val = q_vector->itr_val;
3204         int avg_wire_size = 0;
3205         struct igb_adapter *adapter = q_vector->adapter;
3206
3207         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3208          * ints/sec - ITR timer value of 120 ticks.
3209          */
3210         if (adapter->link_speed != SPEED_1000) {
3211                 new_val = 976;
3212                 goto set_itr_val;
3213         }
3214
3215         if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3216                 struct igb_ring *ring = q_vector->rx_ring;
3217                 avg_wire_size = ring->total_bytes / ring->total_packets;
3218         }
3219
3220         if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3221                 struct igb_ring *ring = q_vector->tx_ring;
3222                 avg_wire_size = max_t(u32, avg_wire_size,
3223                                       (ring->total_bytes /
3224                                        ring->total_packets));
3225         }
3226
3227         /* if avg_wire_size isn't set no work was done */
3228         if (!avg_wire_size)
3229                 goto clear_counts;
3230
3231         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3232         avg_wire_size += 24;
3233
3234         /* Don't starve jumbo frames */
3235         avg_wire_size = min(avg_wire_size, 3000);
3236
3237         /* Give a little boost to mid-size frames */
3238         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3239                 new_val = avg_wire_size / 3;
3240         else
3241                 new_val = avg_wire_size / 2;
3242
3243 set_itr_val:
3244         if (new_val != q_vector->itr_val) {
3245                 q_vector->itr_val = new_val;
3246                 q_vector->set_itr = 1;
3247         }
3248 clear_counts:
3249         if (q_vector->rx_ring) {
3250                 q_vector->rx_ring->total_bytes = 0;
3251                 q_vector->rx_ring->total_packets = 0;
3252         }
3253         if (q_vector->tx_ring) {
3254                 q_vector->tx_ring->total_bytes = 0;
3255                 q_vector->tx_ring->total_packets = 0;
3256         }
3257 }
3258
3259 /**
3260  * igb_update_itr - update the dynamic ITR value based on statistics
3261  *      Stores a new ITR value based on packets and byte
3262  *      counts during the last interrupt.  The advantage of per interrupt
3263  *      computation is faster updates and more accurate ITR for the current
3264  *      traffic pattern.  Constants in this function were computed
3265  *      based on theoretical maximum wire speed and thresholds were set based
3266  *      on testing data as well as attempting to minimize response time
3267  *      while increasing bulk throughput.
3268  *      this functionality is controlled by the InterruptThrottleRate module
3269  *      parameter (see igb_param.c)
3270  *      NOTE:  These calculations are only valid when operating in a single-
3271  *             queue environment.
3272  * @adapter: pointer to adapter
3273  * @itr_setting: current q_vector->itr_val
3274  * @packets: the number of packets during this measurement interval
3275  * @bytes: the number of bytes during this measurement interval
3276  **/
3277 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3278                                    int packets, int bytes)
3279 {
3280         unsigned int retval = itr_setting;
3281
3282         if (packets == 0)
3283                 goto update_itr_done;
3284
3285         switch (itr_setting) {
3286         case lowest_latency:
3287                 /* handle TSO and jumbo frames */
3288                 if (bytes/packets > 8000)
3289                         retval = bulk_latency;
3290                 else if ((packets < 5) && (bytes > 512))
3291                         retval = low_latency;
3292                 break;
3293         case low_latency:  /* 50 usec aka 20000 ints/s */
3294                 if (bytes > 10000) {
3295                         /* this if handles the TSO accounting */
3296                         if (bytes/packets > 8000) {
3297                                 retval = bulk_latency;
3298                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3299                                 retval = bulk_latency;
3300                         } else if ((packets > 35)) {
3301                                 retval = lowest_latency;
3302                         }
3303                 } else if (bytes/packets > 2000) {
3304                         retval = bulk_latency;
3305                 } else if (packets <= 2 && bytes < 512) {
3306                         retval = lowest_latency;
3307                 }
3308                 break;
3309         case bulk_latency: /* 250 usec aka 4000 ints/s */
3310                 if (bytes > 25000) {
3311                         if (packets > 35)
3312                                 retval = low_latency;
3313                 } else if (bytes < 1500) {
3314                         retval = low_latency;
3315                 }
3316                 break;
3317         }
3318
3319 update_itr_done:
3320         return retval;
3321 }
3322
3323 static void igb_set_itr(struct igb_adapter *adapter)
3324 {
3325         struct igb_q_vector *q_vector = adapter->q_vector[0];
3326         u16 current_itr;
3327         u32 new_itr = q_vector->itr_val;
3328
3329         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3330         if (adapter->link_speed != SPEED_1000) {
3331                 current_itr = 0;
3332                 new_itr = 4000;
3333                 goto set_itr_now;
3334         }
3335
3336         adapter->rx_itr = igb_update_itr(adapter,
3337                                     adapter->rx_itr,
3338                                     adapter->rx_ring->total_packets,
3339                                     adapter->rx_ring->total_bytes);
3340
3341         adapter->tx_itr = igb_update_itr(adapter,
3342                                     adapter->tx_itr,
3343                                     adapter->tx_ring->total_packets,
3344                                     adapter->tx_ring->total_bytes);
3345         current_itr = max(adapter->rx_itr, adapter->tx_itr);
3346
3347         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3348         if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3349                 current_itr = low_latency;
3350
3351         switch (current_itr) {
3352         /* counts and packets in update_itr are dependent on these numbers */
3353         case lowest_latency:
3354                 new_itr = 56;  /* aka 70,000 ints/sec */
3355                 break;
3356         case low_latency:
3357                 new_itr = 196; /* aka 20,000 ints/sec */
3358                 break;
3359         case bulk_latency:
3360                 new_itr = 980; /* aka 4,000 ints/sec */
3361                 break;
3362         default:
3363                 break;
3364         }
3365
3366 set_itr_now:
3367         adapter->rx_ring->total_bytes = 0;
3368         adapter->rx_ring->total_packets = 0;
3369         adapter->tx_ring->total_bytes = 0;
3370         adapter->tx_ring->total_packets = 0;
3371
3372         if (new_itr != q_vector->itr_val) {
3373                 /* this attempts to bias the interrupt rate towards Bulk
3374                  * by adding intermediate steps when interrupt rate is
3375                  * increasing */
3376                 new_itr = new_itr > q_vector->itr_val ?
3377                              max((new_itr * q_vector->itr_val) /
3378                                  (new_itr + (q_vector->itr_val >> 2)),
3379                                  new_itr) :
3380                              new_itr;
3381                 /* Don't write the value here; it resets the adapter's
3382                  * internal timer, and causes us to delay far longer than
3383                  * we should between interrupts.  Instead, we write the ITR
3384                  * value at the beginning of the next interrupt so the timing
3385                  * ends up being correct.
3386                  */
3387                 q_vector->itr_val = new_itr;
3388                 q_vector->set_itr = 1;
3389         }
3390
3391         return;
3392 }
3393
3394 #define IGB_TX_FLAGS_CSUM               0x00000001
3395 #define IGB_TX_FLAGS_VLAN               0x00000002
3396 #define IGB_TX_FLAGS_TSO                0x00000004
3397 #define IGB_TX_FLAGS_IPV4               0x00000008
3398 #define IGB_TX_FLAGS_TSTAMP             0x00000010
3399 #define IGB_TX_FLAGS_VLAN_MASK          0xffff0000
3400 #define IGB_TX_FLAGS_VLAN_SHIFT                 16
3401
3402 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3403                               struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3404 {
3405         struct e1000_adv_tx_context_desc *context_desc;
3406         unsigned int i;
3407         int err;
3408         struct igb_buffer *buffer_info;
3409         u32 info = 0, tu_cmd = 0;
3410         u32 mss_l4len_idx, l4len;
3411         *hdr_len = 0;
3412
3413         if (skb_header_cloned(skb)) {
3414                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3415                 if (err)
3416                         return err;
3417         }
3418
3419         l4len = tcp_hdrlen(skb);
3420         *hdr_len += l4len;
3421
3422         if (skb->protocol == htons(ETH_P_IP)) {
3423                 struct iphdr *iph = ip_hdr(skb);
3424                 iph->tot_len = 0;
3425                 iph->check = 0;
3426                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3427                                                          iph->daddr, 0,
3428                                                          IPPROTO_TCP,
3429                                                          0);
3430         } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3431                 ipv6_hdr(skb)->payload_len = 0;
3432                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3433                                                        &ipv6_hdr(skb)->daddr,
3434                                                        0, IPPROTO_TCP, 0);
3435         }
3436
3437         i = tx_ring->next_to_use;
3438
3439         buffer_info = &tx_ring->buffer_info[i];
3440         context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3441         /* VLAN MACLEN IPLEN */
3442         if (tx_flags & IGB_TX_FLAGS_VLAN)
3443                 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3444         info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3445         *hdr_len += skb_network_offset(skb);
3446         info |= skb_network_header_len(skb);
3447         *hdr_len += skb_network_header_len(skb);
3448         context_desc->vlan_macip_lens = cpu_to_le32(info);
3449
3450         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3451         tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3452
3453         if (skb->protocol == htons(ETH_P_IP))
3454                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3455         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3456
3457         context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3458
3459         /* MSS L4LEN IDX */
3460         mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3461         mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3462
3463         /* For 82575, context index must be unique per ring. */
3464         if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3465                 mss_l4len_idx |= tx_ring->reg_idx << 4;
3466
3467         context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3468         context_desc->seqnum_seed = 0;
3469
3470         buffer_info->time_stamp = jiffies;
3471         buffer_info->next_to_watch = i;
3472         buffer_info->dma = 0;
3473         i++;
3474         if (i == tx_ring->count)
3475                 i = 0;
3476
3477         tx_ring->next_to_use = i;
3478
3479         return true;
3480 }
3481
3482 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3483                                    struct sk_buff *skb, u32 tx_flags)
3484 {
3485         struct e1000_adv_tx_context_desc *context_desc;
3486         struct pci_dev *pdev = tx_ring->pdev;
3487         struct igb_buffer *buffer_info;
3488         u32 info = 0, tu_cmd = 0;
3489         unsigned int i;
3490
3491         if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3492             (tx_flags & IGB_TX_FLAGS_VLAN)) {
3493                 i = tx_ring->next_to_use;
3494                 buffer_info = &tx_ring->buffer_info[i];
3495                 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3496
3497                 if (tx_flags & IGB_TX_FLAGS_VLAN)
3498                         info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3499
3500                 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3501                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3502                         info |= skb_network_header_len(skb);
3503
3504                 context_desc->vlan_macip_lens = cpu_to_le32(info);
3505
3506                 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3507
3508                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3509                         __be16 protocol;
3510
3511                         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3512                                 const struct vlan_ethhdr *vhdr =
3513                                           (const struct vlan_ethhdr*)skb->data;
3514
3515                                 protocol = vhdr->h_vlan_encapsulated_proto;
3516                         } else {
3517                                 protocol = skb->protocol;
3518                         }
3519
3520                         switch (protocol) {
3521                         case cpu_to_be16(ETH_P_IP):
3522                                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3523                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3524                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3525                                 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3526                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3527                                 break;
3528                         case cpu_to_be16(ETH_P_IPV6):
3529                                 /* XXX what about other V6 headers?? */
3530                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3531                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3532                                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3533                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3534                                 break;
3535                         default:
3536                                 if (unlikely(net_ratelimit()))
3537                                         dev_warn(&pdev->dev,
3538                                             "partial checksum but proto=%x!\n",
3539                                             skb->protocol);
3540                                 break;
3541                         }
3542                 }
3543
3544                 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3545                 context_desc->seqnum_seed = 0;
3546                 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3547                         context_desc->mss_l4len_idx =
3548                                 cpu_to_le32(tx_ring->reg_idx << 4);
3549
3550                 buffer_info->time_stamp = jiffies;
3551                 buffer_info->next_to_watch = i;
3552                 buffer_info->dma = 0;
3553
3554                 i++;
3555                 if (i == tx_ring->count)
3556                         i = 0;
3557                 tx_ring->next_to_use = i;
3558
3559                 return true;
3560         }
3561         return false;
3562 }
3563
3564 #define IGB_MAX_TXD_PWR 16
3565 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
3566
3567 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3568                                  unsigned int first)
3569 {
3570         struct igb_buffer *buffer_info;
3571         struct pci_dev *pdev = tx_ring->pdev;
3572         unsigned int len = skb_headlen(skb);
3573         unsigned int count = 0, i;
3574         unsigned int f;
3575
3576         i = tx_ring->next_to_use;
3577
3578         buffer_info = &tx_ring->buffer_info[i];
3579         BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3580         buffer_info->length = len;
3581         /* set time_stamp *before* dma to help avoid a possible race */
3582         buffer_info->time_stamp = jiffies;
3583         buffer_info->next_to_watch = i;
3584         buffer_info->dma = pci_map_single(pdev, skb->data, len,
3585                                           PCI_DMA_TODEVICE);
3586         if (pci_dma_mapping_error(pdev, buffer_info->dma))
3587                 goto dma_error;
3588
3589         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3590                 struct skb_frag_struct *frag;
3591
3592                 i++;
3593                 if (i == tx_ring->count)
3594                         i = 0;
3595
3596                 frag = &skb_shinfo(skb)->frags[f];
3597                 len = frag->size;
3598
3599                 buffer_info = &tx_ring->buffer_info[i];
3600                 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3601                 buffer_info->length = len;
3602                 buffer_info->time_stamp = jiffies;
3603                 buffer_info->next_to_watch = i;
3604                 buffer_info->mapped_as_page = true;
3605                 buffer_info->dma = pci_map_page(pdev,
3606                                                 frag->page,
3607                                                 frag->page_offset,
3608                                                 len,
3609                                                 PCI_DMA_TODEVICE);
3610                 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3611                         goto dma_error;
3612
3613                 count++;
3614         }
3615
3616         tx_ring->buffer_info[i].skb = skb;
3617         tx_ring->buffer_info[first].next_to_watch = i;
3618
3619         return ++count;
3620
3621 dma_error:
3622         dev_err(&pdev->dev, "TX DMA map failed\n");
3623
3624         /* clear timestamp and dma mappings for failed buffer_info mapping */
3625         buffer_info->dma = 0;
3626         buffer_info->time_stamp = 0;
3627         buffer_info->length = 0;
3628         buffer_info->next_to_watch = 0;
3629         buffer_info->mapped_as_page = false;
3630         count--;
3631
3632         /* clear timestamp and dma mappings for remaining portion of packet */
3633         while (count >= 0) {
3634                 count--;
3635                 i--;
3636                 if (i < 0)
3637                         i += tx_ring->count;
3638                 buffer_info = &tx_ring->buffer_info[i];
3639                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3640         }
3641
3642         return 0;
3643 }
3644
3645 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3646                                     int tx_flags, int count, u32 paylen,
3647                                     u8 hdr_len)
3648 {
3649         union e1000_adv_tx_desc *tx_desc;
3650         struct igb_buffer *buffer_info;
3651         u32 olinfo_status = 0, cmd_type_len;
3652         unsigned int i = tx_ring->next_to_use;
3653
3654         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3655                         E1000_ADVTXD_DCMD_DEXT);
3656
3657         if (tx_flags & IGB_TX_FLAGS_VLAN)
3658                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3659
3660         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3661                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3662
3663         if (tx_flags & IGB_TX_FLAGS_TSO) {
3664                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3665
3666                 /* insert tcp checksum */
3667                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3668
3669                 /* insert ip checksum */
3670                 if (tx_flags & IGB_TX_FLAGS_IPV4)
3671                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3672
3673         } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3674                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3675         }
3676
3677         if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3678             (tx_flags & (IGB_TX_FLAGS_CSUM |
3679                          IGB_TX_FLAGS_TSO |
3680                          IGB_TX_FLAGS_VLAN)))
3681                 olinfo_status |= tx_ring->reg_idx << 4;
3682
3683         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3684
3685         do {
3686                 buffer_info = &tx_ring->buffer_info[i];
3687                 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3688                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3689                 tx_desc->read.cmd_type_len =
3690                         cpu_to_le32(cmd_type_len | buffer_info->length);
3691                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3692                 count--;
3693                 i++;
3694                 if (i == tx_ring->count)
3695                         i = 0;
3696         } while (count > 0);
3697
3698         tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
3699         /* Force memory writes to complete before letting h/w
3700          * know there are new descriptors to fetch.  (Only
3701          * applicable for weak-ordered memory model archs,
3702          * such as IA-64). */
3703         wmb();
3704
3705         tx_ring->next_to_use = i;
3706         writel(i, tx_ring->tail);
3707         /* we need this if more than one processor can write to our tail
3708          * at a time, it syncronizes IO on IA64/Altix systems */
3709         mmiowb();
3710 }
3711
3712 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3713 {
3714         struct net_device *netdev = tx_ring->netdev;
3715
3716         netif_stop_subqueue(netdev, tx_ring->queue_index);
3717
3718         /* Herbert's original patch had:
3719          *  smp_mb__after_netif_stop_queue();
3720          * but since that doesn't exist yet, just open code it. */
3721         smp_mb();
3722
3723         /* We need to check again in a case another CPU has just
3724          * made room available. */
3725         if (igb_desc_unused(tx_ring) < size)
3726                 return -EBUSY;
3727
3728         /* A reprieve! */
3729         netif_wake_subqueue(netdev, tx_ring->queue_index);
3730         tx_ring->tx_stats.restart_queue++;
3731         return 0;
3732 }
3733
3734 static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3735 {
3736         if (igb_desc_unused(tx_ring) >= size)
3737                 return 0;
3738         return __igb_maybe_stop_tx(tx_ring, size);
3739 }
3740
3741 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3742                                     struct igb_ring *tx_ring)
3743 {
3744         struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
3745         unsigned int first;
3746         unsigned int tx_flags = 0;
3747         u8 hdr_len = 0;
3748         int tso = 0, count;
3749         union skb_shared_tx *shtx = skb_tx(skb);
3750
3751         /* need: 1 descriptor per page,
3752          *       + 2 desc gap to keep tail from touching head,
3753          *       + 1 desc for skb->data,
3754          *       + 1 desc for context descriptor,
3755          * otherwise try next time */
3756         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
3757                 /* this is a hard error */
3758                 return NETDEV_TX_BUSY;
3759         }
3760
3761         if (unlikely(shtx->hardware)) {
3762                 shtx->in_progress = 1;
3763                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
3764         }
3765
3766         if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
3767                 tx_flags |= IGB_TX_FLAGS_VLAN;
3768                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3769         }
3770
3771         if (skb->protocol == htons(ETH_P_IP))
3772                 tx_flags |= IGB_TX_FLAGS_IPV4;
3773
3774         first = tx_ring->next_to_use;
3775         if (skb_is_gso(skb)) {
3776                 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3777
3778                 if (tso < 0) {
3779                         dev_kfree_skb_any(skb);
3780                         return NETDEV_TX_OK;
3781                 }
3782         }
3783
3784         if (tso)
3785                 tx_flags |= IGB_TX_FLAGS_TSO;
3786         else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
3787                  (skb->ip_summed == CHECKSUM_PARTIAL))
3788                 tx_flags |= IGB_TX_FLAGS_CSUM;
3789
3790         /*
3791          * count reflects descriptors mapped, if 0 or less then mapping error
3792          * has occured and we need to rewind the descriptor queue
3793          */
3794         count = igb_tx_map_adv(tx_ring, skb, first);
3795         if (!count) {
3796                 dev_kfree_skb_any(skb);
3797                 tx_ring->buffer_info[first].time_stamp = 0;
3798                 tx_ring->next_to_use = first;
3799                 return NETDEV_TX_OK;
3800         }
3801
3802         igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3803
3804         /* Make sure there is space in the ring for the next send. */
3805         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
3806
3807         return NETDEV_TX_OK;
3808 }
3809
3810 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3811                                       struct net_device *netdev)
3812 {
3813         struct igb_adapter *adapter = netdev_priv(netdev);
3814         struct igb_ring *tx_ring;
3815         int r_idx = 0;
3816
3817         if (test_bit(__IGB_DOWN, &adapter->state)) {
3818                 dev_kfree_skb_any(skb);
3819                 return NETDEV_TX_OK;
3820         }
3821
3822         if (skb->len <= 0) {
3823                 dev_kfree_skb_any(skb);
3824                 return NETDEV_TX_OK;
3825         }
3826
3827         r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
3828         tx_ring = adapter->multi_tx_table[r_idx];
3829
3830         /* This goes back to the question of how to logically map a tx queue
3831          * to a flow.  Right now, performance is impacted slightly negatively
3832          * if using multiple tx queues.  If the stack breaks away from a
3833          * single qdisc implementation, we can look at this again. */
3834         return igb_xmit_frame_ring_adv(skb, tx_ring);
3835 }
3836
3837 /**
3838  * igb_tx_timeout - Respond to a Tx Hang
3839  * @netdev: network interface device structure
3840  **/
3841 static void igb_tx_timeout(struct net_device *netdev)
3842 {
3843         struct igb_adapter *adapter = netdev_priv(netdev);
3844         struct e1000_hw *hw = &adapter->hw;
3845
3846         /* Do the reset outside of interrupt context */
3847         adapter->tx_timeout_count++;
3848
3849         if (hw->mac.type == e1000_82580)
3850                 hw->dev_spec._82575.global_device_reset = true;
3851
3852         schedule_work(&adapter->reset_task);
3853         wr32(E1000_EICS,
3854              (adapter->eims_enable_mask & ~adapter->eims_other));
3855 }
3856
3857 static void igb_reset_task(struct work_struct *work)
3858 {
3859         struct igb_adapter *adapter;
3860         adapter = container_of(work, struct igb_adapter, reset_task);
3861
3862         igb_reinit_locked(adapter);
3863 }
3864
3865 /**
3866  * igb_get_stats - Get System Network Statistics
3867  * @netdev: network interface device structure
3868  *
3869  * Returns the address of the device statistics structure.
3870  * The statistics are actually updated from the timer callback.
3871  **/
3872 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
3873 {
3874         /* only return the current stats */
3875         return &netdev->stats;
3876 }
3877
3878 /**
3879  * igb_change_mtu - Change the Maximum Transfer Unit
3880  * @netdev: network interface device structure
3881  * @new_mtu: new value for maximum frame size
3882  *
3883  * Returns 0 on success, negative on failure
3884  **/
3885 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3886 {
3887         struct igb_adapter *adapter = netdev_priv(netdev);
3888         struct pci_dev *pdev = adapter->pdev;
3889         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3890         u32 rx_buffer_len, i;
3891
3892         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3893                 dev_err(&pdev->dev, "Invalid MTU setting\n");
3894                 return -EINVAL;
3895         }
3896
3897         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3898                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
3899                 return -EINVAL;
3900         }
3901
3902         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3903                 msleep(1);
3904
3905         /* igb_down has a dependency on max_frame_size */
3906         adapter->max_frame_size = max_frame;
3907
3908         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3909          * means we reserve 2 more, this pushes us to allocate from the next
3910          * larger slab size.
3911          * i.e. RXBUFFER_2048 --> size-4096 slab
3912          */
3913
3914         if (max_frame <= IGB_RXBUFFER_1024)
3915                 rx_buffer_len = IGB_RXBUFFER_1024;
3916         else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
3917                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3918         else
3919                 rx_buffer_len = IGB_RXBUFFER_128;
3920
3921         if (netif_running(netdev))
3922                 igb_down(adapter);
3923
3924         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
3925                  netdev->mtu, new_mtu);
3926         netdev->mtu = new_mtu;
3927
3928         for (i = 0; i < adapter->num_rx_queues; i++)
3929                 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3930
3931         if (netif_running(netdev))
3932                 igb_up(adapter);
3933         else
3934                 igb_reset(adapter);
3935
3936         clear_bit(__IGB_RESETTING, &adapter->state);
3937
3938         return 0;
3939 }
3940
3941 /**
3942  * igb_update_stats - Update the board statistics counters
3943  * @adapter: board private structure
3944  **/
3945
3946 void igb_update_stats(struct igb_adapter *adapter)
3947 {
3948         struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
3949         struct e1000_hw *hw = &adapter->hw;
3950         struct pci_dev *pdev = adapter->pdev;
3951         u32 rnbc;
3952         u16 phy_tmp;
3953         int i;
3954         u64 bytes, packets;
3955
3956 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3957
3958         /*
3959          * Prevent stats update while adapter is being reset, or if the pci
3960          * connection is down.
3961          */
3962         if (adapter->link_speed == 0)
3963                 return;
3964         if (pci_channel_offline(pdev))
3965                 return;
3966
3967         bytes = 0;
3968         packets = 0;
3969         for (i = 0; i < adapter->num_rx_queues; i++) {
3970                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3971                 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3972                 net_stats->rx_fifo_errors += rqdpc_tmp;
3973                 bytes += adapter->rx_ring[i].rx_stats.bytes;
3974                 packets += adapter->rx_ring[i].rx_stats.packets;
3975         }
3976
3977         net_stats->rx_bytes = bytes;
3978         net_stats->rx_packets = packets;
3979
3980         bytes = 0;
3981         packets = 0;
3982         for (i = 0; i < adapter->num_tx_queues; i++) {
3983                 bytes += adapter->tx_ring[i].tx_stats.bytes;
3984                 packets += adapter->tx_ring[i].tx_stats.packets;
3985         }
3986         net_stats->tx_bytes = bytes;
3987         net_stats->tx_packets = packets;
3988
3989         /* read stats registers */
3990         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3991         adapter->stats.gprc += rd32(E1000_GPRC);
3992         adapter->stats.gorc += rd32(E1000_GORCL);
3993         rd32(E1000_GORCH); /* clear GORCL */
3994         adapter->stats.bprc += rd32(E1000_BPRC);
3995         adapter->stats.mprc += rd32(E1000_MPRC);
3996         adapter->stats.roc += rd32(E1000_ROC);
3997
3998         adapter->stats.prc64 += rd32(E1000_PRC64);
3999         adapter->stats.prc127 += rd32(E1000_PRC127);
4000         adapter->stats.prc255 += rd32(E1000_PRC255);
4001         adapter->stats.prc511 += rd32(E1000_PRC511);
4002         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4003         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4004         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4005         adapter->stats.sec += rd32(E1000_SEC);
4006
4007         adapter->stats.mpc += rd32(E1000_MPC);
4008         adapter->stats.scc += rd32(E1000_SCC);
4009         adapter->stats.ecol += rd32(E1000_ECOL);
4010         adapter->stats.mcc += rd32(E1000_MCC);
4011         adapter->stats.latecol += rd32(E1000_LATECOL);
4012         adapter->stats.dc += rd32(E1000_DC);
4013         adapter->stats.rlec += rd32(E1000_RLEC);
4014         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4015         adapter->stats.xontxc += rd32(E1000_XONTXC);
4016         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4017         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4018         adapter->stats.fcruc += rd32(E1000_FCRUC);
4019         adapter->stats.gptc += rd32(E1000_GPTC);
4020         adapter->stats.gotc += rd32(E1000_GOTCL);
4021         rd32(E1000_GOTCH); /* clear GOTCL */
4022         rnbc = rd32(E1000_RNBC);
4023         adapter->stats.rnbc += rnbc;
4024         net_stats->rx_fifo_errors += rnbc;
4025         adapter->stats.ruc += rd32(E1000_RUC);
4026         adapter->stats.rfc += rd32(E1000_RFC);
4027         adapter->stats.rjc += rd32(E1000_RJC);
4028         adapter->stats.tor += rd32(E1000_TORH);
4029         adapter->stats.tot += rd32(E1000_TOTH);
4030         adapter->stats.tpr += rd32(E1000_TPR);
4031
4032         adapter->stats.ptc64 += rd32(E1000_PTC64);
4033         adapter->stats.ptc127 += rd32(E1000_PTC127);
4034         adapter->stats.ptc255 += rd32(E1000_PTC255);
4035         adapter->stats.ptc511 += rd32(E1000_PTC511);
4036         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4037         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4038
4039         adapter->stats.mptc += rd32(E1000_MPTC);
4040         adapter->stats.bptc += rd32(E1000_BPTC);
4041
4042         /* used for adaptive IFS */
4043         hw->mac.tx_packet_delta = rd32(E1000_TPT);
4044         adapter->stats.tpt += hw->mac.tx_packet_delta;
4045         hw->mac.collision_delta = rd32(E1000_COLC);
4046         adapter->stats.colc += hw->mac.collision_delta;
4047
4048         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4049         adapter->stats.rxerrc += rd32(E1000_RXERRC);
4050         adapter->stats.tncrs += rd32(E1000_TNCRS);
4051         adapter->stats.tsctc += rd32(E1000_TSCTC);
4052         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4053
4054         adapter->stats.iac += rd32(E1000_IAC);
4055         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4056         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4057         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4058         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4059         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4060         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4061         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4062         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4063
4064         /* Fill out the OS statistics structure */
4065         net_stats->multicast = adapter->stats.mprc;
4066         net_stats->collisions = adapter->stats.colc;
4067
4068         /* Rx Errors */
4069
4070         /* RLEC on some newer hardware can be incorrect so build
4071          * our own version based on RUC and ROC */
4072         net_stats->rx_errors = adapter->stats.rxerrc +
4073                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4074                 adapter->stats.ruc + adapter->stats.roc +
4075                 adapter->stats.cexterr;
4076         net_stats->rx_length_errors = adapter->stats.ruc +
4077                                       adapter->stats.roc;
4078         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4079         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4080         net_stats->rx_missed_errors = adapter->stats.mpc;
4081
4082         /* Tx Errors */
4083         net_stats->tx_errors = adapter->stats.ecol +
4084                                adapter->stats.latecol;
4085         net_stats->tx_aborted_errors = adapter->stats.ecol;
4086         net_stats->tx_window_errors = adapter->stats.latecol;
4087         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4088
4089         /* Tx Dropped needs to be maintained elsewhere */
4090
4091         /* Phy Stats */
4092         if (hw->phy.media_type == e1000_media_type_copper) {
4093                 if ((adapter->link_speed == SPEED_1000) &&
4094                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4095                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4096                         adapter->phy_stats.idle_errors += phy_tmp;
4097                 }
4098         }
4099
4100         /* Management Stats */
4101         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4102         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4103         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4104 }
4105
4106 static irqreturn_t igb_msix_other(int irq, void *data)
4107 {
4108         struct igb_adapter *adapter = data;
4109         struct e1000_hw *hw = &adapter->hw;
4110         u32 icr = rd32(E1000_ICR);
4111         /* reading ICR causes bit 31 of EICR to be cleared */
4112
4113         if (icr & E1000_ICR_DOUTSYNC) {
4114                 /* HW is reporting DMA is out of sync */
4115                 adapter->stats.doosync++;
4116         }
4117
4118         /* Check for a mailbox event */
4119         if (icr & E1000_ICR_VMMB)
4120                 igb_msg_task(adapter);
4121
4122         if (icr & E1000_ICR_LSC) {
4123                 hw->mac.get_link_status = 1;
4124                 /* guard against interrupt when we're going down */
4125                 if (!test_bit(__IGB_DOWN, &adapter->state))
4126                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4127         }
4128
4129         if (adapter->vfs_allocated_count)
4130                 wr32(E1000_IMS, E1000_IMS_LSC |
4131                                 E1000_IMS_VMMB |
4132                                 E1000_IMS_DOUTSYNC);
4133         else
4134                 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4135         wr32(E1000_EIMS, adapter->eims_other);
4136
4137         return IRQ_HANDLED;
4138 }
4139
4140 static void igb_write_itr(struct igb_q_vector *q_vector)
4141 {
4142         u32 itr_val = q_vector->itr_val & 0x7FFC;
4143
4144         if (!q_vector->set_itr)
4145                 return;
4146
4147         if (!itr_val)
4148                 itr_val = 0x4;
4149
4150         if (q_vector->itr_shift)
4151                 itr_val |= itr_val << q_vector->itr_shift;
4152         else
4153                 itr_val |= 0x8000000;
4154
4155         writel(itr_val, q_vector->itr_register);
4156         q_vector->set_itr = 0;
4157 }
4158
4159 static irqreturn_t igb_msix_ring(int irq, void *data)
4160 {
4161         struct igb_q_vector *q_vector = data;
4162
4163         /* Write the ITR value calculated from the previous interrupt. */
4164         igb_write_itr(q_vector);
4165
4166         napi_schedule(&q_vector->napi);
4167
4168         return IRQ_HANDLED;
4169 }
4170
4171 #ifdef CONFIG_IGB_DCA
4172 static void igb_update_dca(struct igb_q_vector *q_vector)
4173 {
4174         struct igb_adapter *adapter = q_vector->adapter;
4175         struct e1000_hw *hw = &adapter->hw;
4176         int cpu = get_cpu();
4177
4178         if (q_vector->cpu == cpu)
4179                 goto out_no_update;
4180
4181         if (q_vector->tx_ring) {
4182                 int q = q_vector->tx_ring->reg_idx;
4183                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4184                 if (hw->mac.type == e1000_82575) {
4185                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4186                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4187                 } else {
4188                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4189                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4190                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4191                 }
4192                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4193                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4194         }
4195         if (q_vector->rx_ring) {
4196                 int q = q_vector->rx_ring->reg_idx;
4197                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4198                 if (hw->mac.type == e1000_82575) {
4199                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4200                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4201                 } else {
4202                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4203                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4204                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4205                 }
4206                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4207                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4208                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4209                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4210         }
4211         q_vector->cpu = cpu;
4212 out_no_update:
4213         put_cpu();
4214 }
4215
4216 static void igb_setup_dca(struct igb_adapter *adapter)
4217 {
4218         struct e1000_hw *hw = &adapter->hw;
4219         int i;
4220
4221         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4222                 return;
4223
4224         /* Always use CB2 mode, difference is masked in the CB driver. */
4225         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4226
4227         for (i = 0; i < adapter->num_q_vectors; i++) {
4228                 struct igb_q_vector *q_vector = adapter->q_vector[i];
4229                 q_vector->cpu = -1;
4230                 igb_update_dca(q_vector);
4231         }
4232 }
4233
4234 static int __igb_notify_dca(struct device *dev, void *data)
4235 {
4236         struct net_device *netdev = dev_get_drvdata(dev);
4237         struct igb_adapter *adapter = netdev_priv(netdev);
4238         struct pci_dev *pdev = adapter->pdev;
4239         struct e1000_hw *hw = &adapter->hw;
4240         unsigned long event = *(unsigned long *)data;
4241
4242         switch (event) {
4243         case DCA_PROVIDER_ADD:
4244                 /* if already enabled, don't do it again */
4245                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4246                         break;
4247                 if (dca_add_requester(dev) == 0) {
4248                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4249                         dev_info(&pdev->dev, "DCA enabled\n");
4250                         igb_setup_dca(adapter);
4251                         break;
4252                 }
4253                 /* Fall Through since DCA is disabled. */
4254         case DCA_PROVIDER_REMOVE:
4255                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4256                         /* without this a class_device is left
4257                          * hanging around in the sysfs model */
4258                         dca_remove_requester(dev);
4259                         dev_info(&pdev->dev, "DCA disabled\n");
4260                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4261                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4262                 }
4263                 break;
4264         }
4265
4266         return 0;
4267 }
4268
4269 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4270                           void *p)
4271 {
4272         int ret_val;
4273
4274         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4275                                          __igb_notify_dca);
4276
4277         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4278 }
4279 #endif /* CONFIG_IGB_DCA */
4280
4281 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4282 {
4283         struct e1000_hw *hw = &adapter->hw;
4284         u32 ping;
4285         int i;
4286
4287         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4288                 ping = E1000_PF_CONTROL_MSG;
4289                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4290                         ping |= E1000_VT_MSGTYPE_CTS;
4291                 igb_write_mbx(hw, &ping, 1, i);
4292         }
4293 }
4294
4295 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4296 {
4297         struct e1000_hw *hw = &adapter->hw;
4298         u32 vmolr = rd32(E1000_VMOLR(vf));
4299         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4300
4301         vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4302                             IGB_VF_FLAG_MULTI_PROMISC);
4303         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4304
4305         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4306                 vmolr |= E1000_VMOLR_MPME;
4307                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4308         } else {
4309                 /*
4310                  * if we have hashes and we are clearing a multicast promisc
4311                  * flag we need to write the hashes to the MTA as this step
4312                  * was previously skipped
4313                  */
4314                 if (vf_data->num_vf_mc_hashes > 30) {
4315                         vmolr |= E1000_VMOLR_MPME;
4316                 } else if (vf_data->num_vf_mc_hashes) {
4317                         int j;
4318                         vmolr |= E1000_VMOLR_ROMPE;
4319                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4320                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4321                 }
4322         }
4323
4324         wr32(E1000_VMOLR(vf), vmolr);
4325
4326         /* there are flags left unprocessed, likely not supported */
4327         if (*msgbuf & E1000_VT_MSGINFO_MASK)
4328                 return -EINVAL;
4329
4330         return 0;
4331
4332 }
4333
4334 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4335                                   u32 *msgbuf, u32 vf)
4336 {
4337         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4338         u16 *hash_list = (u16 *)&msgbuf[1];
4339         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4340         int i;
4341
4342         /* salt away the number of multicast addresses assigned
4343          * to this VF for later use to restore when the PF multi cast
4344          * list changes
4345          */
4346         vf_data->num_vf_mc_hashes = n;
4347
4348         /* only up to 30 hash values supported */
4349         if (n > 30)
4350                 n = 30;
4351
4352         /* store the hashes for later use */
4353         for (i = 0; i < n; i++)
4354                 vf_data->vf_mc_hashes[i] = hash_list[i];
4355
4356         /* Flush and reset the mta with the new values */
4357         igb_set_rx_mode(adapter->netdev);
4358
4359         return 0;
4360 }
4361
4362 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4363 {
4364         struct e1000_hw *hw = &adapter->hw;
4365         struct vf_data_storage *vf_data;
4366         int i, j;
4367
4368         for (i = 0; i < adapter->vfs_allocated_count; i++) {
4369                 u32 vmolr = rd32(E1000_VMOLR(i));
4370                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4371
4372                 vf_data = &adapter->vf_data[i];
4373
4374                 if ((vf_data->num_vf_mc_hashes > 30) ||
4375                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4376                         vmolr |= E1000_VMOLR_MPME;
4377                 } else if (vf_data->num_vf_mc_hashes) {
4378                         vmolr |= E1000_VMOLR_ROMPE;
4379                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4380                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4381                 }
4382                 wr32(E1000_VMOLR(i), vmolr);
4383         }
4384 }
4385
4386 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4387 {
4388         struct e1000_hw *hw = &adapter->hw;
4389         u32 pool_mask, reg, vid;
4390         int i;
4391
4392         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4393
4394         /* Find the vlan filter for this id */
4395         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4396                 reg = rd32(E1000_VLVF(i));
4397
4398                 /* remove the vf from the pool */
4399                 reg &= ~pool_mask;
4400
4401                 /* if pool is empty then remove entry from vfta */
4402                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4403                     (reg & E1000_VLVF_VLANID_ENABLE)) {
4404                         reg = 0;
4405                         vid = reg & E1000_VLVF_VLANID_MASK;
4406                         igb_vfta_set(hw, vid, false);
4407                 }
4408
4409                 wr32(E1000_VLVF(i), reg);
4410         }
4411
4412         adapter->vf_data[vf].vlans_enabled = 0;
4413 }
4414
4415 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4416 {
4417         struct e1000_hw *hw = &adapter->hw;
4418         u32 reg, i;
4419
4420         /* The vlvf table only exists on 82576 hardware and newer */
4421         if (hw->mac.type < e1000_82576)
4422                 return -1;
4423
4424         /* we only need to do this if VMDq is enabled */
4425         if (!adapter->vfs_allocated_count)
4426                 return -1;
4427
4428         /* Find the vlan filter for this id */
4429         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4430                 reg = rd32(E1000_VLVF(i));
4431                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4432                     vid == (reg & E1000_VLVF_VLANID_MASK))
4433                         break;
4434         }
4435
4436         if (add) {
4437                 if (i == E1000_VLVF_ARRAY_SIZE) {
4438                         /* Did not find a matching VLAN ID entry that was
4439                          * enabled.  Search for a free filter entry, i.e.
4440                          * one without the enable bit set
4441                          */
4442                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4443                                 reg = rd32(E1000_VLVF(i));
4444                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4445                                         break;
4446                         }
4447                 }
4448                 if (i < E1000_VLVF_ARRAY_SIZE) {
4449                         /* Found an enabled/available entry */
4450                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4451
4452                         /* if !enabled we need to set this up in vfta */
4453                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4454                                 /* add VID to filter table */
4455                                 igb_vfta_set(hw, vid, true);
4456                                 reg |= E1000_VLVF_VLANID_ENABLE;
4457                         }
4458                         reg &= ~E1000_VLVF_VLANID_MASK;
4459                         reg |= vid;
4460                         wr32(E1000_VLVF(i), reg);
4461
4462                         /* do not modify RLPML for PF devices */
4463                         if (vf >= adapter->vfs_allocated_count)
4464                                 return 0;
4465
4466                         if (!adapter->vf_data[vf].vlans_enabled) {
4467                                 u32 size;
4468                                 reg = rd32(E1000_VMOLR(vf));
4469                                 size = reg & E1000_VMOLR_RLPML_MASK;
4470                                 size += 4;
4471                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4472                                 reg |= size;
4473                                 wr32(E1000_VMOLR(vf), reg);
4474                         }
4475
4476                         adapter->vf_data[vf].vlans_enabled++;
4477                         return 0;
4478                 }
4479         } else {
4480                 if (i < E1000_VLVF_ARRAY_SIZE) {
4481                         /* remove vf from the pool */
4482                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4483                         /* if pool is empty then remove entry from vfta */
4484                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4485                                 reg = 0;
4486                                 igb_vfta_set(hw, vid, false);
4487                         }
4488                         wr32(E1000_VLVF(i), reg);
4489
4490                         /* do not modify RLPML for PF devices */
4491                         if (vf >= adapter->vfs_allocated_count)
4492                                 return 0;
4493
4494                         adapter->vf_data[vf].vlans_enabled--;
4495                         if (!adapter->vf_data[vf].vlans_enabled) {
4496                                 u32 size;
4497                                 reg = rd32(E1000_VMOLR(vf));
4498                                 size = reg & E1000_VMOLR_RLPML_MASK;
4499                                 size -= 4;
4500                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4501                                 reg |= size;
4502                                 wr32(E1000_VMOLR(vf), reg);
4503                         }
4504                         return 0;
4505                 }
4506         }
4507         return -1;
4508 }
4509
4510 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4511 {
4512         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4513         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4514
4515         return igb_vlvf_set(adapter, vid, add, vf);
4516 }
4517
4518 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4519 {
4520         /* clear all flags */
4521         adapter->vf_data[vf].flags = 0;
4522         adapter->vf_data[vf].last_nack = jiffies;
4523
4524         /* reset offloads to defaults */
4525         igb_set_vmolr(adapter, vf);
4526
4527         /* reset vlans for device */
4528         igb_clear_vf_vfta(adapter, vf);
4529
4530         /* reset multicast table array for vf */
4531         adapter->vf_data[vf].num_vf_mc_hashes = 0;
4532
4533         /* Flush and reset the mta with the new values */
4534         igb_set_rx_mode(adapter->netdev);
4535 }
4536
4537 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4538 {
4539         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4540
4541         /* generate a new mac address as we were hotplug removed/added */
4542         random_ether_addr(vf_mac);
4543
4544         /* process remaining reset events */
4545         igb_vf_reset(adapter, vf);
4546 }
4547
4548 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4549 {
4550         struct e1000_hw *hw = &adapter->hw;
4551         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4552         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4553         u32 reg, msgbuf[3];
4554         u8 *addr = (u8 *)(&msgbuf[1]);
4555
4556         /* process all the same items cleared in a function level reset */
4557         igb_vf_reset(adapter, vf);
4558
4559         /* set vf mac address */
4560         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4561
4562         /* enable transmit and receive for vf */
4563         reg = rd32(E1000_VFTE);
4564         wr32(E1000_VFTE, reg | (1 << vf));
4565         reg = rd32(E1000_VFRE);
4566         wr32(E1000_VFRE, reg | (1 << vf));
4567
4568         adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4569
4570         /* reply to reset with ack and vf mac address */
4571         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4572         memcpy(addr, vf_mac, 6);
4573         igb_write_mbx(hw, msgbuf, 3, vf);
4574 }
4575
4576 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4577 {
4578         unsigned char *addr = (char *)&msg[1];
4579         int err = -1;
4580
4581         if (is_valid_ether_addr(addr))
4582                 err = igb_set_vf_mac(adapter, vf, addr);
4583
4584         return err;
4585 }
4586
4587 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4588 {
4589         struct e1000_hw *hw = &adapter->hw;
4590         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4591         u32 msg = E1000_VT_MSGTYPE_NACK;
4592
4593         /* if device isn't clear to send it shouldn't be reading either */
4594         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4595             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4596                 igb_write_mbx(hw, &msg, 1, vf);
4597                 vf_data->last_nack = jiffies;
4598         }
4599 }
4600
4601 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4602 {
4603         struct pci_dev *pdev = adapter->pdev;
4604         u32 msgbuf[E1000_VFMAILBOX_SIZE];
4605         struct e1000_hw *hw = &adapter->hw;
4606         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4607         s32 retval;
4608
4609         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4610
4611         if (retval) {
4612                 /* if receive failed revoke VF CTS stats and restart init */
4613                 dev_err(&pdev->dev, "Error receiving message from VF\n");
4614                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4615                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4616                         return;
4617                 goto out;
4618         }
4619
4620         /* this is a message we already processed, do nothing */
4621         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4622                 return;
4623
4624         /*
4625          * until the vf completes a reset it should not be
4626          * allowed to start any configuration.
4627          */
4628
4629         if (msgbuf[0] == E1000_VF_RESET) {
4630                 igb_vf_reset_msg(adapter, vf);
4631                 return;
4632         }
4633
4634         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4635                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4636                         return;
4637                 retval = -1;
4638                 goto out;
4639         }
4640
4641         switch ((msgbuf[0] & 0xFFFF)) {
4642         case E1000_VF_SET_MAC_ADDR:
4643                 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4644                 break;
4645         case E1000_VF_SET_PROMISC:
4646                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4647                 break;
4648         case E1000_VF_SET_MULTICAST:
4649                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4650                 break;
4651         case E1000_VF_SET_LPE:
4652                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4653                 break;
4654         case E1000_VF_SET_VLAN:
4655                 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4656                 break;
4657         default:
4658                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4659                 retval = -1;
4660                 break;
4661         }
4662
4663         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4664 out:
4665         /* notify the VF of the results of what it sent us */
4666         if (retval)
4667                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4668         else
4669                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4670
4671         igb_write_mbx(hw, msgbuf, 1, vf);
4672 }
4673
4674 static void igb_msg_task(struct igb_adapter *adapter)
4675 {
4676         struct e1000_hw *hw = &adapter->hw;
4677         u32 vf;
4678
4679         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4680                 /* process any reset requests */
4681                 if (!igb_check_for_rst(hw, vf))
4682                         igb_vf_reset_event(adapter, vf);
4683
4684                 /* process any messages pending */
4685                 if (!igb_check_for_msg(hw, vf))
4686                         igb_rcv_msg_from_vf(adapter, vf);
4687
4688                 /* process any acks */
4689                 if (!igb_check_for_ack(hw, vf))
4690                         igb_rcv_ack_from_vf(adapter, vf);
4691         }
4692 }
4693
4694 /**
4695  *  igb_set_uta - Set unicast filter table address
4696  *  @adapter: board private structure
4697  *
4698  *  The unicast table address is a register array of 32-bit registers.
4699  *  The table is meant to be used in a way similar to how the MTA is used
4700  *  however due to certain limitations in the hardware it is necessary to
4701  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4702  *  enable bit to allow vlan tag stripping when promiscous mode is enabled
4703  **/
4704 static void igb_set_uta(struct igb_adapter *adapter)
4705 {
4706         struct e1000_hw *hw = &adapter->hw;
4707         int i;
4708
4709         /* The UTA table only exists on 82576 hardware and newer */
4710         if (hw->mac.type < e1000_82576)
4711                 return;
4712
4713         /* we only need to do this if VMDq is enabled */
4714         if (!adapter->vfs_allocated_count)
4715                 return;
4716
4717         for (i = 0; i < hw->mac.uta_reg_count; i++)
4718                 array_wr32(E1000_UTA, i, ~0);
4719 }
4720
4721 /**
4722  * igb_intr_msi - Interrupt Handler
4723  * @irq: interrupt number
4724  * @data: pointer to a network interface device structure
4725  **/
4726 static irqreturn_t igb_intr_msi(int irq, void *data)
4727 {
4728         struct igb_adapter *adapter = data;
4729         struct igb_q_vector *q_vector = adapter->q_vector[0];
4730         struct e1000_hw *hw = &adapter->hw;
4731         /* read ICR disables interrupts using IAM */
4732         u32 icr = rd32(E1000_ICR);
4733
4734         igb_write_itr(q_vector);
4735
4736         if (icr & E1000_ICR_DOUTSYNC) {
4737                 /* HW is reporting DMA is out of sync */
4738                 adapter->stats.doosync++;
4739         }
4740
4741         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4742                 hw->mac.get_link_status = 1;
4743                 if (!test_bit(__IGB_DOWN, &adapter->state))
4744                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4745         }
4746
4747         napi_schedule(&q_vector->napi);
4748
4749         return IRQ_HANDLED;
4750 }
4751
4752 /**
4753  * igb_intr - Legacy Interrupt Handler
4754  * @irq: interrupt number
4755  * @data: pointer to a network interface device structure
4756  **/
4757 static irqreturn_t igb_intr(int irq, void *data)
4758 {
4759         struct igb_adapter *adapter = data;
4760         struct igb_q_vector *q_vector = adapter->q_vector[0];
4761         struct e1000_hw *hw = &adapter->hw;
4762         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
4763          * need for the IMC write */
4764         u32 icr = rd32(E1000_ICR);
4765         if (!icr)
4766                 return IRQ_NONE;  /* Not our interrupt */
4767
4768         igb_write_itr(q_vector);
4769
4770         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4771          * not set, then the adapter didn't send an interrupt */
4772         if (!(icr & E1000_ICR_INT_ASSERTED))
4773                 return IRQ_NONE;
4774
4775         if (icr & E1000_ICR_DOUTSYNC) {
4776                 /* HW is reporting DMA is out of sync */
4777                 adapter->stats.doosync++;
4778         }
4779
4780         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4781                 hw->mac.get_link_status = 1;
4782                 /* guard against interrupt when we're going down */
4783                 if (!test_bit(__IGB_DOWN, &adapter->state))
4784                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4785         }
4786
4787         napi_schedule(&q_vector->napi);
4788
4789         return IRQ_HANDLED;
4790 }
4791
4792 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
4793 {
4794         struct igb_adapter *adapter = q_vector->adapter;
4795         struct e1000_hw *hw = &adapter->hw;
4796
4797         if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4798             (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
4799                 if (!adapter->msix_entries)
4800                         igb_set_itr(adapter);
4801                 else
4802                         igb_update_ring_itr(q_vector);
4803         }
4804
4805         if (!test_bit(__IGB_DOWN, &adapter->state)) {
4806                 if (adapter->msix_entries)
4807                         wr32(E1000_EIMS, q_vector->eims_value);
4808                 else
4809                         igb_irq_enable(adapter);
4810         }
4811 }
4812
4813 /**
4814  * igb_poll - NAPI Rx polling callback
4815  * @napi: napi polling structure
4816  * @budget: count of how many packets we should handle
4817  **/
4818 static int igb_poll(struct napi_struct *napi, int budget)
4819 {
4820         struct igb_q_vector *q_vector = container_of(napi,
4821                                                      struct igb_q_vector,
4822                                                      napi);
4823         int tx_clean_complete = 1, work_done = 0;
4824
4825 #ifdef CONFIG_IGB_DCA
4826         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4827                 igb_update_dca(q_vector);
4828 #endif
4829         if (q_vector->tx_ring)
4830                 tx_clean_complete = igb_clean_tx_irq(q_vector);
4831
4832         if (q_vector->rx_ring)
4833                 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4834
4835         if (!tx_clean_complete)
4836                 work_done = budget;
4837
4838         /* If not enough Rx work done, exit the polling mode */
4839         if (work_done < budget) {
4840                 napi_complete(napi);
4841                 igb_ring_irq_enable(q_vector);
4842         }
4843
4844         return work_done;
4845 }
4846
4847 /**
4848  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
4849  * @adapter: board private structure
4850  * @shhwtstamps: timestamp structure to update
4851  * @regval: unsigned 64bit system time value.
4852  *
4853  * We need to convert the system time value stored in the RX/TXSTMP registers
4854  * into a hwtstamp which can be used by the upper level timestamping functions
4855  */
4856 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4857                                    struct skb_shared_hwtstamps *shhwtstamps,
4858                                    u64 regval)
4859 {
4860         u64 ns;
4861
4862         /*
4863          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4864          * 24 to match clock shift we setup earlier.
4865          */
4866         if (adapter->hw.mac.type == e1000_82580)
4867                 regval <<= IGB_82580_TSYNC_SHIFT;
4868
4869         ns = timecounter_cyc2time(&adapter->clock, regval);
4870         timecompare_update(&adapter->compare, ns);
4871         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4872         shhwtstamps->hwtstamp = ns_to_ktime(ns);
4873         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4874 }
4875
4876 /**
4877  * igb_tx_hwtstamp - utility function which checks for TX time stamp
4878  * @q_vector: pointer to q_vector containing needed info
4879  * @skb: packet that was just sent
4880  *
4881  * If we were asked to do hardware stamping and such a time stamp is
4882  * available, then it must have been for this skb here because we only
4883  * allow only one such packet into the queue.
4884  */
4885 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
4886 {
4887         struct igb_adapter *adapter = q_vector->adapter;
4888         union skb_shared_tx *shtx = skb_tx(skb);
4889         struct e1000_hw *hw = &adapter->hw;
4890         struct skb_shared_hwtstamps shhwtstamps;
4891         u64 regval;
4892
4893         /* if skb does not support hw timestamp or TX stamp not valid exit */
4894         if (likely(!shtx->hardware) ||
4895             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4896                 return;
4897
4898         regval = rd32(E1000_TXSTMPL);
4899         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4900
4901         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4902         skb_tstamp_tx(skb, &shhwtstamps);
4903 }
4904
4905 /**
4906  * igb_clean_tx_irq - Reclaim resources after transmit completes
4907  * @q_vector: pointer to q_vector containing needed info
4908  * returns true if ring is completely cleaned
4909  **/
4910 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
4911 {
4912         struct igb_adapter *adapter = q_vector->adapter;
4913         struct igb_ring *tx_ring = q_vector->tx_ring;
4914         struct net_device *netdev = tx_ring->netdev;
4915         struct e1000_hw *hw = &adapter->hw;
4916         struct igb_buffer *buffer_info;
4917         struct sk_buff *skb;
4918         union e1000_adv_tx_desc *tx_desc, *eop_desc;
4919         unsigned int total_bytes = 0, total_packets = 0;
4920         unsigned int i, eop, count = 0;
4921         bool cleaned = false;
4922
4923         i = tx_ring->next_to_clean;
4924         eop = tx_ring->buffer_info[i].next_to_watch;
4925         eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4926
4927         while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4928                (count < tx_ring->count)) {
4929                 for (cleaned = false; !cleaned; count++) {
4930                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4931                         buffer_info = &tx_ring->buffer_info[i];
4932                         cleaned = (i == eop);
4933                         skb = buffer_info->skb;
4934
4935                         if (skb) {
4936                                 unsigned int segs, bytecount;
4937                                 /* gso_segs is currently only valid for tcp */
4938                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
4939                                 /* multiply data chunks by size of headers */
4940                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
4941                                             skb->len;
4942                                 total_packets += segs;
4943                                 total_bytes += bytecount;
4944
4945                                 igb_tx_hwtstamp(q_vector, skb);
4946                         }
4947
4948                         igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4949                         tx_desc->wb.status = 0;
4950
4951                         i++;
4952                         if (i == tx_ring->count)
4953                                 i = 0;
4954                 }
4955                 eop = tx_ring->buffer_info[i].next_to_watch;
4956                 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4957         }
4958
4959         tx_ring->next_to_clean = i;
4960
4961         if (unlikely(count &&
4962                      netif_carrier_ok(netdev) &&
4963                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
4964                 /* Make sure that anybody stopping the queue after this
4965                  * sees the new next_to_clean.
4966                  */
4967                 smp_mb();
4968                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4969                     !(test_bit(__IGB_DOWN, &adapter->state))) {
4970                         netif_wake_subqueue(netdev, tx_ring->queue_index);
4971                         tx_ring->tx_stats.restart_queue++;
4972                 }
4973         }
4974
4975         if (tx_ring->detect_tx_hung) {
4976                 /* Detect a transmit hang in hardware, this serializes the
4977                  * check with the clearing of time_stamp and movement of i */
4978                 tx_ring->detect_tx_hung = false;
4979                 if (tx_ring->buffer_info[i].time_stamp &&
4980                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4981                                (adapter->tx_timeout_factor * HZ)) &&
4982                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
4983
4984                         /* detected Tx unit hang */
4985                         dev_err(&tx_ring->pdev->dev,
4986                                 "Detected Tx Unit Hang\n"
4987                                 "  Tx Queue             <%d>\n"
4988                                 "  TDH                  <%x>\n"
4989                                 "  TDT                  <%x>\n"
4990                                 "  next_to_use          <%x>\n"
4991                                 "  next_to_clean        <%x>\n"
4992                                 "buffer_info[next_to_clean]\n"
4993                                 "  time_stamp           <%lx>\n"
4994                                 "  next_to_watch        <%x>\n"
4995                                 "  jiffies              <%lx>\n"
4996                                 "  desc.status          <%x>\n",
4997                                 tx_ring->queue_index,
4998                                 readl(tx_ring->head),
4999                                 readl(tx_ring->tail),
5000                                 tx_ring->next_to_use,
5001                                 tx_ring->next_to_clean,
5002                                 tx_ring->buffer_info[eop].time_stamp,
5003                                 eop,
5004                                 jiffies,
5005                                 eop_desc->wb.status);
5006                         netif_stop_subqueue(netdev, tx_ring->queue_index);
5007                 }
5008         }
5009         tx_ring->total_bytes += total_bytes;
5010         tx_ring->total_packets += total_packets;
5011         tx_ring->tx_stats.bytes += total_bytes;
5012         tx_ring->tx_stats.packets += total_packets;
5013         return (count < tx_ring->count);
5014 }
5015
5016 /**
5017  * igb_receive_skb - helper function to handle rx indications
5018  * @q_vector: structure containing interrupt and ring information
5019  * @skb: packet to send up
5020  * @vlan_tag: vlan tag for packet
5021  **/
5022 static void igb_receive_skb(struct igb_q_vector *q_vector,
5023                             struct sk_buff *skb,
5024                             u16 vlan_tag)
5025 {
5026         struct igb_adapter *adapter = q_vector->adapter;
5027
5028         if (vlan_tag)
5029                 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5030                                  vlan_tag, skb);
5031         else
5032                 napi_gro_receive(&q_vector->napi, skb);
5033 }
5034
5035 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5036                                        u32 status_err, struct sk_buff *skb)
5037 {
5038         skb->ip_summed = CHECKSUM_NONE;
5039
5040         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5041         if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5042              (status_err & E1000_RXD_STAT_IXSM))
5043                 return;
5044
5045         /* TCP/UDP checksum error bit is set */
5046         if (status_err &
5047             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5048                 /*
5049                  * work around errata with sctp packets where the TCPE aka
5050                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5051                  * packets, (aka let the stack check the crc32c)
5052                  */
5053                 if ((skb->len == 60) &&
5054                     (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5055                         ring->rx_stats.csum_err++;
5056
5057                 /* let the stack verify checksum errors */
5058                 return;
5059         }
5060         /* It must be a TCP or UDP packet with a valid checksum */
5061         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5062                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5063
5064         dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
5065 }
5066
5067 static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5068                                    struct sk_buff *skb)
5069 {
5070         struct igb_adapter *adapter = q_vector->adapter;
5071         struct e1000_hw *hw = &adapter->hw;
5072         u64 regval;
5073
5074         /*
5075          * If this bit is set, then the RX registers contain the time stamp. No
5076          * other packet will be time stamped until we read these registers, so
5077          * read the registers to make them available again. Because only one
5078          * packet can be time stamped at a time, we know that the register
5079          * values must belong to this one here and therefore we don't need to
5080          * compare any of the additional attributes stored for it.
5081          *
5082          * If nothing went wrong, then it should have a skb_shared_tx that we
5083          * can turn into a skb_shared_hwtstamps.
5084          */
5085         if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5086                 return;
5087         if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5088                 return;
5089
5090         regval = rd32(E1000_RXSTMPL);
5091         regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5092
5093         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5094 }
5095 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5096                                union e1000_adv_rx_desc *rx_desc)
5097 {
5098         /* HW will not DMA in data larger than the given buffer, even if it
5099          * parses the (NFS, of course) header to be larger.  In that case, it
5100          * fills the header buffer and spills the rest into the page.
5101          */
5102         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5103                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5104         if (hlen > rx_ring->rx_buffer_len)
5105                 hlen = rx_ring->rx_buffer_len;
5106         return hlen;
5107 }
5108
5109 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5110                                  int *work_done, int budget)
5111 {
5112         struct igb_ring *rx_ring = q_vector->rx_ring;
5113         struct net_device *netdev = rx_ring->netdev;
5114         struct pci_dev *pdev = rx_ring->pdev;
5115         union e1000_adv_rx_desc *rx_desc , *next_rxd;
5116         struct igb_buffer *buffer_info , *next_buffer;
5117         struct sk_buff *skb;
5118         bool cleaned = false;
5119         int cleaned_count = 0;
5120         int current_node = numa_node_id();
5121         unsigned int total_bytes = 0, total_packets = 0;
5122         unsigned int i;
5123         u32 staterr;
5124         u16 length;
5125         u16 vlan_tag;
5126
5127         i = rx_ring->next_to_clean;
5128         buffer_info = &rx_ring->buffer_info[i];
5129         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5130         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5131
5132         while (staterr & E1000_RXD_STAT_DD) {
5133                 if (*work_done >= budget)
5134                         break;
5135                 (*work_done)++;
5136
5137                 skb = buffer_info->skb;
5138                 prefetch(skb->data - NET_IP_ALIGN);
5139                 buffer_info->skb = NULL;
5140
5141                 i++;
5142                 if (i == rx_ring->count)
5143                         i = 0;
5144
5145                 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5146                 prefetch(next_rxd);
5147                 next_buffer = &rx_ring->buffer_info[i];
5148
5149                 length = le16_to_cpu(rx_desc->wb.upper.length);
5150                 cleaned = true;
5151                 cleaned_count++;
5152
5153                 if (buffer_info->dma) {
5154                         pci_unmap_single(pdev, buffer_info->dma,
5155                                          rx_ring->rx_buffer_len,
5156                                          PCI_DMA_FROMDEVICE);
5157                         buffer_info->dma = 0;
5158                         if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5159                                 skb_put(skb, length);
5160                                 goto send_up;
5161                         }
5162                         skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5163                 }
5164
5165                 if (length) {
5166                         pci_unmap_page(pdev, buffer_info->page_dma,
5167                                        PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
5168                         buffer_info->page_dma = 0;
5169
5170                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5171                                                 buffer_info->page,
5172                                                 buffer_info->page_offset,
5173                                                 length);
5174
5175                         if ((page_count(buffer_info->page) != 1) ||
5176                             (page_to_nid(buffer_info->page) != current_node))
5177                                 buffer_info->page = NULL;
5178                         else
5179                                 get_page(buffer_info->page);
5180
5181                         skb->len += length;
5182                         skb->data_len += length;
5183                         skb->truesize += length;
5184                 }
5185
5186                 if (!(staterr & E1000_RXD_STAT_EOP)) {
5187                         buffer_info->skb = next_buffer->skb;
5188                         buffer_info->dma = next_buffer->dma;
5189                         next_buffer->skb = skb;
5190                         next_buffer->dma = 0;
5191                         goto next_desc;
5192                 }
5193 send_up:
5194                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5195                         dev_kfree_skb_irq(skb);
5196                         goto next_desc;
5197                 }
5198
5199                 igb_rx_hwtstamp(q_vector, staterr, skb);
5200                 total_bytes += skb->len;
5201                 total_packets++;
5202
5203                 igb_rx_checksum_adv(rx_ring, staterr, skb);
5204
5205                 skb->protocol = eth_type_trans(skb, netdev);
5206                 skb_record_rx_queue(skb, rx_ring->queue_index);
5207
5208                 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5209                             le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5210
5211                 igb_receive_skb(q_vector, skb, vlan_tag);
5212
5213 next_desc:
5214                 rx_desc->wb.upper.status_error = 0;
5215
5216                 /* return some buffers to hardware, one at a time is too slow */
5217                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5218                         igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5219                         cleaned_count = 0;
5220                 }
5221
5222                 /* use prefetched values */
5223                 rx_desc = next_rxd;
5224                 buffer_info = next_buffer;
5225                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5226         }
5227
5228         rx_ring->next_to_clean = i;
5229         cleaned_count = igb_desc_unused(rx_ring);
5230
5231         if (cleaned_count)
5232                 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5233
5234         rx_ring->total_packets += total_packets;
5235         rx_ring->total_bytes += total_bytes;
5236         rx_ring->rx_stats.packets += total_packets;
5237         rx_ring->rx_stats.bytes += total_bytes;
5238         return cleaned;
5239 }
5240
5241 /**
5242  * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5243  * @adapter: address of board private structure
5244  **/
5245 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5246 {
5247         struct net_device *netdev = rx_ring->netdev;
5248         union e1000_adv_rx_desc *rx_desc;
5249         struct igb_buffer *buffer_info;
5250         struct sk_buff *skb;
5251         unsigned int i;
5252         int bufsz;
5253
5254         i = rx_ring->next_to_use;
5255         buffer_info = &rx_ring->buffer_info[i];
5256
5257         bufsz = rx_ring->rx_buffer_len;
5258
5259         while (cleaned_count--) {
5260                 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5261
5262                 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5263                         if (!buffer_info->page) {
5264                                 buffer_info->page = netdev_alloc_page(netdev);
5265                                 if (!buffer_info->page) {
5266                                         rx_ring->rx_stats.alloc_failed++;
5267                                         goto no_buffers;
5268                                 }
5269                                 buffer_info->page_offset = 0;
5270                         } else {
5271                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
5272                         }
5273                         buffer_info->page_dma =
5274                                 pci_map_page(rx_ring->pdev, buffer_info->page,
5275                                              buffer_info->page_offset,
5276                                              PAGE_SIZE / 2,
5277                                              PCI_DMA_FROMDEVICE);
5278                         if (pci_dma_mapping_error(rx_ring->pdev,
5279                                                   buffer_info->page_dma)) {
5280                                 buffer_info->page_dma = 0;
5281                                 rx_ring->rx_stats.alloc_failed++;
5282                                 goto no_buffers;
5283                         }
5284                 }
5285
5286                 skb = buffer_info->skb;
5287                 if (!skb) {
5288                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5289                         if (!skb) {
5290                                 rx_ring->rx_stats.alloc_failed++;
5291                                 goto no_buffers;
5292                         }
5293
5294                         buffer_info->skb = skb;
5295                 }
5296                 if (!buffer_info->dma) {
5297                         buffer_info->dma = pci_map_single(rx_ring->pdev,
5298                                                           skb->data,
5299                                                           bufsz,
5300                                                           PCI_DMA_FROMDEVICE);
5301                         if (pci_dma_mapping_error(rx_ring->pdev,
5302                                                   buffer_info->dma)) {
5303                                 buffer_info->dma = 0;
5304                                 rx_ring->rx_stats.alloc_failed++;
5305                                 goto no_buffers;
5306                         }
5307                 }
5308                 /* Refresh the desc even if buffer_addrs didn't change because
5309                  * each write-back erases this info. */
5310                 if (bufsz < IGB_RXBUFFER_1024) {
5311                         rx_desc->read.pkt_addr =
5312                              cpu_to_le64(buffer_info->page_dma);
5313                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5314                 } else {
5315                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5316                         rx_desc->read.hdr_addr = 0;
5317                 }
5318
5319                 i++;
5320                 if (i == rx_ring->count)
5321                         i = 0;
5322                 buffer_info = &rx_ring->buffer_info[i];
5323         }
5324
5325 no_buffers:
5326         if (rx_ring->next_to_use != i) {
5327                 rx_ring->next_to_use = i;
5328                 if (i == 0)
5329                         i = (rx_ring->count - 1);
5330                 else
5331                         i--;
5332
5333                 /* Force memory writes to complete before letting h/w
5334                  * know there are new descriptors to fetch.  (Only
5335                  * applicable for weak-ordered memory model archs,
5336                  * such as IA-64). */
5337                 wmb();
5338                 writel(i, rx_ring->tail);
5339         }
5340 }
5341
5342 /**
5343  * igb_mii_ioctl -
5344  * @netdev:
5345  * @ifreq:
5346  * @cmd:
5347  **/
5348 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5349 {
5350         struct igb_adapter *adapter = netdev_priv(netdev);
5351         struct mii_ioctl_data *data = if_mii(ifr);
5352
5353         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5354                 return -EOPNOTSUPP;
5355
5356         switch (cmd) {
5357         case SIOCGMIIPHY:
5358                 data->phy_id = adapter->hw.phy.addr;
5359                 break;
5360         case SIOCGMIIREG:
5361                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5362                                      &data->val_out))
5363                         return -EIO;
5364                 break;
5365         case SIOCSMIIREG:
5366         default:
5367                 return -EOPNOTSUPP;
5368         }
5369         return 0;
5370 }
5371
5372 /**
5373  * igb_hwtstamp_ioctl - control hardware time stamping
5374  * @netdev:
5375  * @ifreq:
5376  * @cmd:
5377  *
5378  * Outgoing time stamping can be enabled and disabled. Play nice and
5379  * disable it when requested, although it shouldn't case any overhead
5380  * when no packet needs it. At most one packet in the queue may be
5381  * marked for time stamping, otherwise it would be impossible to tell
5382  * for sure to which packet the hardware time stamp belongs.
5383  *
5384  * Incoming time stamping has to be configured via the hardware
5385  * filters. Not all combinations are supported, in particular event
5386  * type has to be specified. Matching the kind of event packet is
5387  * not supported, with the exception of "all V2 events regardless of
5388  * level 2 or 4".
5389  *
5390  **/
5391 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5392                               struct ifreq *ifr, int cmd)
5393 {
5394         struct igb_adapter *adapter = netdev_priv(netdev);
5395         struct e1000_hw *hw = &adapter->hw;
5396         struct hwtstamp_config config;
5397         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5398         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5399         u32 tsync_rx_cfg = 0;
5400         bool is_l4 = false;
5401         bool is_l2 = false;
5402         u32 regval;
5403
5404         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5405                 return -EFAULT;
5406
5407         /* reserved for future extensions */
5408         if (config.flags)
5409                 return -EINVAL;
5410
5411         switch (config.tx_type) {
5412         case HWTSTAMP_TX_OFF:
5413                 tsync_tx_ctl = 0;
5414         case HWTSTAMP_TX_ON:
5415                 break;
5416         default:
5417                 return -ERANGE;
5418         }
5419
5420         switch (config.rx_filter) {
5421         case HWTSTAMP_FILTER_NONE:
5422                 tsync_rx_ctl = 0;
5423                 break;
5424         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5425         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5426         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5427         case HWTSTAMP_FILTER_ALL:
5428                 /*
5429                  * register TSYNCRXCFG must be set, therefore it is not
5430                  * possible to time stamp both Sync and Delay_Req messages
5431                  * => fall back to time stamping all packets
5432                  */
5433                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5434                 config.rx_filter = HWTSTAMP_FILTER_ALL;
5435                 break;
5436         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5437                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5438                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5439                 is_l4 = true;
5440                 break;
5441         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5442                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5443                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5444                 is_l4 = true;
5445                 break;
5446         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5447         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5448                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5449                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5450                 is_l2 = true;
5451                 is_l4 = true;
5452                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5453                 break;
5454         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5455         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5456                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5457                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5458                 is_l2 = true;
5459                 is_l4 = true;
5460                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5461                 break;
5462         case HWTSTAMP_FILTER_PTP_V2_EVENT:
5463         case HWTSTAMP_FILTER_PTP_V2_SYNC:
5464         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5465                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5466                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5467                 is_l2 = true;
5468                 break;
5469         default:
5470                 return -ERANGE;
5471         }
5472
5473         if (hw->mac.type == e1000_82575) {
5474                 if (tsync_rx_ctl | tsync_tx_ctl)
5475                         return -EINVAL;
5476                 return 0;
5477         }
5478
5479         /* enable/disable TX */
5480         regval = rd32(E1000_TSYNCTXCTL);
5481         regval &= ~E1000_TSYNCTXCTL_ENABLED;
5482         regval |= tsync_tx_ctl;
5483         wr32(E1000_TSYNCTXCTL, regval);
5484
5485         /* enable/disable RX */
5486         regval = rd32(E1000_TSYNCRXCTL);
5487         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5488         regval |= tsync_rx_ctl;
5489         wr32(E1000_TSYNCRXCTL, regval);
5490
5491         /* define which PTP packets are time stamped */
5492         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5493
5494         /* define ethertype filter for timestamped packets */
5495         if (is_l2)
5496                 wr32(E1000_ETQF(3),
5497                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5498                                  E1000_ETQF_1588 | /* enable timestamping */
5499                                  ETH_P_1588));     /* 1588 eth protocol type */
5500         else
5501                 wr32(E1000_ETQF(3), 0);
5502
5503 #define PTP_PORT 319
5504         /* L4 Queue Filter[3]: filter by destination port and protocol */
5505         if (is_l4) {
5506                 u32 ftqf = (IPPROTO_UDP /* UDP */
5507                         | E1000_FTQF_VF_BP /* VF not compared */
5508                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5509                         | E1000_FTQF_MASK); /* mask all inputs */
5510                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5511
5512                 wr32(E1000_IMIR(3), htons(PTP_PORT));
5513                 wr32(E1000_IMIREXT(3),
5514                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5515                 if (hw->mac.type == e1000_82576) {
5516                         /* enable source port check */
5517                         wr32(E1000_SPQF(3), htons(PTP_PORT));
5518                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5519                 }
5520                 wr32(E1000_FTQF(3), ftqf);
5521         } else {
5522                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5523         }
5524         wrfl();
5525
5526         adapter->hwtstamp_config = config;
5527
5528         /* clear TX/RX time stamp registers, just to be sure */
5529         regval = rd32(E1000_TXSTMPH);
5530         regval = rd32(E1000_RXSTMPH);
5531
5532         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5533                 -EFAULT : 0;
5534 }
5535
5536 /**
5537  * igb_ioctl -
5538  * @netdev:
5539  * @ifreq:
5540  * @cmd:
5541  **/
5542 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5543 {
5544         switch (cmd) {
5545         case SIOCGMIIPHY:
5546         case SIOCGMIIREG:
5547         case SIOCSMIIREG:
5548                 return igb_mii_ioctl(netdev, ifr, cmd);
5549         case SIOCSHWTSTAMP:
5550                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5551         default:
5552                 return -EOPNOTSUPP;
5553         }
5554 }
5555
5556 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5557 {
5558         struct igb_adapter *adapter = hw->back;
5559         u16 cap_offset;
5560
5561         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5562         if (!cap_offset)
5563                 return -E1000_ERR_CONFIG;
5564
5565         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5566
5567         return 0;
5568 }
5569
5570 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5571 {
5572         struct igb_adapter *adapter = hw->back;
5573         u16 cap_offset;
5574
5575         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5576         if (!cap_offset)
5577                 return -E1000_ERR_CONFIG;
5578
5579         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5580
5581         return 0;
5582 }
5583
5584 static void igb_vlan_rx_register(struct net_device *netdev,
5585                                  struct vlan_group *grp)
5586 {
5587         struct igb_adapter *adapter = netdev_priv(netdev);
5588         struct e1000_hw *hw = &adapter->hw;
5589         u32 ctrl, rctl;
5590
5591         igb_irq_disable(adapter);
5592         adapter->vlgrp = grp;
5593
5594         if (grp) {
5595                 /* enable VLAN tag insert/strip */
5596                 ctrl = rd32(E1000_CTRL);
5597                 ctrl |= E1000_CTRL_VME;
5598                 wr32(E1000_CTRL, ctrl);
5599
5600                 /* Disable CFI check */
5601                 rctl = rd32(E1000_RCTL);
5602                 rctl &= ~E1000_RCTL_CFIEN;
5603                 wr32(E1000_RCTL, rctl);
5604         } else {
5605                 /* disable VLAN tag insert/strip */
5606                 ctrl = rd32(E1000_CTRL);
5607                 ctrl &= ~E1000_CTRL_VME;
5608                 wr32(E1000_CTRL, ctrl);
5609         }
5610
5611         igb_rlpml_set(adapter);
5612
5613         if (!test_bit(__IGB_DOWN, &adapter->state))
5614                 igb_irq_enable(adapter);
5615 }
5616
5617 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5618 {
5619         struct igb_adapter *adapter = netdev_priv(netdev);
5620         struct e1000_hw *hw = &adapter->hw;
5621         int pf_id = adapter->vfs_allocated_count;
5622
5623         /* attempt to add filter to vlvf array */
5624         igb_vlvf_set(adapter, vid, true, pf_id);
5625
5626         /* add the filter since PF can receive vlans w/o entry in vlvf */
5627         igb_vfta_set(hw, vid, true);
5628 }
5629
5630 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5631 {
5632         struct igb_adapter *adapter = netdev_priv(netdev);
5633         struct e1000_hw *hw = &adapter->hw;
5634         int pf_id = adapter->vfs_allocated_count;
5635         s32 err;
5636
5637         igb_irq_disable(adapter);
5638         vlan_group_set_device(adapter->vlgrp, vid, NULL);
5639
5640         if (!test_bit(__IGB_DOWN, &adapter->state))
5641                 igb_irq_enable(adapter);
5642
5643         /* remove vlan from VLVF table array */
5644         err = igb_vlvf_set(adapter, vid, false, pf_id);
5645
5646         /* if vid was not present in VLVF just remove it from table */
5647         if (err)
5648                 igb_vfta_set(hw, vid, false);
5649 }
5650
5651 static void igb_restore_vlan(struct igb_adapter *adapter)
5652 {
5653         igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5654
5655         if (adapter->vlgrp) {
5656                 u16 vid;
5657                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5658                         if (!vlan_group_get_device(adapter->vlgrp, vid))
5659                                 continue;
5660                         igb_vlan_rx_add_vid(adapter->netdev, vid);
5661                 }
5662         }
5663 }
5664
5665 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5666 {
5667         struct pci_dev *pdev = adapter->pdev;
5668         struct e1000_mac_info *mac = &adapter->hw.mac;
5669
5670         mac->autoneg = 0;
5671
5672         switch (spddplx) {
5673         case SPEED_10 + DUPLEX_HALF:
5674                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5675                 break;
5676         case SPEED_10 + DUPLEX_FULL:
5677                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5678                 break;
5679         case SPEED_100 + DUPLEX_HALF:
5680                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5681                 break;
5682         case SPEED_100 + DUPLEX_FULL:
5683                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5684                 break;
5685         case SPEED_1000 + DUPLEX_FULL:
5686                 mac->autoneg = 1;
5687                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5688                 break;
5689         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5690         default:
5691                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
5692                 return -EINVAL;
5693         }
5694         return 0;
5695 }
5696
5697 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
5698 {
5699         struct net_device *netdev = pci_get_drvdata(pdev);
5700         struct igb_adapter *adapter = netdev_priv(netdev);
5701         struct e1000_hw *hw = &adapter->hw;
5702         u32 ctrl, rctl, status;
5703         u32 wufc = adapter->wol;
5704 #ifdef CONFIG_PM
5705         int retval = 0;
5706 #endif
5707
5708         netif_device_detach(netdev);
5709
5710         if (netif_running(netdev))
5711                 igb_close(netdev);
5712
5713         igb_clear_interrupt_scheme(adapter);
5714
5715 #ifdef CONFIG_PM
5716         retval = pci_save_state(pdev);
5717         if (retval)
5718                 return retval;
5719 #endif
5720
5721         status = rd32(E1000_STATUS);
5722         if (status & E1000_STATUS_LU)
5723                 wufc &= ~E1000_WUFC_LNKC;
5724
5725         if (wufc) {
5726                 igb_setup_rctl(adapter);
5727                 igb_set_rx_mode(netdev);
5728
5729                 /* turn on all-multi mode if wake on multicast is enabled */
5730                 if (wufc & E1000_WUFC_MC) {
5731                         rctl = rd32(E1000_RCTL);
5732                         rctl |= E1000_RCTL_MPE;
5733                         wr32(E1000_RCTL, rctl);
5734                 }
5735
5736                 ctrl = rd32(E1000_CTRL);
5737                 /* advertise wake from D3Cold */
5738                 #define E1000_CTRL_ADVD3WUC 0x00100000
5739                 /* phy power management enable */
5740                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5741                 ctrl |= E1000_CTRL_ADVD3WUC;
5742                 wr32(E1000_CTRL, ctrl);
5743
5744                 /* Allow time for pending master requests to run */
5745                 igb_disable_pcie_master(hw);
5746
5747                 wr32(E1000_WUC, E1000_WUC_PME_EN);
5748                 wr32(E1000_WUFC, wufc);
5749         } else {
5750                 wr32(E1000_WUC, 0);
5751                 wr32(E1000_WUFC, 0);
5752         }
5753
5754         *enable_wake = wufc || adapter->en_mng_pt;
5755         if (!*enable_wake)
5756                 igb_shutdown_serdes_link_82575(hw);
5757
5758         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5759          * would have already happened in close and is redundant. */
5760         igb_release_hw_control(adapter);
5761
5762         pci_disable_device(pdev);
5763
5764         return 0;
5765 }
5766
5767 #ifdef CONFIG_PM
5768 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5769 {
5770         int retval;
5771         bool wake;
5772
5773         retval = __igb_shutdown(pdev, &wake);
5774         if (retval)
5775                 return retval;
5776
5777         if (wake) {
5778                 pci_prepare_to_sleep(pdev);
5779         } else {
5780                 pci_wake_from_d3(pdev, false);
5781                 pci_set_power_state(pdev, PCI_D3hot);
5782         }
5783
5784         return 0;
5785 }
5786
5787 static int igb_resume(struct pci_dev *pdev)
5788 {
5789         struct net_device *netdev = pci_get_drvdata(pdev);
5790         struct igb_adapter *adapter = netdev_priv(netdev);
5791         struct e1000_hw *hw = &adapter->hw;
5792         u32 err;
5793
5794         pci_set_power_state(pdev, PCI_D0);
5795         pci_restore_state(pdev);
5796
5797         err = pci_enable_device_mem(pdev);
5798         if (err) {
5799                 dev_err(&pdev->dev,
5800                         "igb: Cannot enable PCI device from suspend\n");
5801                 return err;
5802         }
5803         pci_set_master(pdev);
5804
5805         pci_enable_wake(pdev, PCI_D3hot, 0);
5806         pci_enable_wake(pdev, PCI_D3cold, 0);
5807
5808         if (igb_init_interrupt_scheme(adapter)) {
5809                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5810                 return -ENOMEM;
5811         }
5812
5813         /* e1000_power_up_phy(adapter); */
5814
5815         igb_reset(adapter);
5816
5817         /* let the f/w know that the h/w is now under the control of the
5818          * driver. */
5819         igb_get_hw_control(adapter);
5820
5821         wr32(E1000_WUS, ~0);
5822
5823         if (netif_running(netdev)) {
5824                 err = igb_open(netdev);
5825                 if (err)
5826                         return err;
5827         }
5828
5829         netif_device_attach(netdev);
5830
5831         return 0;
5832 }
5833 #endif
5834
5835 static void igb_shutdown(struct pci_dev *pdev)
5836 {
5837         bool wake;
5838
5839         __igb_shutdown(pdev, &wake);
5840
5841         if (system_state == SYSTEM_POWER_OFF) {
5842                 pci_wake_from_d3(pdev, wake);
5843                 pci_set_power_state(pdev, PCI_D3hot);
5844         }
5845 }
5846
5847 #ifdef CONFIG_NET_POLL_CONTROLLER
5848 /*
5849  * Polling 'interrupt' - used by things like netconsole to send skbs
5850  * without having to re-enable interrupts. It's not called while
5851  * the interrupt routine is executing.
5852  */
5853 static void igb_netpoll(struct net_device *netdev)
5854 {
5855         struct igb_adapter *adapter = netdev_priv(netdev);
5856         struct e1000_hw *hw = &adapter->hw;
5857         int i;
5858
5859         if (!adapter->msix_entries) {
5860                 struct igb_q_vector *q_vector = adapter->q_vector[0];
5861                 igb_irq_disable(adapter);
5862                 napi_schedule(&q_vector->napi);
5863                 return;
5864         }
5865
5866         for (i = 0; i < adapter->num_q_vectors; i++) {
5867                 struct igb_q_vector *q_vector = adapter->q_vector[i];
5868                 wr32(E1000_EIMC, q_vector->eims_value);
5869                 napi_schedule(&q_vector->napi);
5870         }
5871 }
5872 #endif /* CONFIG_NET_POLL_CONTROLLER */
5873
5874 /**
5875  * igb_io_error_detected - called when PCI error is detected
5876  * @pdev: Pointer to PCI device
5877  * @state: The current pci connection state
5878  *
5879  * This function is called after a PCI bus error affecting
5880  * this device has been detected.
5881  */
5882 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5883                                               pci_channel_state_t state)
5884 {
5885         struct net_device *netdev = pci_get_drvdata(pdev);
5886         struct igb_adapter *adapter = netdev_priv(netdev);
5887
5888         netif_device_detach(netdev);
5889
5890         if (state == pci_channel_io_perm_failure)
5891                 return PCI_ERS_RESULT_DISCONNECT;
5892
5893         if (netif_running(netdev))
5894                 igb_down(adapter);
5895         pci_disable_device(pdev);
5896
5897         /* Request a slot slot reset. */
5898         return PCI_ERS_RESULT_NEED_RESET;
5899 }
5900
5901 /**
5902  * igb_io_slot_reset - called after the pci bus has been reset.
5903  * @pdev: Pointer to PCI device
5904  *
5905  * Restart the card from scratch, as if from a cold-boot. Implementation
5906  * resembles the first-half of the igb_resume routine.
5907  */
5908 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5909 {
5910         struct net_device *netdev = pci_get_drvdata(pdev);
5911         struct igb_adapter *adapter = netdev_priv(netdev);
5912         struct e1000_hw *hw = &adapter->hw;
5913         pci_ers_result_t result;
5914         int err;
5915
5916         if (pci_enable_device_mem(pdev)) {
5917                 dev_err(&pdev->dev,
5918                         "Cannot re-enable PCI device after reset.\n");
5919                 result = PCI_ERS_RESULT_DISCONNECT;
5920         } else {
5921                 pci_set_master(pdev);
5922                 pci_restore_state(pdev);
5923
5924                 pci_enable_wake(pdev, PCI_D3hot, 0);
5925                 pci_enable_wake(pdev, PCI_D3cold, 0);
5926
5927                 igb_reset(adapter);
5928                 wr32(E1000_WUS, ~0);
5929                 result = PCI_ERS_RESULT_RECOVERED;
5930         }
5931
5932         err = pci_cleanup_aer_uncorrect_error_status(pdev);
5933         if (err) {
5934                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5935                         "failed 0x%0x\n", err);
5936                 /* non-fatal, continue */
5937         }
5938
5939         return result;
5940 }
5941
5942 /**
5943  * igb_io_resume - called when traffic can start flowing again.
5944  * @pdev: Pointer to PCI device
5945  *
5946  * This callback is called when the error recovery driver tells us that
5947  * its OK to resume normal operation. Implementation resembles the
5948  * second-half of the igb_resume routine.
5949  */
5950 static void igb_io_resume(struct pci_dev *pdev)
5951 {
5952         struct net_device *netdev = pci_get_drvdata(pdev);
5953         struct igb_adapter *adapter = netdev_priv(netdev);
5954
5955         if (netif_running(netdev)) {
5956                 if (igb_up(adapter)) {
5957                         dev_err(&pdev->dev, "igb_up failed after reset\n");
5958                         return;
5959                 }
5960         }
5961
5962         netif_device_attach(netdev);
5963
5964         /* let the f/w know that the h/w is now under the control of the
5965          * driver. */
5966         igb_get_hw_control(adapter);
5967 }
5968
5969 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5970                              u8 qsel)
5971 {
5972         u32 rar_low, rar_high;
5973         struct e1000_hw *hw = &adapter->hw;
5974
5975         /* HW expects these in little endian so we reverse the byte order
5976          * from network order (big endian) to little endian
5977          */
5978         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5979                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5980         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5981
5982         /* Indicate to hardware the Address is Valid. */
5983         rar_high |= E1000_RAH_AV;
5984
5985         if (hw->mac.type == e1000_82575)
5986                 rar_high |= E1000_RAH_POOL_1 * qsel;
5987         else
5988                 rar_high |= E1000_RAH_POOL_1 << qsel;
5989
5990         wr32(E1000_RAL(index), rar_low);
5991         wrfl();
5992         wr32(E1000_RAH(index), rar_high);
5993         wrfl();
5994 }
5995
5996 static int igb_set_vf_mac(struct igb_adapter *adapter,
5997                           int vf, unsigned char *mac_addr)
5998 {
5999         struct e1000_hw *hw = &adapter->hw;
6000         /* VF MAC addresses start at end of receive addresses and moves
6001          * torwards the first, as a result a collision should not be possible */
6002         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6003
6004         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6005
6006         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6007
6008         return 0;
6009 }
6010
6011 static void igb_vmm_control(struct igb_adapter *adapter)
6012 {
6013         struct e1000_hw *hw = &adapter->hw;
6014         u32 reg;
6015
6016         /* replication is not supported for 82575 */
6017         if (hw->mac.type == e1000_82575)
6018                 return;
6019
6020         /* enable replication vlan tag stripping */
6021         reg = rd32(E1000_RPLOLR);
6022         reg |= E1000_RPLOLR_STRVLAN;
6023         wr32(E1000_RPLOLR, reg);
6024
6025         /* notify HW that the MAC is adding vlan tags */
6026         reg = rd32(E1000_DTXCTL);
6027         reg |= E1000_DTXCTL_VLAN_ADDED;
6028         wr32(E1000_DTXCTL, reg);
6029
6030         if (adapter->vfs_allocated_count) {
6031                 igb_vmdq_set_loopback_pf(hw, true);
6032                 igb_vmdq_set_replication_pf(hw, true);
6033         } else {
6034                 igb_vmdq_set_loopback_pf(hw, false);
6035                 igb_vmdq_set_replication_pf(hw, false);
6036         }
6037 }
6038
6039 /* igb_main.c */