drivers/net/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2009 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/ipv6.h>
  35 #include <linux/slab.h>
  36 #include <net/checksum.h>
  37 #include <net/ip6_checksum.h>
  38 #include <linux/net_tstamp.h>
  39 #include <linux/mii.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/pci.h>
  43 #include <linux/pci-aspm.h>
  44 #include <linux/delay.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/if_ether.h>
  47 #include <linux/aer.h>
  48 #ifdef CONFIG_IGB_DCA
  49 #include <linux/dca.h>
  50 #endif
  51 #include "igb.h"
  52
  53 #define DRV_VERSION "2.1.0-k2"
  54 char igb_driver_name[] = "igb";
  55 char igb_driver_version[] = DRV_VERSION;
  56 static const char igb_driver_string[] =
  57                                 "Intel(R) Gigabit Ethernet Network Driver";
  58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
  59
  60 static const struct e1000_info *igb_info_tbl[] = {
  61         [board_82575] = &e1000_82575_info,
  62 };
  63
  64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  85         /* required last entry */
  86         {0, }
  87 };
  88
  89 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  90
  91 void igb_reset(struct igb_adapter *);
  92 static int igb_setup_all_tx_resources(struct igb_adapter *);
  93 static int igb_setup_all_rx_resources(struct igb_adapter *);
  94 static void igb_free_all_tx_resources(struct igb_adapter *);
  95 static void igb_free_all_rx_resources(struct igb_adapter *);
  96 static void igb_setup_mrqc(struct igb_adapter *);
  97 void igb_update_stats(struct igb_adapter *);
  98 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  99 static void __devexit igb_remove(struct pci_dev *pdev);
 100 static int igb_sw_init(struct igb_adapter *);
 101 static int igb_open(struct net_device *);
 102 static int igb_close(struct net_device *);
 103 static void igb_configure_tx(struct igb_adapter *);
 104 static void igb_configure_rx(struct igb_adapter *);
 105 static void igb_clean_all_tx_rings(struct igb_adapter *);
 106 static void igb_clean_all_rx_rings(struct igb_adapter *);
 107 static void igb_clean_tx_ring(struct igb_ring *);
 108 static void igb_clean_rx_ring(struct igb_ring *);
 109 static void igb_set_rx_mode(struct net_device *);
 110 static void igb_update_phy_info(unsigned long);
 111 static void igb_watchdog(unsigned long);
 112 static void igb_watchdog_task(struct work_struct *);
 113 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
 114 static struct net_device_stats *igb_get_stats(struct net_device *);
 115 static int igb_change_mtu(struct net_device *, int);
 116 static int igb_set_mac(struct net_device *, void *);
 117 static void igb_set_uta(struct igb_adapter *adapter);
 118 static irqreturn_t igb_intr(int irq, void *);
 119 static irqreturn_t igb_intr_msi(int irq, void *);
 120 static irqreturn_t igb_msix_other(int irq, void *);
 121 static irqreturn_t igb_msix_ring(int irq, void *);
 122 #ifdef CONFIG_IGB_DCA
 123 static void igb_update_dca(struct igb_q_vector *);
 124 static void igb_setup_dca(struct igb_adapter *);
 125 #endif /* CONFIG_IGB_DCA */
 126 static bool igb_clean_tx_irq(struct igb_q_vector *);
 127 static int igb_poll(struct napi_struct *, int);
 128 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
 129 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 130 static void igb_tx_timeout(struct net_device *);
 131 static void igb_reset_task(struct work_struct *);
 132 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
 133 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 134 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 135 static void igb_restore_vlan(struct igb_adapter *);
 136 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 137 static void igb_ping_all_vfs(struct igb_adapter *);
 138 static void igb_msg_task(struct igb_adapter *);
 139 static void igb_vmm_control(struct igb_adapter *);
 140 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 141 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 143 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 144                                int vf, u16 vlan, u8 qos);
 145 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 146 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 147                                  struct ifla_vf_info *ivi);
 148
 149 #ifdef CONFIG_PM
 150 static int igb_suspend(struct pci_dev *, pm_message_t);
 151 static int igb_resume(struct pci_dev *);
 152 #endif
 153 static void igb_shutdown(struct pci_dev *);
 154 #ifdef CONFIG_IGB_DCA
 155 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 156 static struct notifier_block dca_notifier = {
 157         .notifier_call  = igb_notify_dca,
 158         .next           = NULL,
 159         .priority       = 0
 160 };
 161 #endif
 162 #ifdef CONFIG_NET_POLL_CONTROLLER
 163 /* for netdump / net console */
 164 static void igb_netpoll(struct net_device *);
 165 #endif
 166 #ifdef CONFIG_PCI_IOV
 167 static unsigned int max_vfs = 0;
 168 module_param(max_vfs, uint, 0);
 169 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 170                  "per physical function");
 171 #endif /* CONFIG_PCI_IOV */
 172
 173 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 174                      pci_channel_state_t);
 175 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 176 static void igb_io_resume(struct pci_dev *);
 177
 178 static struct pci_error_handlers igb_err_handler = {
 179         .error_detected = igb_io_error_detected,
 180         .slot_reset = igb_io_slot_reset,
 181         .resume = igb_io_resume,
 182 };
 183
 184
 185 static struct pci_driver igb_driver = {
 186         .name     = igb_driver_name,
 187         .id_table = igb_pci_tbl,
 188         .probe    = igb_probe,
 189         .remove   = __devexit_p(igb_remove),
 190 #ifdef CONFIG_PM
 191         /* Power Managment Hooks */
 192         .suspend  = igb_suspend,
 193         .resume   = igb_resume,
 194 #endif
 195         .shutdown = igb_shutdown,
 196         .err_handler = &igb_err_handler
 197 };
 198
 199 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 200 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 201 MODULE_LICENSE("GPL");
 202 MODULE_VERSION(DRV_VERSION);
 203
 204 struct igb_reg_info {
 205         u32 ofs;
 206         char *name;
 207 };
 208
 209 static const struct igb_reg_info igb_reg_info_tbl[] = {
 210
 211         /* General Registers */
 212         {E1000_CTRL, "CTRL"},
 213         {E1000_STATUS, "STATUS"},
 214         {E1000_CTRL_EXT, "CTRL_EXT"},
 215
 216         /* Interrupt Registers */
 217         {E1000_ICR, "ICR"},
 218
 219         /* RX Registers */
 220         {E1000_RCTL, "RCTL"},
 221         {E1000_RDLEN(0), "RDLEN"},
 222         {E1000_RDH(0), "RDH"},
 223         {E1000_RDT(0), "RDT"},
 224         {E1000_RXDCTL(0), "RXDCTL"},
 225         {E1000_RDBAL(0), "RDBAL"},
 226         {E1000_RDBAH(0), "RDBAH"},
 227
 228         /* TX Registers */
 229         {E1000_TCTL, "TCTL"},
 230         {E1000_TDBAL(0), "TDBAL"},
 231         {E1000_TDBAH(0), "TDBAH"},
 232         {E1000_TDLEN(0), "TDLEN"},
 233         {E1000_TDH(0), "TDH"},
 234         {E1000_TDT(0), "TDT"},
 235         {E1000_TXDCTL(0), "TXDCTL"},
 236         {E1000_TDFH, "TDFH"},
 237         {E1000_TDFT, "TDFT"},
 238         {E1000_TDFHS, "TDFHS"},
 239         {E1000_TDFPC, "TDFPC"},
 240
 241         /* List Terminator */
 242         {}
 243 };
 244
 245 /*
 246  * igb_regdump - register printout routine
 247  */
 248 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
 249 {
 250         int n = 0;
 251         char rname[16];
 252         u32 regs[8];
 253
 254         switch (reginfo->ofs) {
 255         case E1000_RDLEN(0):
 256                 for (n = 0; n < 4; n++)
 257                         regs[n] = rd32(E1000_RDLEN(n));
 258                 break;
 259         case E1000_RDH(0):
 260                 for (n = 0; n < 4; n++)
 261                         regs[n] = rd32(E1000_RDH(n));
 262                 break;
 263         case E1000_RDT(0):
 264                 for (n = 0; n < 4; n++)
 265                         regs[n] = rd32(E1000_RDT(n));
 266                 break;
 267         case E1000_RXDCTL(0):
 268                 for (n = 0; n < 4; n++)
 269                         regs[n] = rd32(E1000_RXDCTL(n));
 270                 break;
 271         case E1000_RDBAL(0):
 272                 for (n = 0; n < 4; n++)
 273                         regs[n] = rd32(E1000_RDBAL(n));
 274                 break;
 275         case E1000_RDBAH(0):
 276                 for (n = 0; n < 4; n++)
 277                         regs[n] = rd32(E1000_RDBAH(n));
 278                 break;
 279         case E1000_TDBAL(0):
 280                 for (n = 0; n < 4; n++)
 281                         regs[n] = rd32(E1000_RDBAL(n));
 282                 break;
 283         case E1000_TDBAH(0):
 284                 for (n = 0; n < 4; n++)
 285                         regs[n] = rd32(E1000_TDBAH(n));
 286                 break;
 287         case E1000_TDLEN(0):
 288                 for (n = 0; n < 4; n++)
 289                         regs[n] = rd32(E1000_TDLEN(n));
 290                 break;
 291         case E1000_TDH(0):
 292                 for (n = 0; n < 4; n++)
 293                         regs[n] = rd32(E1000_TDH(n));
 294                 break;
 295         case E1000_TDT(0):
 296                 for (n = 0; n < 4; n++)
 297                         regs[n] = rd32(E1000_TDT(n));
 298                 break;
 299         case E1000_TXDCTL(0):
 300                 for (n = 0; n < 4; n++)
 301                         regs[n] = rd32(E1000_TXDCTL(n));
 302                 break;
 303         default:
 304                 printk(KERN_INFO "%-15s %08x\n",
 305                         reginfo->name, rd32(reginfo->ofs));
 306                 return;
 307         }
 308
 309         snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
 310         printk(KERN_INFO "%-15s ", rname);
 311         for (n = 0; n < 4; n++)
 312                 printk(KERN_CONT "%08x ", regs[n]);
 313         printk(KERN_CONT "\n");
 314 }
 315
 316 /*
 317  * igb_dump - Print registers, tx-rings and rx-rings
 318  */
 319 static void igb_dump(struct igb_adapter *adapter)
 320 {
 321         struct net_device *netdev = adapter->netdev;
 322         struct e1000_hw *hw = &adapter->hw;
 323         struct igb_reg_info *reginfo;
 324         int n = 0;
 325         struct igb_ring *tx_ring;
 326         union e1000_adv_tx_desc *tx_desc;
 327         struct my_u0 { u64 a; u64 b; } *u0;
 328         struct igb_buffer *buffer_info;
 329         struct igb_ring *rx_ring;
 330         union e1000_adv_rx_desc *rx_desc;
 331         u32 staterr;
 332         int i = 0;
 333
 334         if (!netif_msg_hw(adapter))
 335                 return;
 336
 337         /* Print netdevice Info */
 338         if (netdev) {
 339                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 340                 printk(KERN_INFO "Device Name     state            "
 341                         "trans_start      last_rx\n");
 342                 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
 343                 netdev->name,
 344                 netdev->state,
 345                 netdev->trans_start,
 346                 netdev->last_rx);
 347         }
 348
 349         /* Print Registers */
 350         dev_info(&adapter->pdev->dev, "Register Dump\n");
 351         printk(KERN_INFO " Register Name   Value\n");
 352         for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
 353              reginfo->name; reginfo++) {
 354                 igb_regdump(hw, reginfo);
 355         }
 356
 357         /* Print TX Ring Summary */
 358         if (!netdev || !netif_running(netdev))
 359                 goto exit;
 360
 361         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 362         printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma  ]"
 363                 " leng ntw timestamp\n");
 364         for (n = 0; n < adapter->num_tx_queues; n++) {
 365                 tx_ring = adapter->tx_ring[n];
 366                 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
 367                 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
 368                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
 369                            (u64)buffer_info->dma,
 370                            buffer_info->length,
 371                            buffer_info->next_to_watch,
 372                            (u64)buffer_info->time_stamp);
 373         }
 374
 375         /* Print TX Rings */
 376         if (!netif_msg_tx_done(adapter))
 377                 goto rx_ring_summary;
 378
 379         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 380
 381         /* Transmit Descriptor Formats
 382          *
 383          * Advanced Transmit Descriptor
 384          *   +--------------------------------------------------------------+
 385          * 0 |         Buffer Address [63:0]                                |
 386          *   +--------------------------------------------------------------+
 387          * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
 388          *   +--------------------------------------------------------------+
 389          *   63      46 45    40 39 38 36 35 32 31   24             15       0
 390          */
 391
 392         for (n = 0; n < adapter->num_tx_queues; n++) {
 393                 tx_ring = adapter->tx_ring[n];
 394                 printk(KERN_INFO "------------------------------------\n");
 395                 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 396                 printk(KERN_INFO "------------------------------------\n");
 397                 printk(KERN_INFO "T [desc]     [address 63:0  ] "
 398                         "[PlPOCIStDDM Ln] [bi->dma       ] "
 399                         "leng  ntw timestamp        bi->skb\n");
 400
 401                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 402                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
 403                         buffer_info = &tx_ring->buffer_info[i];
 404                         u0 = (struct my_u0 *)tx_desc;
 405                         printk(KERN_INFO "T [0x%03X]    %016llX %016llX %016llX"
 406                                 " %04X  %3X %016llX %p", i,
 407                                 le64_to_cpu(u0->a),
 408                                 le64_to_cpu(u0->b),
 409                                 (u64)buffer_info->dma,
 410                                 buffer_info->length,
 411                                 buffer_info->next_to_watch,
 412                                 (u64)buffer_info->time_stamp,
 413                                 buffer_info->skb);
 414                         if (i == tx_ring->next_to_use &&
 415                                 i == tx_ring->next_to_clean)
 416                                 printk(KERN_CONT " NTC/U\n");
 417                         else if (i == tx_ring->next_to_use)
 418                                 printk(KERN_CONT " NTU\n");
 419                         else if (i == tx_ring->next_to_clean)
 420                                 printk(KERN_CONT " NTC\n");
 421                         else
 422                                 printk(KERN_CONT "\n");
 423
 424                         if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
 425                                 print_hex_dump(KERN_INFO, "",
 426                                         DUMP_PREFIX_ADDRESS,
 427                                         16, 1, phys_to_virt(buffer_info->dma),
 428                                         buffer_info->length, true);
 429                 }
 430         }
 431
 432         /* Print RX Rings Summary */
 433 rx_ring_summary:
 434         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 435         printk(KERN_INFO "Queue [NTU] [NTC]\n");
 436         for (n = 0; n < adapter->num_rx_queues; n++) {
 437                 rx_ring = adapter->rx_ring[n];
 438                 printk(KERN_INFO " %5d %5X %5X\n", n,
 439                            rx_ring->next_to_use, rx_ring->next_to_clean);
 440         }
 441
 442         /* Print RX Rings */
 443         if (!netif_msg_rx_status(adapter))
 444                 goto exit;
 445
 446         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 447
 448         /* Advanced Receive Descriptor (Read) Format
 449          *    63                                           1        0
 450          *    +-----------------------------------------------------+
 451          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 452          *    +----------------------------------------------+------+
 453          *  8 |       Header Buffer Address [63:1]           |  DD  |
 454          *    +-----------------------------------------------------+
 455          *
 456          *
 457          * Advanced Receive Descriptor (Write-Back) Format
 458          *
 459          *   63       48 47    32 31  30      21 20 17 16   4 3     0
 460          *   +------------------------------------------------------+
 461          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 462          *   | Checksum   Ident  |   |           |    | Type | Type |
 463          *   +------------------------------------------------------+
 464          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 465          *   +------------------------------------------------------+
 466          *   63       48 47    32 31            20 19               0
 467          */
 468
 469         for (n = 0; n < adapter->num_rx_queues; n++) {
 470                 rx_ring = adapter->rx_ring[n];
 471                 printk(KERN_INFO "------------------------------------\n");
 472                 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 473                 printk(KERN_INFO "------------------------------------\n");
 474                 printk(KERN_INFO "R  [desc]      [ PktBuf     A0] "
 475                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
 476                         "<-- Adv Rx Read format\n");
 477                 printk(KERN_INFO "RWB[desc]      [PcsmIpSHl PtRs] "
 478                         "[vl er S cks ln] ---------------- [bi->skb] "
 479                         "<-- Adv Rx Write-Back format\n");
 480
 481                 for (i = 0; i < rx_ring->count; i++) {
 482                         buffer_info = &rx_ring->buffer_info[i];
 483                         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
 484                         u0 = (struct my_u0 *)rx_desc;
 485                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 486                         if (staterr & E1000_RXD_STAT_DD) {
 487                                 /* Descriptor Done */
 488                                 printk(KERN_INFO "RWB[0x%03X]     %016llX "
 489                                         "%016llX ---------------- %p", i,
 490                                         le64_to_cpu(u0->a),
 491                                         le64_to_cpu(u0->b),
 492                                         buffer_info->skb);
 493                         } else {
 494                                 printk(KERN_INFO "R  [0x%03X]     %016llX "
 495                                         "%016llX %016llX %p", i,
 496                                         le64_to_cpu(u0->a),
 497                                         le64_to_cpu(u0->b),
 498                                         (u64)buffer_info->dma,
 499                                         buffer_info->skb);
 500
 501                                 if (netif_msg_pktdata(adapter)) {
 502                                         print_hex_dump(KERN_INFO, "",
 503                                                 DUMP_PREFIX_ADDRESS,
 504                                                 16, 1,
 505                                                 phys_to_virt(buffer_info->dma),
 506                                                 rx_ring->rx_buffer_len, true);
 507                                         if (rx_ring->rx_buffer_len
 508                                                 < IGB_RXBUFFER_1024)
 509                                                 print_hex_dump(KERN_INFO, "",
 510                                                   DUMP_PREFIX_ADDRESS,
 511                                                   16, 1,
 512                                                   phys_to_virt(
 513                                                     buffer_info->page_dma +
 514                                                     buffer_info->page_offset),
 515                                                   PAGE_SIZE/2, true);
 516                                 }
 517                         }
 518
 519                         if (i == rx_ring->next_to_use)
 520                                 printk(KERN_CONT " NTU\n");
 521                         else if (i == rx_ring->next_to_clean)
 522                                 printk(KERN_CONT " NTC\n");
 523                         else
 524                                 printk(KERN_CONT "\n");
 525
 526                 }
 527         }
 528
 529 exit:
 530         return;
 531 }
 532
 533
 534 /**
 535  * igb_read_clock - read raw cycle counter (to be used by time counter)
 536  */
 537 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 538 {
 539         struct igb_adapter *adapter =
 540                 container_of(tc, struct igb_adapter, cycles);
 541         struct e1000_hw *hw = &adapter->hw;
 542         u64 stamp = 0;
 543         int shift = 0;
 544
 545         /*
 546          * The timestamp latches on lowest register read. For the 82580
 547          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 548          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 549          */
 550         if (hw->mac.type == e1000_82580) {
 551                 stamp = rd32(E1000_SYSTIMR) >> 8;
 552                 shift = IGB_82580_TSYNC_SHIFT;
 553         }
 554
 555         stamp |= (u64)rd32(E1000_SYSTIML) << shift;
 556         stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
 557         return stamp;
 558 }
 559
 560 /**
 561  * igb_get_hw_dev - return device
 562  * used by hardware layer to print debugging information
 563  **/
 564 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 565 {
 566         struct igb_adapter *adapter = hw->back;
 567         return adapter->netdev;
 568 }
 569
 570 /**
 571  * igb_init_module - Driver Registration Routine
 572  *
 573  * igb_init_module is the first routine called when the driver is
 574  * loaded. All it does is register with the PCI subsystem.
 575  **/
 576 static int __init igb_init_module(void)
 577 {
 578         int ret;
 579         printk(KERN_INFO "%s - version %s\n",
 580                igb_driver_string, igb_driver_version);
 581
 582         printk(KERN_INFO "%s\n", igb_copyright);
 583
 584 #ifdef CONFIG_IGB_DCA
 585         dca_register_notify(&dca_notifier);
 586 #endif
 587         ret = pci_register_driver(&igb_driver);
 588         return ret;
 589 }
 590
 591 module_init(igb_init_module);
 592
 593 /**
 594  * igb_exit_module - Driver Exit Cleanup Routine
 595  *
 596  * igb_exit_module is called just before the driver is removed
 597  * from memory.
 598  **/
 599 static void __exit igb_exit_module(void)
 600 {
 601 #ifdef CONFIG_IGB_DCA
 602         dca_unregister_notify(&dca_notifier);
 603 #endif
 604         pci_unregister_driver(&igb_driver);
 605 }
 606
 607 module_exit(igb_exit_module);
 608
 609 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 610 /**
 611  * igb_cache_ring_register - Descriptor ring to register mapping
 612  * @adapter: board private structure to initialize
 613  *
 614  * Once we know the feature-set enabled for the device, we'll cache
 615  * the register offset the descriptor ring is assigned to.
 616  **/
 617 static void igb_cache_ring_register(struct igb_adapter *adapter)
 618 {
 619         int i = 0, j = 0;
 620         u32 rbase_offset = adapter->vfs_allocated_count;
 621
 622         switch (adapter->hw.mac.type) {
 623         case e1000_82576:
 624                 /* The queues are allocated for virtualization such that VF 0
 625                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 626                  * In order to avoid collision we start at the first free queue
 627                  * and continue consuming queues in the same sequence
 628                  */
 629                 if (adapter->vfs_allocated_count) {
 630                         for (; i < adapter->rss_queues; i++)
 631                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 632                                                                Q_IDX_82576(i);
 633                         for (; j < adapter->rss_queues; j++)
 634                                 adapter->tx_ring[j]->reg_idx = rbase_offset +
 635                                                                Q_IDX_82576(j);
 636                 }
 637         case e1000_82575:
 638         case e1000_82580:
 639         case e1000_i350:
 640         default:
 641                 for (; i < adapter->num_rx_queues; i++)
 642                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 643                 for (; j < adapter->num_tx_queues; j++)
 644                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 645                 break;
 646         }
 647 }
 648
 649 static void igb_free_queues(struct igb_adapter *adapter)
 650 {
 651         int i;
 652
 653         for (i = 0; i < adapter->num_tx_queues; i++) {
 654                 kfree(adapter->tx_ring[i]);
 655                 adapter->tx_ring[i] = NULL;
 656         }
 657         for (i = 0; i < adapter->num_rx_queues; i++) {
 658                 kfree(adapter->rx_ring[i]);
 659                 adapter->rx_ring[i] = NULL;
 660         }
 661         adapter->num_rx_queues = 0;
 662         adapter->num_tx_queues = 0;
 663 }
 664
 665 /**
 666  * igb_alloc_queues - Allocate memory for all rings
 667  * @adapter: board private structure to initialize
 668  *
 669  * We allocate one ring per queue at run-time since we don't know the
 670  * number of queues at compile-time.
 671  **/
 672 static int igb_alloc_queues(struct igb_adapter *adapter)
 673 {
 674         struct igb_ring *ring;
 675         int i;
 676
 677         for (i = 0; i < adapter->num_tx_queues; i++) {
 678                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 679                 if (!ring)
 680                         goto err;
 681                 ring->count = adapter->tx_ring_count;
 682                 ring->queue_index = i;
 683                 ring->dev = &adapter->pdev->dev;
 684                 ring->netdev = adapter->netdev;
 685                 /* For 82575, context index must be unique per ring. */
 686                 if (adapter->hw.mac.type == e1000_82575)
 687                         ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
 688                 adapter->tx_ring[i] = ring;
 689         }
 690
 691         for (i = 0; i < adapter->num_rx_queues; i++) {
 692                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 693                 if (!ring)
 694                         goto err;
 695                 ring->count = adapter->rx_ring_count;
 696                 ring->queue_index = i;
 697                 ring->dev = &adapter->pdev->dev;
 698                 ring->netdev = adapter->netdev;
 699                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 700                 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
 701                 /* set flag indicating ring supports SCTP checksum offload */
 702                 if (adapter->hw.mac.type >= e1000_82576)
 703                         ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
 704                 adapter->rx_ring[i] = ring;
 705         }
 706
 707         igb_cache_ring_register(adapter);
 708
 709         return 0;
 710
 711 err:
 712         igb_free_queues(adapter);
 713
 714         return -ENOMEM;
 715 }
 716
 717 #define IGB_N0_QUEUE -1
 718 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 719 {
 720         u32 msixbm = 0;
 721         struct igb_adapter *adapter = q_vector->adapter;
 722         struct e1000_hw *hw = &adapter->hw;
 723         u32 ivar, index;
 724         int rx_queue = IGB_N0_QUEUE;
 725         int tx_queue = IGB_N0_QUEUE;
 726
 727         if (q_vector->rx_ring)
 728                 rx_queue = q_vector->rx_ring->reg_idx;
 729         if (q_vector->tx_ring)
 730                 tx_queue = q_vector->tx_ring->reg_idx;
 731
 732         switch (hw->mac.type) {
 733         case e1000_82575:
 734                 /* The 82575 assigns vectors using a bitmask, which matches the
 735                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 736                    or more queues to a vector, we write the appropriate bits
 737                    into the MSIXBM register for that vector. */
 738                 if (rx_queue > IGB_N0_QUEUE)
 739                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 740                 if (tx_queue > IGB_N0_QUEUE)
 741                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 742                 if (!adapter->msix_entries && msix_vector == 0)
 743                         msixbm |= E1000_EIMS_OTHER;
 744                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 745                 q_vector->eims_value = msixbm;
 746                 break;
 747         case e1000_82576:
 748                 /* 82576 uses a table-based method for assigning vectors.
 749                    Each queue has a single entry in the table to which we write
 750                    a vector number along with a "valid" bit.  Sadly, the layout
 751                    of the table is somewhat counterintuitive. */
 752                 if (rx_queue > IGB_N0_QUEUE) {
 753                         index = (rx_queue & 0x7);
 754                         ivar = array_rd32(E1000_IVAR0, index);
 755                         if (rx_queue < 8) {
 756                                 /* vector goes into low byte of register */
 757                                 ivar = ivar & 0xFFFFFF00;
 758                                 ivar |= msix_vector | E1000_IVAR_VALID;
 759                         } else {
 760                                 /* vector goes into third byte of register */
 761                                 ivar = ivar & 0xFF00FFFF;
 762                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 763                         }
 764                         array_wr32(E1000_IVAR0, index, ivar);
 765                 }
 766                 if (tx_queue > IGB_N0_QUEUE) {
 767                         index = (tx_queue & 0x7);
 768                         ivar = array_rd32(E1000_IVAR0, index);
 769                         if (tx_queue < 8) {
 770                                 /* vector goes into second byte of register */
 771                                 ivar = ivar & 0xFFFF00FF;
 772                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 773                         } else {
 774                                 /* vector goes into high byte of register */
 775                                 ivar = ivar & 0x00FFFFFF;
 776                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 777                         }
 778                         array_wr32(E1000_IVAR0, index, ivar);
 779                 }
 780                 q_vector->eims_value = 1 << msix_vector;
 781                 break;
 782         case e1000_82580:
 783         case e1000_i350:
 784                 /* 82580 uses the same table-based approach as 82576 but has fewer
 785                    entries as a result we carry over for queues greater than 4. */
 786                 if (rx_queue > IGB_N0_QUEUE) {
 787                         index = (rx_queue >> 1);
 788                         ivar = array_rd32(E1000_IVAR0, index);
 789                         if (rx_queue & 0x1) {
 790                                 /* vector goes into third byte of register */
 791                                 ivar = ivar & 0xFF00FFFF;
 792                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 793                         } else {
 794                                 /* vector goes into low byte of register */
 795                                 ivar = ivar & 0xFFFFFF00;
 796                                 ivar |= msix_vector | E1000_IVAR_VALID;
 797                         }
 798                         array_wr32(E1000_IVAR0, index, ivar);
 799                 }
 800                 if (tx_queue > IGB_N0_QUEUE) {
 801                         index = (tx_queue >> 1);
 802                         ivar = array_rd32(E1000_IVAR0, index);
 803                         if (tx_queue & 0x1) {
 804                                 /* vector goes into high byte of register */
 805                                 ivar = ivar & 0x00FFFFFF;
 806                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 807                         } else {
 808                                 /* vector goes into second byte of register */
 809                                 ivar = ivar & 0xFFFF00FF;
 810                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 811                         }
 812                         array_wr32(E1000_IVAR0, index, ivar);
 813                 }
 814                 q_vector->eims_value = 1 << msix_vector;
 815                 break;
 816         default:
 817                 BUG();
 818                 break;
 819         }
 820
 821         /* add q_vector eims value to global eims_enable_mask */
 822         adapter->eims_enable_mask |= q_vector->eims_value;
 823
 824         /* configure q_vector to set itr on first interrupt */
 825         q_vector->set_itr = 1;
 826 }
 827
 828 /**
 829  * igb_configure_msix - Configure MSI-X hardware
 830  *
 831  * igb_configure_msix sets up the hardware to properly
 832  * generate MSI-X interrupts.
 833  **/
 834 static void igb_configure_msix(struct igb_adapter *adapter)
 835 {
 836         u32 tmp;
 837         int i, vector = 0;
 838         struct e1000_hw *hw = &adapter->hw;
 839
 840         adapter->eims_enable_mask = 0;
 841
 842         /* set vector for other causes, i.e. link changes */
 843         switch (hw->mac.type) {
 844         case e1000_82575:
 845                 tmp = rd32(E1000_CTRL_EXT);
 846                 /* enable MSI-X PBA support*/
 847                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 848
 849                 /* Auto-Mask interrupts upon ICR read. */
 850                 tmp |= E1000_CTRL_EXT_EIAME;
 851                 tmp |= E1000_CTRL_EXT_IRCA;
 852
 853                 wr32(E1000_CTRL_EXT, tmp);
 854
 855                 /* enable msix_other interrupt */
 856                 array_wr32(E1000_MSIXBM(0), vector++,
 857                                       E1000_EIMS_OTHER);
 858                 adapter->eims_other = E1000_EIMS_OTHER;
 859
 860                 break;
 861
 862         case e1000_82576:
 863         case e1000_82580:
 864         case e1000_i350:
 865                 /* Turn on MSI-X capability first, or our settings
 866                  * won't stick.  And it will take days to debug. */
 867                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 868                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 869                                 E1000_GPIE_NSICR);
 870
 871                 /* enable msix_other interrupt */
 872                 adapter->eims_other = 1 << vector;
 873                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 874
 875                 wr32(E1000_IVAR_MISC, tmp);
 876                 break;
 877         default:
 878                 /* do nothing, since nothing else supports MSI-X */
 879                 break;
 880         } /* switch (hw->mac.type) */
 881
 882         adapter->eims_enable_mask |= adapter->eims_other;
 883
 884         for (i = 0; i < adapter->num_q_vectors; i++)
 885                 igb_assign_vector(adapter->q_vector[i], vector++);
 886
 887         wrfl();
 888 }
 889
 890 /**
 891  * igb_request_msix - Initialize MSI-X interrupts
 892  *
 893  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 894  * kernel.
 895  **/
 896 static int igb_request_msix(struct igb_adapter *adapter)
 897 {
 898         struct net_device *netdev = adapter->netdev;
 899         struct e1000_hw *hw = &adapter->hw;
 900         int i, err = 0, vector = 0;
 901
 902         err = request_irq(adapter->msix_entries[vector].vector,
 903                           igb_msix_other, 0, netdev->name, adapter);
 904         if (err)
 905                 goto out;
 906         vector++;
 907
 908         for (i = 0; i < adapter->num_q_vectors; i++) {
 909                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 910
 911                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 912
 913                 if (q_vector->rx_ring && q_vector->tx_ring)
 914                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 915                                 q_vector->rx_ring->queue_index);
 916                 else if (q_vector->tx_ring)
 917                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 918                                 q_vector->tx_ring->queue_index);
 919                 else if (q_vector->rx_ring)
 920                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 921                                 q_vector->rx_ring->queue_index);
 922                 else
 923                         sprintf(q_vector->name, "%s-unused", netdev->name);
 924
 925                 err = request_irq(adapter->msix_entries[vector].vector,
 926                                   igb_msix_ring, 0, q_vector->name,
 927                                   q_vector);
 928                 if (err)
 929                         goto out;
 930                 vector++;
 931         }
 932
 933         igb_configure_msix(adapter);
 934         return 0;
 935 out:
 936         return err;
 937 }
 938
 939 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 940 {
 941         if (adapter->msix_entries) {
 942                 pci_disable_msix(adapter->pdev);
 943                 kfree(adapter->msix_entries);
 944                 adapter->msix_entries = NULL;
 945         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 946                 pci_disable_msi(adapter->pdev);
 947         }
 948 }
 949
 950 /**
 951  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 952  * @adapter: board private structure to initialize
 953  *
 954  * This function frees the memory allocated to the q_vectors.  In addition if
 955  * NAPI is enabled it will delete any references to the NAPI struct prior
 956  * to freeing the q_vector.
 957  **/
 958 static void igb_free_q_vectors(struct igb_adapter *adapter)
 959 {
 960         int v_idx;
 961
 962         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 963                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 964                 adapter->q_vector[v_idx] = NULL;
 965                 if (!q_vector)
 966                         continue;
 967                 netif_napi_del(&q_vector->napi);
 968                 kfree(q_vector);
 969         }
 970         adapter->num_q_vectors = 0;
 971 }
 972
 973 /**
 974  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 975  *
 976  * This function resets the device so that it has 0 rx queues, tx queues, and
 977  * MSI-X interrupts allocated.
 978  */
 979 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 980 {
 981         igb_free_queues(adapter);
 982         igb_free_q_vectors(adapter);
 983         igb_reset_interrupt_capability(adapter);
 984 }
 985
 986 /**
 987  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 988  *
 989  * Attempt to configure interrupts using the best available
 990  * capabilities of the hardware and kernel.
 991  **/
 992 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
 993 {
 994         int err;
 995         int numvecs, i;
 996
 997         /* Number of supported queues. */
 998         adapter->num_rx_queues = adapter->rss_queues;
 999         adapter->num_tx_queues = adapter->rss_queues;
1000
1001         /* start with one vector for every rx queue */
1002         numvecs = adapter->num_rx_queues;
1003
1004         /* if tx handler is separate add 1 for every tx queue */
1005         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006                 numvecs += adapter->num_tx_queues;
1007
1008         /* store the number of vectors reserved for queues */
1009         adapter->num_q_vectors = numvecs;
1010
1011         /* add 1 vector for link status interrupts */
1012         numvecs++;
1013         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1014                                         GFP_KERNEL);
1015         if (!adapter->msix_entries)
1016                 goto msi_only;
1017
1018         for (i = 0; i < numvecs; i++)
1019                 adapter->msix_entries[i].entry = i;
1020
1021         err = pci_enable_msix(adapter->pdev,
1022                               adapter->msix_entries,
1023                               numvecs);
1024         if (err == 0)
1025                 goto out;
1026
1027         igb_reset_interrupt_capability(adapter);
1028
1029         /* If we can't do MSI-X, try MSI */
1030 msi_only:
1031 #ifdef CONFIG_PCI_IOV
1032         /* disable SR-IOV for non MSI-X configurations */
1033         if (adapter->vf_data) {
1034                 struct e1000_hw *hw = &adapter->hw;
1035                 /* disable iov and allow time for transactions to clear */
1036                 pci_disable_sriov(adapter->pdev);
1037                 msleep(500);
1038
1039                 kfree(adapter->vf_data);
1040                 adapter->vf_data = NULL;
1041                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1042                 msleep(100);
1043                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1044         }
1045 #endif
1046         adapter->vfs_allocated_count = 0;
1047         adapter->rss_queues = 1;
1048         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1049         adapter->num_rx_queues = 1;
1050         adapter->num_tx_queues = 1;
1051         adapter->num_q_vectors = 1;
1052         if (!pci_enable_msi(adapter->pdev))
1053                 adapter->flags |= IGB_FLAG_HAS_MSI;
1054 out:
1055         /* Notify the stack of the (possibly) reduced Tx Queue count. */
1056         adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
1057 }
1058
1059 /**
1060  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061  * @adapter: board private structure to initialize
1062  *
1063  * We allocate one q_vector per queue interrupt.  If allocation fails we
1064  * return -ENOMEM.
1065  **/
1066 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1067 {
1068         struct igb_q_vector *q_vector;
1069         struct e1000_hw *hw = &adapter->hw;
1070         int v_idx;
1071
1072         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073                 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1074                 if (!q_vector)
1075                         goto err_out;
1076                 q_vector->adapter = adapter;
1077                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078                 q_vector->itr_val = IGB_START_ITR;
1079                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080                 adapter->q_vector[v_idx] = q_vector;
1081         }
1082         return 0;
1083
1084 err_out:
1085         igb_free_q_vectors(adapter);
1086         return -ENOMEM;
1087 }
1088
1089 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090                                       int ring_idx, int v_idx)
1091 {
1092         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1093
1094         q_vector->rx_ring = adapter->rx_ring[ring_idx];
1095         q_vector->rx_ring->q_vector = q_vector;
1096         q_vector->itr_val = adapter->rx_itr_setting;
1097         if (q_vector->itr_val && q_vector->itr_val <= 3)
1098                 q_vector->itr_val = IGB_START_ITR;
1099 }
1100
1101 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102                                       int ring_idx, int v_idx)
1103 {
1104         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1105
1106         q_vector->tx_ring = adapter->tx_ring[ring_idx];
1107         q_vector->tx_ring->q_vector = q_vector;
1108         q_vector->itr_val = adapter->tx_itr_setting;
1109         if (q_vector->itr_val && q_vector->itr_val <= 3)
1110                 q_vector->itr_val = IGB_START_ITR;
1111 }
1112
1113 /**
1114  * igb_map_ring_to_vector - maps allocated queues to vectors
1115  *
1116  * This function maps the recently allocated queues to vectors.
1117  **/
1118 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1119 {
1120         int i;
1121         int v_idx = 0;
1122
1123         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124             (adapter->num_q_vectors < adapter->num_tx_queues))
1125                 return -ENOMEM;
1126
1127         if (adapter->num_q_vectors >=
1128             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129                 for (i = 0; i < adapter->num_rx_queues; i++)
1130                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131                 for (i = 0; i < adapter->num_tx_queues; i++)
1132                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1133         } else {
1134                 for (i = 0; i < adapter->num_rx_queues; i++) {
1135                         if (i < adapter->num_tx_queues)
1136                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1138                 }
1139                 for (; i < adapter->num_tx_queues; i++)
1140                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141         }
1142         return 0;
1143 }
1144
1145 /**
1146  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1147  *
1148  * This function initializes the interrupts and allocates all of the queues.
1149  **/
1150 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1151 {
1152         struct pci_dev *pdev = adapter->pdev;
1153         int err;
1154
1155         igb_set_interrupt_capability(adapter);
1156
1157         err = igb_alloc_q_vectors(adapter);
1158         if (err) {
1159                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160                 goto err_alloc_q_vectors;
1161         }
1162
1163         err = igb_alloc_queues(adapter);
1164         if (err) {
1165                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166                 goto err_alloc_queues;
1167         }
1168
1169         err = igb_map_ring_to_vector(adapter);
1170         if (err) {
1171                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172                 goto err_map_queues;
1173         }
1174
1175
1176         return 0;
1177 err_map_queues:
1178         igb_free_queues(adapter);
1179 err_alloc_queues:
1180         igb_free_q_vectors(adapter);
1181 err_alloc_q_vectors:
1182         igb_reset_interrupt_capability(adapter);
1183         return err;
1184 }
1185
1186 /**
1187  * igb_request_irq - initialize interrupts
1188  *
1189  * Attempts to configure interrupts using the best available
1190  * capabilities of the hardware and kernel.
1191  **/
1192 static int igb_request_irq(struct igb_adapter *adapter)
1193 {
1194         struct net_device *netdev = adapter->netdev;
1195         struct pci_dev *pdev = adapter->pdev;
1196         int err = 0;
1197
1198         if (adapter->msix_entries) {
1199                 err = igb_request_msix(adapter);
1200                 if (!err)
1201                         goto request_done;
1202                 /* fall back to MSI */
1203                 igb_clear_interrupt_scheme(adapter);
1204                 if (!pci_enable_msi(adapter->pdev))
1205                         adapter->flags |= IGB_FLAG_HAS_MSI;
1206                 igb_free_all_tx_resources(adapter);
1207                 igb_free_all_rx_resources(adapter);
1208                 adapter->num_tx_queues = 1;
1209                 adapter->num_rx_queues = 1;
1210                 adapter->num_q_vectors = 1;
1211                 err = igb_alloc_q_vectors(adapter);
1212                 if (err) {
1213                         dev_err(&pdev->dev,
1214                                 "Unable to allocate memory for vectors\n");
1215                         goto request_done;
1216                 }
1217                 err = igb_alloc_queues(adapter);
1218                 if (err) {
1219                         dev_err(&pdev->dev,
1220                                 "Unable to allocate memory for queues\n");
1221                         igb_free_q_vectors(adapter);
1222                         goto request_done;
1223                 }
1224                 igb_setup_all_tx_resources(adapter);
1225                 igb_setup_all_rx_resources(adapter);
1226         } else {
1227                 igb_assign_vector(adapter->q_vector[0], 0);
1228         }
1229
1230         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1231                 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1232                                   netdev->name, adapter);
1233                 if (!err)
1234                         goto request_done;
1235
1236                 /* fall back to legacy interrupts */
1237                 igb_reset_interrupt_capability(adapter);
1238                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1239         }
1240
1241         err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1242                           netdev->name, adapter);
1243
1244         if (err)
1245                 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1246                         err);
1247
1248 request_done:
1249         return err;
1250 }
1251
1252 static void igb_free_irq(struct igb_adapter *adapter)
1253 {
1254         if (adapter->msix_entries) {
1255                 int vector = 0, i;
1256
1257                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1258
1259                 for (i = 0; i < adapter->num_q_vectors; i++) {
1260                         struct igb_q_vector *q_vector = adapter->q_vector[i];
1261                         free_irq(adapter->msix_entries[vector++].vector,
1262                                  q_vector);
1263                 }
1264         } else {
1265                 free_irq(adapter->pdev->irq, adapter);
1266         }
1267 }
1268
1269 /**
1270  * igb_irq_disable - Mask off interrupt generation on the NIC
1271  * @adapter: board private structure
1272  **/
1273 static void igb_irq_disable(struct igb_adapter *adapter)
1274 {
1275         struct e1000_hw *hw = &adapter->hw;
1276
1277         /*
1278          * we need to be careful when disabling interrupts.  The VFs are also
1279          * mapped into these registers and so clearing the bits can cause
1280          * issues on the VF drivers so we only need to clear what we set
1281          */
1282         if (adapter->msix_entries) {
1283                 u32 regval = rd32(E1000_EIAM);
1284                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285                 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286                 regval = rd32(E1000_EIAC);
1287                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1288         }
1289
1290         wr32(E1000_IAM, 0);
1291         wr32(E1000_IMC, ~0);
1292         wrfl();
1293         synchronize_irq(adapter->pdev->irq);
1294 }
1295
1296 /**
1297  * igb_irq_enable - Enable default interrupt generation settings
1298  * @adapter: board private structure
1299  **/
1300 static void igb_irq_enable(struct igb_adapter *adapter)
1301 {
1302         struct e1000_hw *hw = &adapter->hw;
1303
1304         if (adapter->msix_entries) {
1305                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1306                 u32 regval = rd32(E1000_EIAC);
1307                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308                 regval = rd32(E1000_EIAM);
1309                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1310                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1311                 if (adapter->vfs_allocated_count) {
1312                         wr32(E1000_MBVFIMR, 0xFF);
1313                         ims |= E1000_IMS_VMMB;
1314                 }
1315                 if (adapter->hw.mac.type == e1000_82580)
1316                         ims |= E1000_IMS_DRSTA;
1317
1318                 wr32(E1000_IMS, ims);
1319         } else {
1320                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1321                                 E1000_IMS_DRSTA);
1322                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1323                                 E1000_IMS_DRSTA);
1324         }
1325 }
1326
1327 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1328 {
1329         struct e1000_hw *hw = &adapter->hw;
1330         u16 vid = adapter->hw.mng_cookie.vlan_id;
1331         u16 old_vid = adapter->mng_vlan_id;
1332
1333         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334                 /* add VID to filter table */
1335                 igb_vfta_set(hw, vid, true);
1336                 adapter->mng_vlan_id = vid;
1337         } else {
1338                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1339         }
1340
1341         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1342             (vid != old_vid) &&
1343             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344                 /* remove VID from filter table */
1345                 igb_vfta_set(hw, old_vid, false);
1346         }
1347 }
1348
1349 /**
1350  * igb_release_hw_control - release control of the h/w to f/w
1351  * @adapter: address of board private structure
1352  *
1353  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354  * For ASF and Pass Through versions of f/w this means that the
1355  * driver is no longer loaded.
1356  *
1357  **/
1358 static void igb_release_hw_control(struct igb_adapter *adapter)
1359 {
1360         struct e1000_hw *hw = &adapter->hw;
1361         u32 ctrl_ext;
1362
1363         /* Let firmware take over control of h/w */
1364         ctrl_ext = rd32(E1000_CTRL_EXT);
1365         wr32(E1000_CTRL_EXT,
1366                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1367 }
1368
1369 /**
1370  * igb_get_hw_control - get control of the h/w from f/w
1371  * @adapter: address of board private structure
1372  *
1373  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374  * For ASF and Pass Through versions of f/w this means that
1375  * the driver is loaded.
1376  *
1377  **/
1378 static void igb_get_hw_control(struct igb_adapter *adapter)
1379 {
1380         struct e1000_hw *hw = &adapter->hw;
1381         u32 ctrl_ext;
1382
1383         /* Let firmware know the driver has taken over */
1384         ctrl_ext = rd32(E1000_CTRL_EXT);
1385         wr32(E1000_CTRL_EXT,
1386                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1387 }
1388
1389 /**
1390  * igb_configure - configure the hardware for RX and TX
1391  * @adapter: private board structure
1392  **/
1393 static void igb_configure(struct igb_adapter *adapter)
1394 {
1395         struct net_device *netdev = adapter->netdev;
1396         int i;
1397
1398         igb_get_hw_control(adapter);
1399         igb_set_rx_mode(netdev);
1400
1401         igb_restore_vlan(adapter);
1402
1403         igb_setup_tctl(adapter);
1404         igb_setup_mrqc(adapter);
1405         igb_setup_rctl(adapter);
1406
1407         igb_configure_tx(adapter);
1408         igb_configure_rx(adapter);
1409
1410         igb_rx_fifo_flush_82575(&adapter->hw);
1411
1412         /* call igb_desc_unused which always leaves
1413          * at least 1 descriptor unused to make sure
1414          * next_to_use != next_to_clean */
1415         for (i = 0; i < adapter->num_rx_queues; i++) {
1416                 struct igb_ring *ring = adapter->rx_ring[i];
1417                 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1418         }
1419 }
1420
1421 /**
1422  * igb_power_up_link - Power up the phy/serdes link
1423  * @adapter: address of board private structure
1424  **/
1425 void igb_power_up_link(struct igb_adapter *adapter)
1426 {
1427         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428                 igb_power_up_phy_copper(&adapter->hw);
1429         else
1430                 igb_power_up_serdes_link_82575(&adapter->hw);
1431 }
1432
1433 /**
1434  * igb_power_down_link - Power down the phy/serdes link
1435  * @adapter: address of board private structure
1436  */
1437 static void igb_power_down_link(struct igb_adapter *adapter)
1438 {
1439         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440                 igb_power_down_phy_copper_82575(&adapter->hw);
1441         else
1442                 igb_shutdown_serdes_link_82575(&adapter->hw);
1443 }
1444
1445 /**
1446  * igb_up - Open the interface and prepare it to handle traffic
1447  * @adapter: board private structure
1448  **/
1449 int igb_up(struct igb_adapter *adapter)
1450 {
1451         struct e1000_hw *hw = &adapter->hw;
1452         int i;
1453
1454         /* hardware has been reset, we need to reload some things */
1455         igb_configure(adapter);
1456
1457         clear_bit(__IGB_DOWN, &adapter->state);
1458
1459         for (i = 0; i < adapter->num_q_vectors; i++) {
1460                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461                 napi_enable(&q_vector->napi);
1462         }
1463         if (adapter->msix_entries)
1464                 igb_configure_msix(adapter);
1465         else
1466                 igb_assign_vector(adapter->q_vector[0], 0);
1467
1468         /* Clear any pending interrupts. */
1469         rd32(E1000_ICR);
1470         igb_irq_enable(adapter);
1471
1472         /* notify VFs that reset has been completed */
1473         if (adapter->vfs_allocated_count) {
1474                 u32 reg_data = rd32(E1000_CTRL_EXT);
1475                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476                 wr32(E1000_CTRL_EXT, reg_data);
1477         }
1478
1479         netif_tx_start_all_queues(adapter->netdev);
1480
1481         /* start the watchdog. */
1482         hw->mac.get_link_status = 1;
1483         schedule_work(&adapter->watchdog_task);
1484
1485         return 0;
1486 }
1487
1488 void igb_down(struct igb_adapter *adapter)
1489 {
1490         struct net_device *netdev = adapter->netdev;
1491         struct e1000_hw *hw = &adapter->hw;
1492         u32 tctl, rctl;
1493         int i;
1494
1495         /* signal that we're down so the interrupt handler does not
1496          * reschedule our watchdog timer */
1497         set_bit(__IGB_DOWN, &adapter->state);
1498
1499         /* disable receives in the hardware */
1500         rctl = rd32(E1000_RCTL);
1501         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502         /* flush and sleep below */
1503
1504         netif_tx_stop_all_queues(netdev);
1505
1506         /* disable transmits in the hardware */
1507         tctl = rd32(E1000_TCTL);
1508         tctl &= ~E1000_TCTL_EN;
1509         wr32(E1000_TCTL, tctl);
1510         /* flush both disables and wait for them to finish */
1511         wrfl();
1512         msleep(10);
1513
1514         for (i = 0; i < adapter->num_q_vectors; i++) {
1515                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516                 napi_disable(&q_vector->napi);
1517         }
1518
1519         igb_irq_disable(adapter);
1520
1521         del_timer_sync(&adapter->watchdog_timer);
1522         del_timer_sync(&adapter->phy_info_timer);
1523
1524         netif_carrier_off(netdev);
1525
1526         /* record the stats before reset*/
1527         igb_update_stats(adapter);
1528
1529         adapter->link_speed = 0;
1530         adapter->link_duplex = 0;
1531
1532         if (!pci_channel_offline(adapter->pdev))
1533                 igb_reset(adapter);
1534         igb_clean_all_tx_rings(adapter);
1535         igb_clean_all_rx_rings(adapter);
1536 #ifdef CONFIG_IGB_DCA
1537
1538         /* since we reset the hardware DCA settings were cleared */
1539         igb_setup_dca(adapter);
1540 #endif
1541 }
1542
1543 void igb_reinit_locked(struct igb_adapter *adapter)
1544 {
1545         WARN_ON(in_interrupt());
1546         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1547                 msleep(1);
1548         igb_down(adapter);
1549         igb_up(adapter);
1550         clear_bit(__IGB_RESETTING, &adapter->state);
1551 }
1552
1553 void igb_reset(struct igb_adapter *adapter)
1554 {
1555         struct pci_dev *pdev = adapter->pdev;
1556         struct e1000_hw *hw = &adapter->hw;
1557         struct e1000_mac_info *mac = &hw->mac;
1558         struct e1000_fc_info *fc = &hw->fc;
1559         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1560         u16 hwm;
1561
1562         /* Repartition Pba for greater than 9k mtu
1563          * To take effect CTRL.RST is required.
1564          */
1565         switch (mac->type) {
1566         case e1000_i350:
1567         case e1000_82580:
1568                 pba = rd32(E1000_RXPBS);
1569                 pba = igb_rxpbs_adjust_82580(pba);
1570                 break;
1571         case e1000_82576:
1572                 pba = rd32(E1000_RXPBS);
1573                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1574                 break;
1575         case e1000_82575:
1576         default:
1577                 pba = E1000_PBA_34K;
1578                 break;
1579         }
1580
1581         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582             (mac->type < e1000_82576)) {
1583                 /* adjust PBA for jumbo frames */
1584                 wr32(E1000_PBA, pba);
1585
1586                 /* To maintain wire speed transmits, the Tx FIFO should be
1587                  * large enough to accommodate two full transmit packets,
1588                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1589                  * the Rx FIFO should be large enough to accommodate at least
1590                  * one full receive packet and is similarly rounded up and
1591                  * expressed in KB. */
1592                 pba = rd32(E1000_PBA);
1593                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594                 tx_space = pba >> 16;
1595                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1596                 pba &= 0xffff;
1597                 /* the tx fifo also stores 16 bytes of information about the tx
1598                  * but don't include ethernet FCS because hardware appends it */
1599                 min_tx_space = (adapter->max_frame_size +
1600                                 sizeof(union e1000_adv_tx_desc) -
1601                                 ETH_FCS_LEN) * 2;
1602                 min_tx_space = ALIGN(min_tx_space, 1024);
1603                 min_tx_space >>= 10;
1604                 /* software strips receive CRC, so leave room for it */
1605                 min_rx_space = adapter->max_frame_size;
1606                 min_rx_space = ALIGN(min_rx_space, 1024);
1607                 min_rx_space >>= 10;
1608
1609                 /* If current Tx allocation is less than the min Tx FIFO size,
1610                  * and the min Tx FIFO size is less than the current Rx FIFO
1611                  * allocation, take space away from current Rx allocation */
1612                 if (tx_space < min_tx_space &&
1613                     ((min_tx_space - tx_space) < pba)) {
1614                         pba = pba - (min_tx_space - tx_space);
1615
1616                         /* if short on rx space, rx wins and must trump tx
1617                          * adjustment */
1618                         if (pba < min_rx_space)
1619                                 pba = min_rx_space;
1620                 }
1621                 wr32(E1000_PBA, pba);
1622         }
1623
1624         /* flow control settings */
1625         /* The high water mark must be low enough to fit one full frame
1626          * (or the size used for early receive) above it in the Rx FIFO.
1627          * Set it to the lower of:
1628          * - 90% of the Rx FIFO size, or
1629          * - the full Rx FIFO size minus one full frame */
1630         hwm = min(((pba << 10) * 9 / 10),
1631                         ((pba << 10) - 2 * adapter->max_frame_size));
1632
1633         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1634         fc->low_water = fc->high_water - 16;
1635         fc->pause_time = 0xFFFF;
1636         fc->send_xon = 1;
1637         fc->current_mode = fc->requested_mode;
1638
1639         /* disable receive for all VFs and wait one second */
1640         if (adapter->vfs_allocated_count) {
1641                 int i;
1642                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1643                         adapter->vf_data[i].flags = 0;
1644
1645                 /* ping all the active vfs to let them know we are going down */
1646                 igb_ping_all_vfs(adapter);
1647
1648                 /* disable transmits and receives */
1649                 wr32(E1000_VFRE, 0);
1650                 wr32(E1000_VFTE, 0);
1651         }
1652
1653         /* Allow time for pending master requests to run */
1654         hw->mac.ops.reset_hw(hw);
1655         wr32(E1000_WUC, 0);
1656
1657         if (hw->mac.ops.init_hw(hw))
1658                 dev_err(&pdev->dev, "Hardware Error\n");
1659
1660         if (hw->mac.type == e1000_82580) {
1661                 u32 reg = rd32(E1000_PCIEMISC);
1662                 wr32(E1000_PCIEMISC,
1663                                 reg & ~E1000_PCIEMISC_LX_DECISION);
1664         }
1665         if (!netif_running(adapter->netdev))
1666                 igb_power_down_link(adapter);
1667
1668         igb_update_mng_vlan(adapter);
1669
1670         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1672
1673         igb_get_phy_info(hw);
1674 }
1675
1676 static const struct net_device_ops igb_netdev_ops = {
1677         .ndo_open               = igb_open,
1678         .ndo_stop               = igb_close,
1679         .ndo_start_xmit         = igb_xmit_frame_adv,
1680         .ndo_get_stats          = igb_get_stats,
1681         .ndo_set_rx_mode        = igb_set_rx_mode,
1682         .ndo_set_multicast_list = igb_set_rx_mode,
1683         .ndo_set_mac_address    = igb_set_mac,
1684         .ndo_change_mtu         = igb_change_mtu,
1685         .ndo_do_ioctl           = igb_ioctl,
1686         .ndo_tx_timeout         = igb_tx_timeout,
1687         .ndo_validate_addr      = eth_validate_addr,
1688         .ndo_vlan_rx_register   = igb_vlan_rx_register,
1689         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1690         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1691         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1692         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1693         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1694         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1695 #ifdef CONFIG_NET_POLL_CONTROLLER
1696         .ndo_poll_controller    = igb_netpoll,
1697 #endif
1698 };
1699
1700 /**
1701  * igb_probe - Device Initialization Routine
1702  * @pdev: PCI device information struct
1703  * @ent: entry in igb_pci_tbl
1704  *
1705  * Returns 0 on success, negative on failure
1706  *
1707  * igb_probe initializes an adapter identified by a pci_dev structure.
1708  * The OS initialization, configuring of the adapter private structure,
1709  * and a hardware reset occur.
1710  **/
1711 static int __devinit igb_probe(struct pci_dev *pdev,
1712                                const struct pci_device_id *ent)
1713 {
1714         struct net_device *netdev;
1715         struct igb_adapter *adapter;
1716         struct e1000_hw *hw;
1717         u16 eeprom_data = 0;
1718         static int global_quad_port_a; /* global quad port a indication */
1719         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720         unsigned long mmio_start, mmio_len;
1721         int err, pci_using_dac;
1722         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1723         u32 part_num;
1724
1725         err = pci_enable_device_mem(pdev);
1726         if (err)
1727                 return err;
1728
1729         pci_using_dac = 0;
1730         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1731         if (!err) {
1732                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1733                 if (!err)
1734                         pci_using_dac = 1;
1735         } else {
1736                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1737                 if (err) {
1738                         err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1739                         if (err) {
1740                                 dev_err(&pdev->dev, "No usable DMA "
1741                                         "configuration, aborting\n");
1742                                 goto err_dma;
1743                         }
1744                 }
1745         }
1746
1747         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1748                                            IORESOURCE_MEM),
1749                                            igb_driver_name);
1750         if (err)
1751                 goto err_pci_reg;
1752
1753         pci_enable_pcie_error_reporting(pdev);
1754
1755         pci_set_master(pdev);
1756         pci_save_state(pdev);
1757
1758         err = -ENOMEM;
1759         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1760                                    IGB_ABS_MAX_TX_QUEUES);
1761         if (!netdev)
1762                 goto err_alloc_etherdev;
1763
1764         SET_NETDEV_DEV(netdev, &pdev->dev);
1765
1766         pci_set_drvdata(pdev, netdev);
1767         adapter = netdev_priv(netdev);
1768         adapter->netdev = netdev;
1769         adapter->pdev = pdev;
1770         hw = &adapter->hw;
1771         hw->back = adapter;
1772         adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1773
1774         mmio_start = pci_resource_start(pdev, 0);
1775         mmio_len = pci_resource_len(pdev, 0);
1776
1777         err = -EIO;
1778         hw->hw_addr = ioremap(mmio_start, mmio_len);
1779         if (!hw->hw_addr)
1780                 goto err_ioremap;
1781
1782         netdev->netdev_ops = &igb_netdev_ops;
1783         igb_set_ethtool_ops(netdev);
1784         netdev->watchdog_timeo = 5 * HZ;
1785
1786         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1787
1788         netdev->mem_start = mmio_start;
1789         netdev->mem_end = mmio_start + mmio_len;
1790
1791         /* PCI config space info */
1792         hw->vendor_id = pdev->vendor;
1793         hw->device_id = pdev->device;
1794         hw->revision_id = pdev->revision;
1795         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1796         hw->subsystem_device_id = pdev->subsystem_device;
1797
1798         /* Copy the default MAC, PHY and NVM function pointers */
1799         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1800         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1801         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1802         /* Initialize skew-specific constants */
1803         err = ei->get_invariants(hw);
1804         if (err)
1805                 goto err_sw_init;
1806
1807         /* setup the private structure */
1808         err = igb_sw_init(adapter);
1809         if (err)
1810                 goto err_sw_init;
1811
1812         igb_get_bus_info_pcie(hw);
1813
1814         hw->phy.autoneg_wait_to_complete = false;
1815
1816         /* Copper options */
1817         if (hw->phy.media_type == e1000_media_type_copper) {
1818                 hw->phy.mdix = AUTO_ALL_MODES;
1819                 hw->phy.disable_polarity_correction = false;
1820                 hw->phy.ms_type = e1000_ms_hw_default;
1821         }
1822
1823         if (igb_check_reset_block(hw))
1824                 dev_info(&pdev->dev,
1825                         "PHY reset is blocked due to SOL/IDER session.\n");
1826
1827         netdev->features = NETIF_F_SG |
1828                            NETIF_F_IP_CSUM |
1829                            NETIF_F_HW_VLAN_TX |
1830                            NETIF_F_HW_VLAN_RX |
1831                            NETIF_F_HW_VLAN_FILTER;
1832
1833         netdev->features |= NETIF_F_IPV6_CSUM;
1834         netdev->features |= NETIF_F_TSO;
1835         netdev->features |= NETIF_F_TSO6;
1836         netdev->features |= NETIF_F_GRO;
1837
1838         netdev->vlan_features |= NETIF_F_TSO;
1839         netdev->vlan_features |= NETIF_F_TSO6;
1840         netdev->vlan_features |= NETIF_F_IP_CSUM;
1841         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1842         netdev->vlan_features |= NETIF_F_SG;
1843
1844         if (pci_using_dac)
1845                 netdev->features |= NETIF_F_HIGHDMA;
1846
1847         if (hw->mac.type >= e1000_82576)
1848                 netdev->features |= NETIF_F_SCTP_CSUM;
1849
1850         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1851
1852         /* before reading the NVM, reset the controller to put the device in a
1853          * known good starting state */
1854         hw->mac.ops.reset_hw(hw);
1855
1856         /* make sure the NVM is good */
1857         if (igb_validate_nvm_checksum(hw) < 0) {
1858                 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1859                 err = -EIO;
1860                 goto err_eeprom;
1861         }
1862
1863         /* copy the MAC address out of the NVM */
1864         if (hw->mac.ops.read_mac_addr(hw))
1865                 dev_err(&pdev->dev, "NVM Read Error\n");
1866
1867         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1868         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1869
1870         if (!is_valid_ether_addr(netdev->perm_addr)) {
1871                 dev_err(&pdev->dev, "Invalid MAC Address\n");
1872                 err = -EIO;
1873                 goto err_eeprom;
1874         }
1875
1876         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1877                     (unsigned long) adapter);
1878         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1879                     (unsigned long) adapter);
1880
1881         INIT_WORK(&adapter->reset_task, igb_reset_task);
1882         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1883
1884         /* Initialize link properties that are user-changeable */
1885         adapter->fc_autoneg = true;
1886         hw->mac.autoneg = true;
1887         hw->phy.autoneg_advertised = 0x2f;
1888
1889         hw->fc.requested_mode = e1000_fc_default;
1890         hw->fc.current_mode = e1000_fc_default;
1891
1892         igb_validate_mdi_setting(hw);
1893
1894         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1895          * enable the ACPI Magic Packet filter
1896          */
1897
1898         if (hw->bus.func == 0)
1899                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1900         else if (hw->mac.type == e1000_82580)
1901                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1902                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1903                                  &eeprom_data);
1904         else if (hw->bus.func == 1)
1905                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1906
1907         if (eeprom_data & eeprom_apme_mask)
1908                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1909
1910         /* now that we have the eeprom settings, apply the special cases where
1911          * the eeprom may be wrong or the board simply won't support wake on
1912          * lan on a particular port */
1913         switch (pdev->device) {
1914         case E1000_DEV_ID_82575GB_QUAD_COPPER:
1915                 adapter->eeprom_wol = 0;
1916                 break;
1917         case E1000_DEV_ID_82575EB_FIBER_SERDES:
1918         case E1000_DEV_ID_82576_FIBER:
1919         case E1000_DEV_ID_82576_SERDES:
1920                 /* Wake events only supported on port A for dual fiber
1921                  * regardless of eeprom setting */
1922                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1923                         adapter->eeprom_wol = 0;
1924                 break;
1925         case E1000_DEV_ID_82576_QUAD_COPPER:
1926         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1927                 /* if quad port adapter, disable WoL on all but port A */
1928                 if (global_quad_port_a != 0)
1929                         adapter->eeprom_wol = 0;
1930                 else
1931                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1932                 /* Reset for multiple quad port adapters */
1933                 if (++global_quad_port_a == 4)
1934                         global_quad_port_a = 0;
1935                 break;
1936         }
1937
1938         /* initialize the wol settings based on the eeprom settings */
1939         adapter->wol = adapter->eeprom_wol;
1940         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1941
1942         /* reset the hardware with the new settings */
1943         igb_reset(adapter);
1944
1945         /* let the f/w know that the h/w is now under the control of the
1946          * driver. */
1947         igb_get_hw_control(adapter);
1948
1949         strcpy(netdev->name, "eth%d");
1950         err = register_netdev(netdev);
1951         if (err)
1952                 goto err_register;
1953
1954         /* carrier off reporting is important to ethtool even BEFORE open */
1955         netif_carrier_off(netdev);
1956
1957 #ifdef CONFIG_IGB_DCA
1958         if (dca_add_requester(&pdev->dev) == 0) {
1959                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1960                 dev_info(&pdev->dev, "DCA enabled\n");
1961                 igb_setup_dca(adapter);
1962         }
1963
1964 #endif
1965         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1966         /* print bus type/speed/width info */
1967         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1968                  netdev->name,
1969                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1970                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1971                                                             "unknown"),
1972                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1973                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1974                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1975                    "unknown"),
1976                  netdev->dev_addr);
1977
1978         igb_read_part_num(hw, &part_num);
1979         dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1980                 (part_num >> 8), (part_num & 0xff));
1981
1982         dev_info(&pdev->dev,
1983                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1984                 adapter->msix_entries ? "MSI-X" :
1985                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1986                 adapter->num_rx_queues, adapter->num_tx_queues);
1987
1988         return 0;
1989
1990 err_register:
1991         igb_release_hw_control(adapter);
1992 err_eeprom:
1993         if (!igb_check_reset_block(hw))
1994                 igb_reset_phy(hw);
1995
1996         if (hw->flash_address)
1997                 iounmap(hw->flash_address);
1998 err_sw_init:
1999         igb_clear_interrupt_scheme(adapter);
2000         iounmap(hw->hw_addr);
2001 err_ioremap:
2002         free_netdev(netdev);
2003 err_alloc_etherdev:
2004         pci_release_selected_regions(pdev,
2005                                      pci_select_bars(pdev, IORESOURCE_MEM));
2006 err_pci_reg:
2007 err_dma:
2008         pci_disable_device(pdev);
2009         return err;
2010 }
2011
2012 /**
2013  * igb_remove - Device Removal Routine
2014  * @pdev: PCI device information struct
2015  *
2016  * igb_remove is called by the PCI subsystem to alert the driver
2017  * that it should release a PCI device.  The could be caused by a
2018  * Hot-Plug event, or because the driver is going to be removed from
2019  * memory.
2020  **/
2021 static void __devexit igb_remove(struct pci_dev *pdev)
2022 {
2023         struct net_device *netdev = pci_get_drvdata(pdev);
2024         struct igb_adapter *adapter = netdev_priv(netdev);
2025         struct e1000_hw *hw = &adapter->hw;
2026
2027         /* flush_scheduled work may reschedule our watchdog task, so
2028          * explicitly disable watchdog tasks from being rescheduled  */
2029         set_bit(__IGB_DOWN, &adapter->state);
2030         del_timer_sync(&adapter->watchdog_timer);
2031         del_timer_sync(&adapter->phy_info_timer);
2032
2033         flush_scheduled_work();
2034
2035 #ifdef CONFIG_IGB_DCA
2036         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2037                 dev_info(&pdev->dev, "DCA disabled\n");
2038                 dca_remove_requester(&pdev->dev);
2039                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2040                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2041         }
2042 #endif
2043
2044         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2045          * would have already happened in close and is redundant. */
2046         igb_release_hw_control(adapter);
2047
2048         unregister_netdev(netdev);
2049
2050         igb_clear_interrupt_scheme(adapter);
2051
2052 #ifdef CONFIG_PCI_IOV
2053         /* reclaim resources allocated to VFs */
2054         if (adapter->vf_data) {
2055                 /* disable iov and allow time for transactions to clear */
2056                 pci_disable_sriov(pdev);
2057                 msleep(500);
2058
2059                 kfree(adapter->vf_data);
2060                 adapter->vf_data = NULL;
2061                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2062                 msleep(100);
2063                 dev_info(&pdev->dev, "IOV Disabled\n");
2064         }
2065 #endif
2066
2067         iounmap(hw->hw_addr);
2068         if (hw->flash_address)
2069                 iounmap(hw->flash_address);
2070         pci_release_selected_regions(pdev,
2071                                      pci_select_bars(pdev, IORESOURCE_MEM));
2072
2073         free_netdev(netdev);
2074
2075         pci_disable_pcie_error_reporting(pdev);
2076
2077         pci_disable_device(pdev);
2078 }
2079
2080 /**
2081  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2082  * @adapter: board private structure to initialize
2083  *
2084  * This function initializes the vf specific data storage and then attempts to
2085  * allocate the VFs.  The reason for ordering it this way is because it is much
2086  * mor expensive time wise to disable SR-IOV than it is to allocate and free
2087  * the memory for the VFs.
2088  **/
2089 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2090 {
2091 #ifdef CONFIG_PCI_IOV
2092         struct pci_dev *pdev = adapter->pdev;
2093
2094         if (adapter->vfs_allocated_count > 7)
2095                 adapter->vfs_allocated_count = 7;
2096
2097         if (adapter->vfs_allocated_count) {
2098                 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2099                                            sizeof(struct vf_data_storage),
2100                                            GFP_KERNEL);
2101                 /* if allocation failed then we do not support SR-IOV */
2102                 if (!adapter->vf_data) {
2103                         adapter->vfs_allocated_count = 0;
2104                         dev_err(&pdev->dev, "Unable to allocate memory for VF "
2105                                 "Data Storage\n");
2106                 }
2107         }
2108
2109         if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2110                 kfree(adapter->vf_data);
2111                 adapter->vf_data = NULL;
2112 #endif /* CONFIG_PCI_IOV */
2113                 adapter->vfs_allocated_count = 0;
2114 #ifdef CONFIG_PCI_IOV
2115         } else {
2116                 unsigned char mac_addr[ETH_ALEN];
2117                 int i;
2118                 dev_info(&pdev->dev, "%d vfs allocated\n",
2119                          adapter->vfs_allocated_count);
2120                 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2121                         random_ether_addr(mac_addr);
2122                         igb_set_vf_mac(adapter, i, mac_addr);
2123                 }
2124         }
2125 #endif /* CONFIG_PCI_IOV */
2126 }
2127
2128
2129 /**
2130  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2131  * @adapter: board private structure to initialize
2132  *
2133  * igb_init_hw_timer initializes the function pointer and values for the hw
2134  * timer found in hardware.
2135  **/
2136 static void igb_init_hw_timer(struct igb_adapter *adapter)
2137 {
2138         struct e1000_hw *hw = &adapter->hw;
2139
2140         switch (hw->mac.type) {
2141         case e1000_i350:
2142         case e1000_82580:
2143                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2144                 adapter->cycles.read = igb_read_clock;
2145                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2146                 adapter->cycles.mult = 1;
2147                 /*
2148                  * The 82580 timesync updates the system timer every 8ns by 8ns
2149                  * and the value cannot be shifted.  Instead we need to shift
2150                  * the registers to generate a 64bit timer value.  As a result
2151                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2152                  * 24 in order to generate a larger value for synchronization.
2153                  */
2154                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2155                 /* disable system timer temporarily by setting bit 31 */
2156                 wr32(E1000_TSAUXC, 0x80000000);
2157                 wrfl();
2158
2159                 /* Set registers so that rollover occurs soon to test this. */
2160                 wr32(E1000_SYSTIMR, 0x00000000);
2161                 wr32(E1000_SYSTIML, 0x80000000);
2162                 wr32(E1000_SYSTIMH, 0x000000FF);
2163                 wrfl();
2164
2165                 /* enable system timer by clearing bit 31 */
2166                 wr32(E1000_TSAUXC, 0x0);
2167                 wrfl();
2168
2169                 timecounter_init(&adapter->clock,
2170                                  &adapter->cycles,
2171                                  ktime_to_ns(ktime_get_real()));
2172                 /*
2173                  * Synchronize our NIC clock against system wall clock. NIC
2174                  * time stamp reading requires ~3us per sample, each sample
2175                  * was pretty stable even under load => only require 10
2176                  * samples for each offset comparison.
2177                  */
2178                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2179                 adapter->compare.source = &adapter->clock;
2180                 adapter->compare.target = ktime_get_real;
2181                 adapter->compare.num_samples = 10;
2182                 timecompare_update(&adapter->compare, 0);
2183                 break;
2184         case e1000_82576:
2185                 /*
2186                  * Initialize hardware timer: we keep it running just in case
2187                  * that some program needs it later on.
2188                  */
2189                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2190                 adapter->cycles.read = igb_read_clock;
2191                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2192                 adapter->cycles.mult = 1;
2193                 /**
2194                  * Scale the NIC clock cycle by a large factor so that
2195                  * relatively small clock corrections can be added or
2196                  * substracted at each clock tick. The drawbacks of a large
2197                  * factor are a) that the clock register overflows more quickly
2198                  * (not such a big deal) and b) that the increment per tick has
2199                  * to fit into 24 bits.  As a result we need to use a shift of
2200                  * 19 so we can fit a value of 16 into the TIMINCA register.
2201                  */
2202                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2203                 wr32(E1000_TIMINCA,
2204                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
2205                                 (16 << IGB_82576_TSYNC_SHIFT));
2206
2207                 /* Set registers so that rollover occurs soon to test this. */
2208                 wr32(E1000_SYSTIML, 0x00000000);
2209                 wr32(E1000_SYSTIMH, 0xFF800000);
2210                 wrfl();
2211
2212                 timecounter_init(&adapter->clock,
2213                                  &adapter->cycles,
2214                                  ktime_to_ns(ktime_get_real()));
2215                 /*
2216                  * Synchronize our NIC clock against system wall clock. NIC
2217                  * time stamp reading requires ~3us per sample, each sample
2218                  * was pretty stable even under load => only require 10
2219                  * samples for each offset comparison.
2220                  */
2221                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2222                 adapter->compare.source = &adapter->clock;
2223                 adapter->compare.target = ktime_get_real;
2224                 adapter->compare.num_samples = 10;
2225                 timecompare_update(&adapter->compare, 0);
2226                 break;
2227         case e1000_82575:
2228                 /* 82575 does not support timesync */
2229         default:
2230                 break;
2231         }
2232
2233 }
2234
2235 /**
2236  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2237  * @adapter: board private structure to initialize
2238  *
2239  * igb_sw_init initializes the Adapter private data structure.
2240  * Fields are initialized based on PCI device information and
2241  * OS network device settings (MTU size).
2242  **/
2243 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2244 {
2245         struct e1000_hw *hw = &adapter->hw;
2246         struct net_device *netdev = adapter->netdev;
2247         struct pci_dev *pdev = adapter->pdev;
2248
2249         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2250
2251         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2252         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2253         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2254         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2255
2256         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2257         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2258
2259 #ifdef CONFIG_PCI_IOV
2260         if (hw->mac.type == e1000_82576)
2261                 adapter->vfs_allocated_count = max_vfs;
2262
2263 #endif /* CONFIG_PCI_IOV */
2264         adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2265
2266         /*
2267          * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2268          * then we should combine the queues into a queue pair in order to
2269          * conserve interrupts due to limited supply
2270          */
2271         if ((adapter->rss_queues > 4) ||
2272             ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2273                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2274
2275         /* This call may decrease the number of queues */
2276         if (igb_init_interrupt_scheme(adapter)) {
2277                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2278                 return -ENOMEM;
2279         }
2280
2281         igb_init_hw_timer(adapter);
2282         igb_probe_vfs(adapter);
2283
2284         /* Explicitly disable IRQ since the NIC can be in any state. */
2285         igb_irq_disable(adapter);
2286
2287         set_bit(__IGB_DOWN, &adapter->state);
2288         return 0;
2289 }
2290
2291 /**
2292  * igb_open - Called when a network interface is made active
2293  * @netdev: network interface device structure
2294  *
2295  * Returns 0 on success, negative value on failure
2296  *
2297  * The open entry point is called when a network interface is made
2298  * active by the system (IFF_UP).  At this point all resources needed
2299  * for transmit and receive operations are allocated, the interrupt
2300  * handler is registered with the OS, the watchdog timer is started,
2301  * and the stack is notified that the interface is ready.
2302  **/
2303 static int igb_open(struct net_device *netdev)
2304 {
2305         struct igb_adapter *adapter = netdev_priv(netdev);
2306         struct e1000_hw *hw = &adapter->hw;
2307         int err;
2308         int i;
2309
2310         /* disallow open during test */
2311         if (test_bit(__IGB_TESTING, &adapter->state))
2312                 return -EBUSY;
2313
2314         netif_carrier_off(netdev);
2315
2316         /* allocate transmit descriptors */
2317         err = igb_setup_all_tx_resources(adapter);
2318         if (err)
2319                 goto err_setup_tx;
2320
2321         /* allocate receive descriptors */
2322         err = igb_setup_all_rx_resources(adapter);
2323         if (err)
2324                 goto err_setup_rx;
2325
2326         igb_power_up_link(adapter);
2327
2328         /* before we allocate an interrupt, we must be ready to handle it.
2329          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2330          * as soon as we call pci_request_irq, so we have to setup our
2331          * clean_rx handler before we do so.  */
2332         igb_configure(adapter);
2333
2334         err = igb_request_irq(adapter);
2335         if (err)
2336                 goto err_req_irq;
2337
2338         /* From here on the code is the same as igb_up() */
2339         clear_bit(__IGB_DOWN, &adapter->state);
2340
2341         for (i = 0; i < adapter->num_q_vectors; i++) {
2342                 struct igb_q_vector *q_vector = adapter->q_vector[i];
2343                 napi_enable(&q_vector->napi);
2344         }
2345
2346         /* Clear any pending interrupts. */
2347         rd32(E1000_ICR);
2348
2349         igb_irq_enable(adapter);
2350
2351         /* notify VFs that reset has been completed */
2352         if (adapter->vfs_allocated_count) {
2353                 u32 reg_data = rd32(E1000_CTRL_EXT);
2354                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2355                 wr32(E1000_CTRL_EXT, reg_data);
2356         }
2357
2358         netif_tx_start_all_queues(netdev);
2359
2360         /* start the watchdog. */
2361         hw->mac.get_link_status = 1;
2362         schedule_work(&adapter->watchdog_task);
2363
2364         return 0;
2365
2366 err_req_irq:
2367         igb_release_hw_control(adapter);
2368         igb_power_down_link(adapter);
2369         igb_free_all_rx_resources(adapter);
2370 err_setup_rx:
2371         igb_free_all_tx_resources(adapter);
2372 err_setup_tx:
2373         igb_reset(adapter);
2374
2375         return err;
2376 }
2377
2378 /**
2379  * igb_close - Disables a network interface
2380  * @netdev: network interface device structure
2381  *
2382  * Returns 0, this is not allowed to fail
2383  *
2384  * The close entry point is called when an interface is de-activated
2385  * by the OS.  The hardware is still under the driver's control, but
2386  * needs to be disabled.  A global MAC reset is issued to stop the
2387  * hardware, and all transmit and receive resources are freed.
2388  **/
2389 static int igb_close(struct net_device *netdev)
2390 {
2391         struct igb_adapter *adapter = netdev_priv(netdev);
2392
2393         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2394         igb_down(adapter);
2395
2396         igb_free_irq(adapter);
2397
2398         igb_free_all_tx_resources(adapter);
2399         igb_free_all_rx_resources(adapter);
2400
2401         return 0;
2402 }
2403
2404 /**
2405  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2406  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2407  *
2408  * Return 0 on success, negative on failure
2409  **/
2410 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2411 {
2412         struct device *dev = tx_ring->dev;
2413         int size;
2414
2415         size = sizeof(struct igb_buffer) * tx_ring->count;
2416         tx_ring->buffer_info = vmalloc(size);
2417         if (!tx_ring->buffer_info)
2418                 goto err;
2419         memset(tx_ring->buffer_info, 0, size);
2420
2421         /* round up to nearest 4K */
2422         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2423         tx_ring->size = ALIGN(tx_ring->size, 4096);
2424
2425         tx_ring->desc = dma_alloc_coherent(dev,
2426                                            tx_ring->size,
2427                                            &tx_ring->dma,
2428                                            GFP_KERNEL);
2429
2430         if (!tx_ring->desc)
2431                 goto err;
2432
2433         tx_ring->next_to_use = 0;
2434         tx_ring->next_to_clean = 0;
2435         return 0;
2436
2437 err:
2438         vfree(tx_ring->buffer_info);
2439         dev_err(dev,
2440                 "Unable to allocate memory for the transmit descriptor ring\n");
2441         return -ENOMEM;
2442 }
2443
2444 /**
2445  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2446  *                                (Descriptors) for all queues
2447  * @adapter: board private structure
2448  *
2449  * Return 0 on success, negative on failure
2450  **/
2451 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2452 {
2453         struct pci_dev *pdev = adapter->pdev;
2454         int i, err = 0;
2455
2456         for (i = 0; i < adapter->num_tx_queues; i++) {
2457                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2458                 if (err) {
2459                         dev_err(&pdev->dev,
2460                                 "Allocation for Tx Queue %u failed\n", i);
2461                         for (i--; i >= 0; i--)
2462                                 igb_free_tx_resources(adapter->tx_ring[i]);
2463                         break;
2464                 }
2465         }
2466
2467         for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2468                 int r_idx = i % adapter->num_tx_queues;
2469                 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2470         }
2471         return err;
2472 }
2473
2474 /**
2475  * igb_setup_tctl - configure the transmit control registers
2476  * @adapter: Board private structure
2477  **/
2478 void igb_setup_tctl(struct igb_adapter *adapter)
2479 {
2480         struct e1000_hw *hw = &adapter->hw;
2481         u32 tctl;
2482
2483         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2484         wr32(E1000_TXDCTL(0), 0);
2485
2486         /* Program the Transmit Control Register */
2487         tctl = rd32(E1000_TCTL);
2488         tctl &= ~E1000_TCTL_CT;
2489         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2490                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2491
2492         igb_config_collision_dist(hw);
2493
2494         /* Enable transmits */
2495         tctl |= E1000_TCTL_EN;
2496
2497         wr32(E1000_TCTL, tctl);
2498 }
2499
2500 /**
2501  * igb_configure_tx_ring - Configure transmit ring after Reset
2502  * @adapter: board private structure
2503  * @ring: tx ring to configure
2504  *
2505  * Configure a transmit ring after a reset.
2506  **/
2507 void igb_configure_tx_ring(struct igb_adapter *adapter,
2508                            struct igb_ring *ring)
2509 {
2510         struct e1000_hw *hw = &adapter->hw;
2511         u32 txdctl;
2512         u64 tdba = ring->dma;
2513         int reg_idx = ring->reg_idx;
2514
2515         /* disable the queue */
2516         txdctl = rd32(E1000_TXDCTL(reg_idx));
2517         wr32(E1000_TXDCTL(reg_idx),
2518                         txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2519         wrfl();
2520         mdelay(10);
2521
2522         wr32(E1000_TDLEN(reg_idx),
2523                         ring->count * sizeof(union e1000_adv_tx_desc));
2524         wr32(E1000_TDBAL(reg_idx),
2525                         tdba & 0x00000000ffffffffULL);
2526         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2527
2528         ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2529         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2530         writel(0, ring->head);
2531         writel(0, ring->tail);
2532
2533         txdctl |= IGB_TX_PTHRESH;
2534         txdctl |= IGB_TX_HTHRESH << 8;
2535         txdctl |= IGB_TX_WTHRESH << 16;
2536
2537         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2538         wr32(E1000_TXDCTL(reg_idx), txdctl);
2539 }
2540
2541 /**
2542  * igb_configure_tx - Configure transmit Unit after Reset
2543  * @adapter: board private structure
2544  *
2545  * Configure the Tx unit of the MAC after a reset.
2546  **/
2547 static void igb_configure_tx(struct igb_adapter *adapter)
2548 {
2549         int i;
2550
2551         for (i = 0; i < adapter->num_tx_queues; i++)
2552                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2553 }
2554
2555 /**
2556  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2557  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2558  *
2559  * Returns 0 on success, negative on failure
2560  **/
2561 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2562 {
2563         struct device *dev = rx_ring->dev;
2564         int size, desc_len;
2565
2566         size = sizeof(struct igb_buffer) * rx_ring->count;
2567         rx_ring->buffer_info = vmalloc(size);
2568         if (!rx_ring->buffer_info)
2569                 goto err;
2570         memset(rx_ring->buffer_info, 0, size);
2571
2572         desc_len = sizeof(union e1000_adv_rx_desc);
2573
2574         /* Round up to nearest 4K */
2575         rx_ring->size = rx_ring->count * desc_len;
2576         rx_ring->size = ALIGN(rx_ring->size, 4096);
2577
2578         rx_ring->desc = dma_alloc_coherent(dev,
2579                                            rx_ring->size,
2580                                            &rx_ring->dma,
2581                                            GFP_KERNEL);
2582
2583         if (!rx_ring->desc)
2584                 goto err;
2585
2586         rx_ring->next_to_clean = 0;
2587         rx_ring->next_to_use = 0;
2588
2589         return 0;
2590
2591 err:
2592         vfree(rx_ring->buffer_info);
2593         rx_ring->buffer_info = NULL;
2594         dev_err(dev, "Unable to allocate memory for the receive descriptor"
2595                 " ring\n");
2596         return -ENOMEM;
2597 }
2598
2599 /**
2600  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2601  *                                (Descriptors) for all queues
2602  * @adapter: board private structure
2603  *
2604  * Return 0 on success, negative on failure
2605  **/
2606 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2607 {
2608         struct pci_dev *pdev = adapter->pdev;
2609         int i, err = 0;
2610
2611         for (i = 0; i < adapter->num_rx_queues; i++) {
2612                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2613                 if (err) {
2614                         dev_err(&pdev->dev,
2615                                 "Allocation for Rx Queue %u failed\n", i);
2616                         for (i--; i >= 0; i--)
2617                                 igb_free_rx_resources(adapter->rx_ring[i]);
2618                         break;
2619                 }
2620         }
2621
2622         return err;
2623 }
2624
2625 /**
2626  * igb_setup_mrqc - configure the multiple receive queue control registers
2627  * @adapter: Board private structure
2628  **/
2629 static void igb_setup_mrqc(struct igb_adapter *adapter)
2630 {
2631         struct e1000_hw *hw = &adapter->hw;
2632         u32 mrqc, rxcsum;
2633         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2634         union e1000_reta {
2635                 u32 dword;
2636                 u8  bytes[4];
2637         } reta;
2638         static const u8 rsshash[40] = {
2639                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2640                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2641                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2642                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2643
2644         /* Fill out hash function seeds */
2645         for (j = 0; j < 10; j++) {
2646                 u32 rsskey = rsshash[(j * 4)];
2647                 rsskey |= rsshash[(j * 4) + 1] << 8;
2648                 rsskey |= rsshash[(j * 4) + 2] << 16;
2649                 rsskey |= rsshash[(j * 4) + 3] << 24;
2650                 array_wr32(E1000_RSSRK(0), j, rsskey);
2651         }
2652
2653         num_rx_queues = adapter->rss_queues;
2654
2655         if (adapter->vfs_allocated_count) {
2656                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2657                 switch (hw->mac.type) {
2658                 case e1000_i350:
2659                 case e1000_82580:
2660                         num_rx_queues = 1;
2661                         shift = 0;
2662                         break;
2663                 case e1000_82576:
2664                         shift = 3;
2665                         num_rx_queues = 2;
2666                         break;
2667                 case e1000_82575:
2668                         shift = 2;
2669                         shift2 = 6;
2670                 default:
2671                         break;
2672                 }
2673         } else {
2674                 if (hw->mac.type == e1000_82575)
2675                         shift = 6;
2676         }
2677
2678         for (j = 0; j < (32 * 4); j++) {
2679                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2680                 if (shift2)
2681                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2682                 if ((j & 3) == 3)
2683                         wr32(E1000_RETA(j >> 2), reta.dword);
2684         }
2685
2686         /*
2687          * Disable raw packet checksumming so that RSS hash is placed in
2688          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2689          * offloads as they are enabled by default
2690          */
2691         rxcsum = rd32(E1000_RXCSUM);
2692         rxcsum |= E1000_RXCSUM_PCSD;
2693
2694         if (adapter->hw.mac.type >= e1000_82576)
2695                 /* Enable Receive Checksum Offload for SCTP */
2696                 rxcsum |= E1000_RXCSUM_CRCOFL;
2697
2698         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2699         wr32(E1000_RXCSUM, rxcsum);
2700
2701         /* If VMDq is enabled then we set the appropriate mode for that, else
2702          * we default to RSS so that an RSS hash is calculated per packet even
2703          * if we are only using one queue */
2704         if (adapter->vfs_allocated_count) {
2705                 if (hw->mac.type > e1000_82575) {
2706                         /* Set the default pool for the PF's first queue */
2707                         u32 vtctl = rd32(E1000_VT_CTL);
2708                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2709                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2710                         vtctl |= adapter->vfs_allocated_count <<
2711                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2712                         wr32(E1000_VT_CTL, vtctl);
2713                 }
2714                 if (adapter->rss_queues > 1)
2715                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2716                 else
2717                         mrqc = E1000_MRQC_ENABLE_VMDQ;
2718         } else {
2719                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2720         }
2721         igb_vmm_control(adapter);
2722
2723         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2724                  E1000_MRQC_RSS_FIELD_IPV4_TCP);
2725         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2726                  E1000_MRQC_RSS_FIELD_IPV6_TCP);
2727         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2728                  E1000_MRQC_RSS_FIELD_IPV6_UDP);
2729         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2730                  E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2731
2732         wr32(E1000_MRQC, mrqc);
2733 }
2734
2735 /**
2736  * igb_setup_rctl - configure the receive control registers
2737  * @adapter: Board private structure
2738  **/
2739 void igb_setup_rctl(struct igb_adapter *adapter)
2740 {
2741         struct e1000_hw *hw = &adapter->hw;
2742         u32 rctl;
2743
2744         rctl = rd32(E1000_RCTL);
2745
2746         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2747         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2748
2749         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2750                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2751
2752         /*
2753          * enable stripping of CRC. It's unlikely this will break BMC
2754          * redirection as it did with e1000. Newer features require
2755          * that the HW strips the CRC.
2756          */
2757         rctl |= E1000_RCTL_SECRC;
2758
2759         /* disable store bad packets and clear size bits. */
2760         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2761
2762         /* enable LPE to prevent packets larger than max_frame_size */
2763         rctl |= E1000_RCTL_LPE;
2764
2765         /* disable queue 0 to prevent tail write w/o re-config */
2766         wr32(E1000_RXDCTL(0), 0);
2767
2768         /* Attention!!!  For SR-IOV PF driver operations you must enable
2769          * queue drop for all VF and PF queues to prevent head of line blocking
2770          * if an un-trusted VF does not provide descriptors to hardware.
2771          */
2772         if (adapter->vfs_allocated_count) {
2773                 /* set all queue drop enable bits */
2774                 wr32(E1000_QDE, ALL_QUEUES);
2775         }
2776
2777         wr32(E1000_RCTL, rctl);
2778 }
2779
2780 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2781                                    int vfn)
2782 {
2783         struct e1000_hw *hw = &adapter->hw;
2784         u32 vmolr;
2785
2786         /* if it isn't the PF check to see if VFs are enabled and
2787          * increase the size to support vlan tags */
2788         if (vfn < adapter->vfs_allocated_count &&
2789             adapter->vf_data[vfn].vlans_enabled)
2790                 size += VLAN_TAG_SIZE;
2791
2792         vmolr = rd32(E1000_VMOLR(vfn));
2793         vmolr &= ~E1000_VMOLR_RLPML_MASK;
2794         vmolr |= size | E1000_VMOLR_LPE;
2795         wr32(E1000_VMOLR(vfn), vmolr);
2796
2797         return 0;
2798 }
2799
2800 /**
2801  * igb_rlpml_set - set maximum receive packet size
2802  * @adapter: board private structure
2803  *
2804  * Configure maximum receivable packet size.
2805  **/
2806 static void igb_rlpml_set(struct igb_adapter *adapter)
2807 {
2808         u32 max_frame_size = adapter->max_frame_size;
2809         struct e1000_hw *hw = &adapter->hw;
2810         u16 pf_id = adapter->vfs_allocated_count;
2811
2812         if (adapter->vlgrp)
2813                 max_frame_size += VLAN_TAG_SIZE;
2814
2815         /* if vfs are enabled we set RLPML to the largest possible request
2816          * size and set the VMOLR RLPML to the size we need */
2817         if (pf_id) {
2818                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2819                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2820         }
2821
2822         wr32(E1000_RLPML, max_frame_size);
2823 }
2824
2825 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2826                                  int vfn, bool aupe)
2827 {
2828         struct e1000_hw *hw = &adapter->hw;
2829         u32 vmolr;
2830
2831         /*
2832          * This register exists only on 82576 and newer so if we are older then
2833          * we should exit and do nothing
2834          */
2835         if (hw->mac.type < e1000_82576)
2836                 return;
2837
2838         vmolr = rd32(E1000_VMOLR(vfn));
2839         vmolr |= E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
2840         if (aupe)
2841                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
2842         else
2843                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2844
2845         /* clear all bits that might not be set */
2846         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2847
2848         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2849                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2850         /*
2851          * for VMDq only allow the VFs and pool 0 to accept broadcast and
2852          * multicast packets
2853          */
2854         if (vfn <= adapter->vfs_allocated_count)
2855                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
2856
2857         wr32(E1000_VMOLR(vfn), vmolr);
2858 }
2859
2860 /**
2861  * igb_configure_rx_ring - Configure a receive ring after Reset
2862  * @adapter: board private structure
2863  * @ring: receive ring to be configured
2864  *
2865  * Configure the Rx unit of the MAC after a reset.
2866  **/
2867 void igb_configure_rx_ring(struct igb_adapter *adapter,
2868                            struct igb_ring *ring)
2869 {
2870         struct e1000_hw *hw = &adapter->hw;
2871         u64 rdba = ring->dma;
2872         int reg_idx = ring->reg_idx;
2873         u32 srrctl, rxdctl;
2874
2875         /* disable the queue */
2876         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2877         wr32(E1000_RXDCTL(reg_idx),
2878                         rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2879
2880         /* Set DMA base address registers */
2881         wr32(E1000_RDBAL(reg_idx),
2882              rdba & 0x00000000ffffffffULL);
2883         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2884         wr32(E1000_RDLEN(reg_idx),
2885                        ring->count * sizeof(union e1000_adv_rx_desc));
2886
2887         /* initialize head and tail */
2888         ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2889         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2890         writel(0, ring->head);
2891         writel(0, ring->tail);
2892
2893         /* set descriptor configuration */
2894         if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2895                 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2896                          E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2897 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2898                 srrctl |= IGB_RXBUFFER_16384 >>
2899                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2900 #else
2901                 srrctl |= (PAGE_SIZE / 2) >>
2902                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2903 #endif
2904                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2905         } else {
2906                 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2907                          E1000_SRRCTL_BSIZEPKT_SHIFT;
2908                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2909         }
2910         if (hw->mac.type == e1000_82580)
2911                 srrctl |= E1000_SRRCTL_TIMESTAMP;
2912         /* Only set Drop Enable if we are supporting multiple queues */
2913         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2914                 srrctl |= E1000_SRRCTL_DROP_EN;
2915
2916         wr32(E1000_SRRCTL(reg_idx), srrctl);
2917
2918         /* set filtering for VMDQ pools */
2919         igb_set_vmolr(adapter, reg_idx & 0x7, true);
2920
2921         /* enable receive descriptor fetching */
2922         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2923         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2924         rxdctl &= 0xFFF00000;
2925         rxdctl |= IGB_RX_PTHRESH;
2926         rxdctl |= IGB_RX_HTHRESH << 8;
2927         rxdctl |= IGB_RX_WTHRESH << 16;
2928         wr32(E1000_RXDCTL(reg_idx), rxdctl);
2929 }
2930
2931 /**
2932  * igb_configure_rx - Configure receive Unit after Reset
2933  * @adapter: board private structure
2934  *
2935  * Configure the Rx unit of the MAC after a reset.
2936  **/
2937 static void igb_configure_rx(struct igb_adapter *adapter)
2938 {
2939         int i;
2940
2941         /* set UTA to appropriate mode */
2942         igb_set_uta(adapter);
2943
2944         /* set the correct pool for the PF default MAC address in entry 0 */
2945         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2946                          adapter->vfs_allocated_count);
2947
2948         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2949          * the Base and Length of the Rx Descriptor Ring */
2950         for (i = 0; i < adapter->num_rx_queues; i++)
2951                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2952 }
2953
2954 /**
2955  * igb_free_tx_resources - Free Tx Resources per Queue
2956  * @tx_ring: Tx descriptor ring for a specific queue
2957  *
2958  * Free all transmit software resources
2959  **/
2960 void igb_free_tx_resources(struct igb_ring *tx_ring)
2961 {
2962         igb_clean_tx_ring(tx_ring);
2963
2964         vfree(tx_ring->buffer_info);
2965         tx_ring->buffer_info = NULL;
2966
2967         /* if not set, then don't free */
2968         if (!tx_ring->desc)
2969                 return;
2970
2971         dma_free_coherent(tx_ring->dev, tx_ring->size,
2972                           tx_ring->desc, tx_ring->dma);
2973
2974         tx_ring->desc = NULL;
2975 }
2976
2977 /**
2978  * igb_free_all_tx_resources - Free Tx Resources for All Queues
2979  * @adapter: board private structure
2980  *
2981  * Free all transmit software resources
2982  **/
2983 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2984 {
2985         int i;
2986
2987         for (i = 0; i < adapter->num_tx_queues; i++)
2988                 igb_free_tx_resources(adapter->tx_ring[i]);
2989 }
2990
2991 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2992                                     struct igb_buffer *buffer_info)
2993 {
2994         if (buffer_info->dma) {
2995                 if (buffer_info->mapped_as_page)
2996                         dma_unmap_page(tx_ring->dev,
2997                                         buffer_info->dma,
2998                                         buffer_info->length,
2999                                         DMA_TO_DEVICE);
3000                 else
3001                         dma_unmap_single(tx_ring->dev,
3002                                         buffer_info->dma,
3003                                         buffer_info->length,
3004                                         DMA_TO_DEVICE);
3005                 buffer_info->dma = 0;
3006         }
3007         if (buffer_info->skb) {
3008                 dev_kfree_skb_any(buffer_info->skb);
3009                 buffer_info->skb = NULL;
3010         }
3011         buffer_info->time_stamp = 0;
3012         buffer_info->length = 0;
3013         buffer_info->next_to_watch = 0;
3014         buffer_info->mapped_as_page = false;
3015 }
3016
3017 /**
3018  * igb_clean_tx_ring - Free Tx Buffers
3019  * @tx_ring: ring to be cleaned
3020  **/
3021 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3022 {
3023         struct igb_buffer *buffer_info;
3024         unsigned long size;
3025         unsigned int i;
3026
3027         if (!tx_ring->buffer_info)
3028                 return;
3029         /* Free all the Tx ring sk_buffs */
3030
3031         for (i = 0; i < tx_ring->count; i++) {
3032                 buffer_info = &tx_ring->buffer_info[i];
3033                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3034         }
3035
3036         size = sizeof(struct igb_buffer) * tx_ring->count;
3037         memset(tx_ring->buffer_info, 0, size);
3038
3039         /* Zero out the descriptor ring */
3040         memset(tx_ring->desc, 0, tx_ring->size);
3041
3042         tx_ring->next_to_use = 0;
3043         tx_ring->next_to_clean = 0;
3044 }
3045
3046 /**
3047  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3048  * @adapter: board private structure
3049  **/
3050 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3051 {
3052         int i;
3053
3054         for (i = 0; i < adapter->num_tx_queues; i++)
3055                 igb_clean_tx_ring(adapter->tx_ring[i]);
3056 }
3057
3058 /**
3059  * igb_free_rx_resources - Free Rx Resources
3060  * @rx_ring: ring to clean the resources from
3061  *
3062  * Free all receive software resources
3063  **/
3064 void igb_free_rx_resources(struct igb_ring *rx_ring)
3065 {
3066         igb_clean_rx_ring(rx_ring);
3067
3068         vfree(rx_ring->buffer_info);
3069         rx_ring->buffer_info = NULL;
3070
3071         /* if not set, then don't free */
3072         if (!rx_ring->desc)
3073                 return;
3074
3075         dma_free_coherent(rx_ring->dev, rx_ring->size,
3076                           rx_ring->desc, rx_ring->dma);
3077
3078         rx_ring->desc = NULL;
3079 }
3080
3081 /**
3082  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3083  * @adapter: board private structure
3084  *
3085  * Free all receive software resources
3086  **/
3087 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3088 {
3089         int i;
3090
3091         for (i = 0; i < adapter->num_rx_queues; i++)
3092                 igb_free_rx_resources(adapter->rx_ring[i]);
3093 }
3094
3095 /**
3096  * igb_clean_rx_ring - Free Rx Buffers per Queue
3097  * @rx_ring: ring to free buffers from
3098  **/
3099 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3100 {
3101         struct igb_buffer *buffer_info;
3102         unsigned long size;
3103         unsigned int i;
3104
3105         if (!rx_ring->buffer_info)
3106                 return;
3107
3108         /* Free all the Rx ring sk_buffs */
3109         for (i = 0; i < rx_ring->count; i++) {
3110                 buffer_info = &rx_ring->buffer_info[i];
3111                 if (buffer_info->dma) {
3112                         dma_unmap_single(rx_ring->dev,
3113                                          buffer_info->dma,
3114                                          rx_ring->rx_buffer_len,
3115                                          DMA_FROM_DEVICE);
3116                         buffer_info->dma = 0;
3117                 }
3118
3119                 if (buffer_info->skb) {
3120                         dev_kfree_skb(buffer_info->skb);
3121                         buffer_info->skb = NULL;
3122                 }
3123                 if (buffer_info->page_dma) {
3124                         dma_unmap_page(rx_ring->dev,
3125                                        buffer_info->page_dma,
3126                                        PAGE_SIZE / 2,
3127                                        DMA_FROM_DEVICE);
3128                         buffer_info->page_dma = 0;
3129                 }
3130                 if (buffer_info->page) {
3131                         put_page(buffer_info->page);
3132                         buffer_info->page = NULL;
3133                         buffer_info->page_offset = 0;
3134                 }
3135         }
3136
3137         size = sizeof(struct igb_buffer) * rx_ring->count;
3138         memset(rx_ring->buffer_info, 0, size);
3139
3140         /* Zero out the descriptor ring */
3141         memset(rx_ring->desc, 0, rx_ring->size);
3142
3143         rx_ring->next_to_clean = 0;
3144         rx_ring->next_to_use = 0;
3145 }
3146
3147 /**
3148  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3149  * @adapter: board private structure
3150  **/
3151 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3152 {
3153         int i;
3154
3155         for (i = 0; i < adapter->num_rx_queues; i++)
3156                 igb_clean_rx_ring(adapter->rx_ring[i]);
3157 }
3158
3159 /**
3160  * igb_set_mac - Change the Ethernet Address of the NIC
3161  * @netdev: network interface device structure
3162  * @p: pointer to an address structure
3163  *
3164  * Returns 0 on success, negative on failure
3165  **/
3166 static int igb_set_mac(struct net_device *netdev, void *p)
3167 {
3168         struct igb_adapter *adapter = netdev_priv(netdev);
3169         struct e1000_hw *hw = &adapter->hw;
3170         struct sockaddr *addr = p;
3171
3172         if (!is_valid_ether_addr(addr->sa_data))
3173                 return -EADDRNOTAVAIL;
3174
3175         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3176         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3177
3178         /* set the correct pool for the new PF MAC address in entry 0 */
3179         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3180                          adapter->vfs_allocated_count);
3181
3182         return 0;
3183 }
3184
3185 /**
3186  * igb_write_mc_addr_list - write multicast addresses to MTA
3187  * @netdev: network interface device structure
3188  *
3189  * Writes multicast address list to the MTA hash table.
3190  * Returns: -ENOMEM on failure
3191  *                0 on no addresses written
3192  *                X on writing X addresses to MTA
3193  **/
3194 static int igb_write_mc_addr_list(struct net_device *netdev)
3195 {
3196         struct igb_adapter *adapter = netdev_priv(netdev);
3197         struct e1000_hw *hw = &adapter->hw;
3198         struct netdev_hw_addr *ha;
3199         u8  *mta_list;
3200         int i;
3201
3202         if (netdev_mc_empty(netdev)) {
3203                 /* nothing to program, so clear mc list */
3204                 igb_update_mc_addr_list(hw, NULL, 0);
3205                 igb_restore_vf_multicasts(adapter);
3206                 return 0;
3207         }
3208
3209         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3210         if (!mta_list)
3211                 return -ENOMEM;
3212
3213         /* The shared function expects a packed array of only addresses. */
3214         i = 0;
3215         netdev_for_each_mc_addr(ha, netdev)
3216                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3217
3218         igb_update_mc_addr_list(hw, mta_list, i);
3219         kfree(mta_list);
3220
3221         return netdev_mc_count(netdev);
3222 }
3223
3224 /**
3225  * igb_write_uc_addr_list - write unicast addresses to RAR table
3226  * @netdev: network interface device structure
3227  *
3228  * Writes unicast address list to the RAR table.
3229  * Returns: -ENOMEM on failure/insufficient address space
3230  *                0 on no addresses written
3231  *                X on writing X addresses to the RAR table
3232  **/
3233 static int igb_write_uc_addr_list(struct net_device *netdev)
3234 {
3235         struct igb_adapter *adapter = netdev_priv(netdev);
3236         struct e1000_hw *hw = &adapter->hw;
3237         unsigned int vfn = adapter->vfs_allocated_count;
3238         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3239         int count = 0;
3240
3241         /* return ENOMEM indicating insufficient memory for addresses */
3242         if (netdev_uc_count(netdev) > rar_entries)
3243                 return -ENOMEM;
3244
3245         if (!netdev_uc_empty(netdev) && rar_entries) {
3246                 struct netdev_hw_addr *ha;
3247
3248                 netdev_for_each_uc_addr(ha, netdev) {
3249                         if (!rar_entries)
3250                                 break;
3251                         igb_rar_set_qsel(adapter, ha->addr,
3252                                          rar_entries--,
3253                                          vfn);
3254                         count++;
3255                 }
3256         }
3257         /* write the addresses in reverse order to avoid write combining */
3258         for (; rar_entries > 0 ; rar_entries--) {
3259                 wr32(E1000_RAH(rar_entries), 0);
3260                 wr32(E1000_RAL(rar_entries), 0);
3261         }
3262         wrfl();
3263
3264         return count;
3265 }
3266
3267 /**
3268  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3269  * @netdev: network interface device structure
3270  *
3271  * The set_rx_mode entry point is called whenever the unicast or multicast
3272  * address lists or the network interface flags are updated.  This routine is
3273  * responsible for configuring the hardware for proper unicast, multicast,
3274  * promiscuous mode, and all-multi behavior.
3275  **/
3276 static void igb_set_rx_mode(struct net_device *netdev)
3277 {
3278         struct igb_adapter *adapter = netdev_priv(netdev);
3279         struct e1000_hw *hw = &adapter->hw;
3280         unsigned int vfn = adapter->vfs_allocated_count;
3281         u32 rctl, vmolr = 0;
3282         int count;
3283
3284         /* Check for Promiscuous and All Multicast modes */
3285         rctl = rd32(E1000_RCTL);
3286
3287         /* clear the effected bits */
3288         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3289
3290         if (netdev->flags & IFF_PROMISC) {
3291                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3292                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3293         } else {
3294                 if (netdev->flags & IFF_ALLMULTI) {
3295                         rctl |= E1000_RCTL_MPE;
3296                         vmolr |= E1000_VMOLR_MPME;
3297                 } else {
3298                         /*
3299                          * Write addresses to the MTA, if the attempt fails
3300                          * then we should just turn on promiscous mode so
3301                          * that we can at least receive multicast traffic
3302                          */
3303                         count = igb_write_mc_addr_list(netdev);
3304                         if (count < 0) {
3305                                 rctl |= E1000_RCTL_MPE;
3306                                 vmolr |= E1000_VMOLR_MPME;
3307                         } else if (count) {
3308                                 vmolr |= E1000_VMOLR_ROMPE;
3309                         }
3310                 }
3311                 /*
3312                  * Write addresses to available RAR registers, if there is not
3313                  * sufficient space to store all the addresses then enable
3314                  * unicast promiscous mode
3315                  */
3316                 count = igb_write_uc_addr_list(netdev);
3317                 if (count < 0) {
3318                         rctl |= E1000_RCTL_UPE;
3319                         vmolr |= E1000_VMOLR_ROPE;
3320                 }
3321                 rctl |= E1000_RCTL_VFE;
3322         }
3323         wr32(E1000_RCTL, rctl);
3324
3325         /*
3326          * In order to support SR-IOV and eventually VMDq it is necessary to set
3327          * the VMOLR to enable the appropriate modes.  Without this workaround
3328          * we will have issues with VLAN tag stripping not being done for frames
3329          * that are only arriving because we are the default pool
3330          */
3331         if (hw->mac.type < e1000_82576)
3332                 return;
3333
3334         vmolr |= rd32(E1000_VMOLR(vfn)) &
3335                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3336         wr32(E1000_VMOLR(vfn), vmolr);
3337         igb_restore_vf_multicasts(adapter);
3338 }
3339
3340 /* Need to wait a few seconds after link up to get diagnostic information from
3341  * the phy */
3342 static void igb_update_phy_info(unsigned long data)
3343 {
3344         struct igb_adapter *adapter = (struct igb_adapter *) data;
3345         igb_get_phy_info(&adapter->hw);
3346 }
3347
3348 /**
3349  * igb_has_link - check shared code for link and determine up/down
3350  * @adapter: pointer to driver private info
3351  **/
3352 bool igb_has_link(struct igb_adapter *adapter)
3353 {
3354         struct e1000_hw *hw = &adapter->hw;
3355         bool link_active = false;
3356         s32 ret_val = 0;
3357
3358         /* get_link_status is set on LSC (link status) interrupt or
3359          * rx sequence error interrupt.  get_link_status will stay
3360          * false until the e1000_check_for_link establishes link
3361          * for copper adapters ONLY
3362          */
3363         switch (hw->phy.media_type) {
3364         case e1000_media_type_copper:
3365                 if (hw->mac.get_link_status) {
3366                         ret_val = hw->mac.ops.check_for_link(hw);
3367                         link_active = !hw->mac.get_link_status;
3368                 } else {
3369                         link_active = true;
3370                 }
3371                 break;
3372         case e1000_media_type_internal_serdes:
3373                 ret_val = hw->mac.ops.check_for_link(hw);
3374                 link_active = hw->mac.serdes_has_link;
3375                 break;
3376         default:
3377         case e1000_media_type_unknown:
3378                 break;
3379         }
3380
3381         return link_active;
3382 }
3383
3384 /**
3385  * igb_watchdog - Timer Call-back
3386  * @data: pointer to adapter cast into an unsigned long
3387  **/
3388 static void igb_watchdog(unsigned long data)
3389 {
3390         struct igb_adapter *adapter = (struct igb_adapter *)data;
3391         /* Do the rest outside of interrupt context */
3392         schedule_work(&adapter->watchdog_task);
3393 }
3394
3395 static void igb_watchdog_task(struct work_struct *work)
3396 {
3397         struct igb_adapter *adapter = container_of(work,
3398                                                    struct igb_adapter,
3399                                                    watchdog_task);
3400         struct e1000_hw *hw = &adapter->hw;
3401         struct net_device *netdev = adapter->netdev;
3402         u32 link;
3403         int i;
3404
3405         link = igb_has_link(adapter);
3406         if (link) {
3407                 if (!netif_carrier_ok(netdev)) {
3408                         u32 ctrl;
3409                         hw->mac.ops.get_speed_and_duplex(hw,
3410                                                          &adapter->link_speed,
3411                                                          &adapter->link_duplex);
3412
3413                         ctrl = rd32(E1000_CTRL);
3414                         /* Links status message must follow this format */
3415                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3416                                  "Flow Control: %s\n",
3417                                netdev->name,
3418                                adapter->link_speed,
3419                                adapter->link_duplex == FULL_DUPLEX ?
3420                                  "Full Duplex" : "Half Duplex",
3421                                ((ctrl & E1000_CTRL_TFCE) &&
3422                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3423                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
3424                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
3425
3426                         /* adjust timeout factor according to speed/duplex */
3427                         adapter->tx_timeout_factor = 1;
3428                         switch (adapter->link_speed) {
3429                         case SPEED_10:
3430                                 adapter->tx_timeout_factor = 14;
3431                                 break;
3432                         case SPEED_100:
3433                                 /* maybe add some timeout factor ? */
3434                                 break;
3435                         }
3436
3437                         netif_carrier_on(netdev);
3438
3439                         igb_ping_all_vfs(adapter);
3440
3441                         /* link state has changed, schedule phy info update */
3442                         if (!test_bit(__IGB_DOWN, &adapter->state))
3443                                 mod_timer(&adapter->phy_info_timer,
3444                                           round_jiffies(jiffies + 2 * HZ));
3445                 }
3446         } else {
3447                 if (netif_carrier_ok(netdev)) {
3448                         adapter->link_speed = 0;
3449                         adapter->link_duplex = 0;
3450                         /* Links status message must follow this format */
3451                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3452                                netdev->name);
3453                         netif_carrier_off(netdev);
3454
3455                         igb_ping_all_vfs(adapter);
3456
3457                         /* link state has changed, schedule phy info update */
3458                         if (!test_bit(__IGB_DOWN, &adapter->state))
3459                                 mod_timer(&adapter->phy_info_timer,
3460                                           round_jiffies(jiffies + 2 * HZ));
3461                 }
3462         }
3463
3464         igb_update_stats(adapter);
3465
3466         for (i = 0; i < adapter->num_tx_queues; i++) {
3467                 struct igb_ring *tx_ring = adapter->tx_ring[i];
3468                 if (!netif_carrier_ok(netdev)) {
3469                         /* We've lost link, so the controller stops DMA,
3470                          * but we've got queued Tx work that's never going
3471                          * to get done, so reset controller to flush Tx.
3472                          * (Do the reset outside of interrupt context). */
3473                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3474                                 adapter->tx_timeout_count++;
3475                                 schedule_work(&adapter->reset_task);
3476                                 /* return immediately since reset is imminent */
3477                                 return;
3478                         }
3479                 }
3480
3481                 /* Force detection of hung controller every watchdog period */
3482                 tx_ring->detect_tx_hung = true;
3483         }
3484
3485         /* Cause software interrupt to ensure rx ring is cleaned */
3486         if (adapter->msix_entries) {
3487                 u32 eics = 0;
3488                 for (i = 0; i < adapter->num_q_vectors; i++) {
3489                         struct igb_q_vector *q_vector = adapter->q_vector[i];
3490                         eics |= q_vector->eims_value;
3491                 }
3492                 wr32(E1000_EICS, eics);
3493         } else {
3494                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3495         }
3496
3497         /* Reset the timer */
3498         if (!test_bit(__IGB_DOWN, &adapter->state))
3499                 mod_timer(&adapter->watchdog_timer,
3500                           round_jiffies(jiffies + 2 * HZ));
3501 }
3502
3503 enum latency_range {
3504         lowest_latency = 0,
3505         low_latency = 1,
3506         bulk_latency = 2,
3507         latency_invalid = 255
3508 };
3509
3510 /**
3511  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3512  *
3513  *      Stores a new ITR value based on strictly on packet size.  This
3514  *      algorithm is less sophisticated than that used in igb_update_itr,
3515  *      due to the difficulty of synchronizing statistics across multiple
3516  *      receive rings.  The divisors and thresholds used by this fuction
3517  *      were determined based on theoretical maximum wire speed and testing
3518  *      data, in order to minimize response time while increasing bulk
3519  *      throughput.
3520  *      This functionality is controlled by the InterruptThrottleRate module
3521  *      parameter (see igb_param.c)
3522  *      NOTE:  This function is called only when operating in a multiqueue
3523  *             receive environment.
3524  * @q_vector: pointer to q_vector
3525  **/
3526 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3527 {
3528         int new_val = q_vector->itr_val;
3529         int avg_wire_size = 0;
3530         struct igb_adapter *adapter = q_vector->adapter;
3531
3532         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3533          * ints/sec - ITR timer value of 120 ticks.
3534          */
3535         if (adapter->link_speed != SPEED_1000) {
3536                 new_val = 976;
3537                 goto set_itr_val;
3538         }
3539
3540         if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3541                 struct igb_ring *ring = q_vector->rx_ring;
3542                 avg_wire_size = ring->total_bytes / ring->total_packets;
3543         }
3544
3545         if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3546                 struct igb_ring *ring = q_vector->tx_ring;
3547                 avg_wire_size = max_t(u32, avg_wire_size,
3548                                       (ring->total_bytes /
3549                                        ring->total_packets));
3550         }
3551
3552         /* if avg_wire_size isn't set no work was done */
3553         if (!avg_wire_size)
3554                 goto clear_counts;
3555
3556         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3557         avg_wire_size += 24;
3558
3559         /* Don't starve jumbo frames */
3560         avg_wire_size = min(avg_wire_size, 3000);
3561
3562         /* Give a little boost to mid-size frames */
3563         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3564                 new_val = avg_wire_size / 3;
3565         else
3566                 new_val = avg_wire_size / 2;
3567
3568         /* when in itr mode 3 do not exceed 20K ints/sec */
3569         if (adapter->rx_itr_setting == 3 && new_val < 196)
3570                 new_val = 196;
3571
3572 set_itr_val:
3573         if (new_val != q_vector->itr_val) {
3574                 q_vector->itr_val = new_val;
3575                 q_vector->set_itr = 1;
3576         }
3577 clear_counts:
3578         if (q_vector->rx_ring) {
3579                 q_vector->rx_ring->total_bytes = 0;
3580                 q_vector->rx_ring->total_packets = 0;
3581         }
3582         if (q_vector->tx_ring) {
3583                 q_vector->tx_ring->total_bytes = 0;
3584                 q_vector->tx_ring->total_packets = 0;
3585         }
3586 }
3587
3588 /**
3589  * igb_update_itr - update the dynamic ITR value based on statistics
3590  *      Stores a new ITR value based on packets and byte
3591  *      counts during the last interrupt.  The advantage of per interrupt
3592  *      computation is faster updates and more accurate ITR for the current
3593  *      traffic pattern.  Constants in this function were computed
3594  *      based on theoretical maximum wire speed and thresholds were set based
3595  *      on testing data as well as attempting to minimize response time
3596  *      while increasing bulk throughput.
3597  *      this functionality is controlled by the InterruptThrottleRate module
3598  *      parameter (see igb_param.c)
3599  *      NOTE:  These calculations are only valid when operating in a single-
3600  *             queue environment.
3601  * @adapter: pointer to adapter
3602  * @itr_setting: current q_vector->itr_val
3603  * @packets: the number of packets during this measurement interval
3604  * @bytes: the number of bytes during this measurement interval
3605  **/
3606 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3607                                    int packets, int bytes)
3608 {
3609         unsigned int retval = itr_setting;
3610
3611         if (packets == 0)
3612                 goto update_itr_done;
3613
3614         switch (itr_setting) {
3615         case lowest_latency:
3616                 /* handle TSO and jumbo frames */
3617                 if (bytes/packets > 8000)
3618                         retval = bulk_latency;
3619                 else if ((packets < 5) && (bytes > 512))
3620                         retval = low_latency;
3621                 break;
3622         case low_latency:  /* 50 usec aka 20000 ints/s */
3623                 if (bytes > 10000) {
3624                         /* this if handles the TSO accounting */
3625                         if (bytes/packets > 8000) {
3626                                 retval = bulk_latency;
3627                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3628                                 retval = bulk_latency;
3629                         } else if ((packets > 35)) {
3630                                 retval = lowest_latency;
3631                         }
3632                 } else if (bytes/packets > 2000) {
3633                         retval = bulk_latency;
3634                 } else if (packets <= 2 && bytes < 512) {
3635                         retval = lowest_latency;
3636                 }
3637                 break;
3638         case bulk_latency: /* 250 usec aka 4000 ints/s */
3639                 if (bytes > 25000) {
3640                         if (packets > 35)
3641                                 retval = low_latency;
3642                 } else if (bytes < 1500) {
3643                         retval = low_latency;
3644                 }
3645                 break;
3646         }
3647
3648 update_itr_done:
3649         return retval;
3650 }
3651
3652 static void igb_set_itr(struct igb_adapter *adapter)
3653 {
3654         struct igb_q_vector *q_vector = adapter->q_vector[0];
3655         u16 current_itr;
3656         u32 new_itr = q_vector->itr_val;
3657
3658         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3659         if (adapter->link_speed != SPEED_1000) {
3660                 current_itr = 0;
3661                 new_itr = 4000;
3662                 goto set_itr_now;
3663         }
3664
3665         adapter->rx_itr = igb_update_itr(adapter,
3666                                     adapter->rx_itr,
3667                                     q_vector->rx_ring->total_packets,
3668                                     q_vector->rx_ring->total_bytes);
3669
3670         adapter->tx_itr = igb_update_itr(adapter,
3671                                     adapter->tx_itr,
3672                                     q_vector->tx_ring->total_packets,
3673                                     q_vector->tx_ring->total_bytes);
3674         current_itr = max(adapter->rx_itr, adapter->tx_itr);
3675
3676         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3677         if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3678                 current_itr = low_latency;
3679
3680         switch (current_itr) {
3681         /* counts and packets in update_itr are dependent on these numbers */
3682         case lowest_latency:
3683                 new_itr = 56;  /* aka 70,000 ints/sec */
3684                 break;
3685         case low_latency:
3686                 new_itr = 196; /* aka 20,000 ints/sec */
3687                 break;
3688         case bulk_latency:
3689                 new_itr = 980; /* aka 4,000 ints/sec */
3690                 break;
3691         default:
3692                 break;
3693         }
3694
3695 set_itr_now:
3696         q_vector->rx_ring->total_bytes = 0;
3697         q_vector->rx_ring->total_packets = 0;
3698         q_vector->tx_ring->total_bytes = 0;
3699         q_vector->tx_ring->total_packets = 0;
3700
3701         if (new_itr != q_vector->itr_val) {
3702                 /* this attempts to bias the interrupt rate towards Bulk
3703                  * by adding intermediate steps when interrupt rate is
3704                  * increasing */
3705                 new_itr = new_itr > q_vector->itr_val ?
3706                              max((new_itr * q_vector->itr_val) /
3707                                  (new_itr + (q_vector->itr_val >> 2)),
3708                                  new_itr) :
3709                              new_itr;
3710                 /* Don't write the value here; it resets the adapter's
3711                  * internal timer, and causes us to delay far longer than
3712                  * we should between interrupts.  Instead, we write the ITR
3713                  * value at the beginning of the next interrupt so the timing
3714                  * ends up being correct.
3715                  */
3716                 q_vector->itr_val = new_itr;
3717                 q_vector->set_itr = 1;
3718         }
3719 }
3720
3721 #define IGB_TX_FLAGS_CSUM               0x00000001
3722 #define IGB_TX_FLAGS_VLAN               0x00000002
3723 #define IGB_TX_FLAGS_TSO                0x00000004
3724 #define IGB_TX_FLAGS_IPV4               0x00000008
3725 #define IGB_TX_FLAGS_TSTAMP             0x00000010
3726 #define IGB_TX_FLAGS_VLAN_MASK          0xffff0000
3727 #define IGB_TX_FLAGS_VLAN_SHIFT                 16
3728
3729 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3730                               struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3731 {
3732         struct e1000_adv_tx_context_desc *context_desc;
3733         unsigned int i;
3734         int err;
3735         struct igb_buffer *buffer_info;
3736         u32 info = 0, tu_cmd = 0;
3737         u32 mss_l4len_idx;
3738         u8 l4len;
3739
3740         if (skb_header_cloned(skb)) {
3741                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3742                 if (err)
3743                         return err;
3744         }
3745
3746         l4len = tcp_hdrlen(skb);
3747         *hdr_len += l4len;
3748
3749         if (skb->protocol == htons(ETH_P_IP)) {
3750                 struct iphdr *iph = ip_hdr(skb);
3751                 iph->tot_len = 0;
3752                 iph->check = 0;
3753                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3754                                                          iph->daddr, 0,
3755                                                          IPPROTO_TCP,
3756                                                          0);
3757         } else if (skb_is_gso_v6(skb)) {
3758                 ipv6_hdr(skb)->payload_len = 0;
3759                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3760                                                        &ipv6_hdr(skb)->daddr,
3761                                                        0, IPPROTO_TCP, 0);
3762         }
3763
3764         i = tx_ring->next_to_use;
3765
3766         buffer_info = &tx_ring->buffer_info[i];
3767         context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3768         /* VLAN MACLEN IPLEN */
3769         if (tx_flags & IGB_TX_FLAGS_VLAN)
3770                 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3771         info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3772         *hdr_len += skb_network_offset(skb);
3773         info |= skb_network_header_len(skb);
3774         *hdr_len += skb_network_header_len(skb);
3775         context_desc->vlan_macip_lens = cpu_to_le32(info);
3776
3777         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3778         tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3779
3780         if (skb->protocol == htons(ETH_P_IP))
3781                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3782         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3783
3784         context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3785
3786         /* MSS L4LEN IDX */
3787         mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3788         mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3789
3790         /* For 82575, context index must be unique per ring. */
3791         if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3792                 mss_l4len_idx |= tx_ring->reg_idx << 4;
3793
3794         context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3795         context_desc->seqnum_seed = 0;
3796
3797         buffer_info->time_stamp = jiffies;
3798         buffer_info->next_to_watch = i;
3799         buffer_info->dma = 0;
3800         i++;
3801         if (i == tx_ring->count)
3802                 i = 0;
3803
3804         tx_ring->next_to_use = i;
3805
3806         return true;
3807 }
3808
3809 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3810                                    struct sk_buff *skb, u32 tx_flags)
3811 {
3812         struct e1000_adv_tx_context_desc *context_desc;
3813         struct device *dev = tx_ring->dev;
3814         struct igb_buffer *buffer_info;
3815         u32 info = 0, tu_cmd = 0;
3816         unsigned int i;
3817
3818         if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3819             (tx_flags & IGB_TX_FLAGS_VLAN)) {
3820                 i = tx_ring->next_to_use;
3821                 buffer_info = &tx_ring->buffer_info[i];
3822                 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3823
3824                 if (tx_flags & IGB_TX_FLAGS_VLAN)
3825                         info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3826
3827                 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3828                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3829                         info |= skb_network_header_len(skb);
3830
3831                 context_desc->vlan_macip_lens = cpu_to_le32(info);
3832
3833                 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3834
3835                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3836                         __be16 protocol;
3837
3838                         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3839                                 const struct vlan_ethhdr *vhdr =
3840                                           (const struct vlan_ethhdr*)skb->data;
3841
3842                                 protocol = vhdr->h_vlan_encapsulated_proto;
3843                         } else {
3844                                 protocol = skb->protocol;
3845                         }
3846
3847                         switch (protocol) {
3848                         case cpu_to_be16(ETH_P_IP):
3849                                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3850                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3851                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3852                                 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3853                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3854                                 break;
3855                         case cpu_to_be16(ETH_P_IPV6):
3856                                 /* XXX what about other V6 headers?? */
3857                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3858                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3859                                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3860                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3861                                 break;
3862                         default:
3863                                 if (unlikely(net_ratelimit()))
3864                                         dev_warn(dev,
3865                                             "partial checksum but proto=%x!\n",
3866                                             skb->protocol);
3867                                 break;
3868                         }
3869                 }
3870
3871                 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3872                 context_desc->seqnum_seed = 0;
3873                 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3874                         context_desc->mss_l4len_idx =
3875                                 cpu_to_le32(tx_ring->reg_idx << 4);
3876
3877                 buffer_info->time_stamp = jiffies;
3878                 buffer_info->next_to_watch = i;
3879                 buffer_info->dma = 0;
3880
3881                 i++;
3882                 if (i == tx_ring->count)
3883                         i = 0;
3884                 tx_ring->next_to_use = i;
3885
3886                 return true;
3887         }
3888         return false;
3889 }
3890
3891 #define IGB_MAX_TXD_PWR 16
3892 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
3893
3894 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3895                                  unsigned int first)
3896 {
3897         struct igb_buffer *buffer_info;
3898         struct device *dev = tx_ring->dev;
3899         unsigned int hlen = skb_headlen(skb);
3900         unsigned int count = 0, i;
3901         unsigned int f;
3902         u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3903
3904         i = tx_ring->next_to_use;
3905
3906         buffer_info = &tx_ring->buffer_info[i];
3907         BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3908         buffer_info->length = hlen;
3909         /* set time_stamp *before* dma to help avoid a possible race */
3910         buffer_info->time_stamp = jiffies;
3911         buffer_info->next_to_watch = i;
3912         buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3913                                           DMA_TO_DEVICE);
3914         if (dma_mapping_error(dev, buffer_info->dma))
3915                 goto dma_error;
3916
3917         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3918                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3919                 unsigned int len = frag->size;
3920
3921                 count++;
3922                 i++;
3923                 if (i == tx_ring->count)
3924                         i = 0;
3925
3926                 buffer_info = &tx_ring->buffer_info[i];
3927                 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3928                 buffer_info->length = len;
3929                 buffer_info->time_stamp = jiffies;
3930                 buffer_info->next_to_watch = i;
3931                 buffer_info->mapped_as_page = true;
3932                 buffer_info->dma = dma_map_page(dev,
3933                                                 frag->page,
3934                                                 frag->page_offset,
3935                                                 len,
3936                                                 DMA_TO_DEVICE);
3937                 if (dma_mapping_error(dev, buffer_info->dma))
3938                         goto dma_error;
3939
3940         }
3941
3942         tx_ring->buffer_info[i].skb = skb;
3943         tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3944         /* multiply data chunks by size of headers */
3945         tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3946         tx_ring->buffer_info[i].gso_segs = gso_segs;
3947         tx_ring->buffer_info[first].next_to_watch = i;
3948
3949         return ++count;
3950
3951 dma_error:
3952         dev_err(dev, "TX DMA map failed\n");
3953
3954         /* clear timestamp and dma mappings for failed buffer_info mapping */
3955         buffer_info->dma = 0;
3956         buffer_info->time_stamp = 0;
3957         buffer_info->length = 0;
3958         buffer_info->next_to_watch = 0;
3959         buffer_info->mapped_as_page = false;
3960
3961         /* clear timestamp and dma mappings for remaining portion of packet */
3962         while (count--) {
3963                 if (i == 0)
3964                         i = tx_ring->count;
3965                 i--;
3966                 buffer_info = &tx_ring->buffer_info[i];
3967                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3968         }
3969
3970         return 0;
3971 }
3972
3973 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3974                                     u32 tx_flags, int count, u32 paylen,
3975                                     u8 hdr_len)
3976 {
3977         union e1000_adv_tx_desc *tx_desc;
3978         struct igb_buffer *buffer_info;
3979         u32 olinfo_status = 0, cmd_type_len;
3980         unsigned int i = tx_ring->next_to_use;
3981
3982         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3983                         E1000_ADVTXD_DCMD_DEXT);
3984
3985         if (tx_flags & IGB_TX_FLAGS_VLAN)
3986                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3987
3988         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3989                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3990
3991         if (tx_flags & IGB_TX_FLAGS_TSO) {
3992                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3993
3994                 /* insert tcp checksum */
3995                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3996
3997                 /* insert ip checksum */
3998                 if (tx_flags & IGB_TX_FLAGS_IPV4)
3999                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4000
4001         } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4002                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4003         }
4004
4005         if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4006             (tx_flags & (IGB_TX_FLAGS_CSUM |
4007                          IGB_TX_FLAGS_TSO |
4008                          IGB_TX_FLAGS_VLAN)))
4009                 olinfo_status |= tx_ring->reg_idx << 4;
4010
4011         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4012
4013         do {
4014                 buffer_info = &tx_ring->buffer_info[i];
4015                 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4016                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4017                 tx_desc->read.cmd_type_len =
4018                         cpu_to_le32(cmd_type_len | buffer_info->length);
4019                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4020                 count--;
4021                 i++;
4022                 if (i == tx_ring->count)
4023                         i = 0;
4024         } while (count > 0);
4025
4026         tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4027         /* Force memory writes to complete before letting h/w
4028          * know there are new descriptors to fetch.  (Only
4029          * applicable for weak-ordered memory model archs,
4030          * such as IA-64). */
4031         wmb();
4032
4033         tx_ring->next_to_use = i;
4034         writel(i, tx_ring->tail);
4035         /* we need this if more than one processor can write to our tail
4036          * at a time, it syncronizes IO on IA64/Altix systems */
4037         mmiowb();
4038 }
4039
4040 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4041 {
4042         struct net_device *netdev = tx_ring->netdev;
4043
4044         netif_stop_subqueue(netdev, tx_ring->queue_index);
4045
4046         /* Herbert's original patch had:
4047          *  smp_mb__after_netif_stop_queue();
4048          * but since that doesn't exist yet, just open code it. */
4049         smp_mb();
4050
4051         /* We need to check again in a case another CPU has just
4052          * made room available. */
4053         if (igb_desc_unused(tx_ring) < size)
4054                 return -EBUSY;
4055
4056         /* A reprieve! */
4057         netif_wake_subqueue(netdev, tx_ring->queue_index);
4058         tx_ring->tx_stats.restart_queue++;
4059         return 0;
4060 }
4061
4062 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4063 {
4064         if (igb_desc_unused(tx_ring) >= size)
4065                 return 0;
4066         return __igb_maybe_stop_tx(tx_ring, size);
4067 }
4068
4069 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4070                                     struct igb_ring *tx_ring)
4071 {
4072         struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4073         int tso = 0, count;
4074         u32 tx_flags = 0;
4075         u16 first;
4076         u8 hdr_len = 0;
4077         union skb_shared_tx *shtx = skb_tx(skb);
4078
4079         /* need: 1 descriptor per page,
4080          *       + 2 desc gap to keep tail from touching head,
4081          *       + 1 desc for skb->data,
4082          *       + 1 desc for context descriptor,
4083          * otherwise try next time */
4084         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4085                 /* this is a hard error */
4086                 return NETDEV_TX_BUSY;
4087         }
4088
4089         if (unlikely(shtx->hardware)) {
4090                 shtx->in_progress = 1;
4091                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4092         }
4093
4094         if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
4095                 tx_flags |= IGB_TX_FLAGS_VLAN;
4096                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4097         }
4098
4099         if (skb->protocol == htons(ETH_P_IP))
4100                 tx_flags |= IGB_TX_FLAGS_IPV4;
4101
4102         first = tx_ring->next_to_use;
4103         if (skb_is_gso(skb)) {
4104                 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4105
4106                 if (tso < 0) {
4107                         dev_kfree_skb_any(skb);
4108                         return NETDEV_TX_OK;
4109                 }
4110         }
4111
4112         if (tso)
4113                 tx_flags |= IGB_TX_FLAGS_TSO;
4114         else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4115                  (skb->ip_summed == CHECKSUM_PARTIAL))
4116                 tx_flags |= IGB_TX_FLAGS_CSUM;
4117
4118         /*
4119          * count reflects descriptors mapped, if 0 or less then mapping error
4120          * has occured and we need to rewind the descriptor queue
4121          */
4122         count = igb_tx_map_adv(tx_ring, skb, first);
4123         if (!count) {
4124                 dev_kfree_skb_any(skb);
4125                 tx_ring->buffer_info[first].time_stamp = 0;
4126                 tx_ring->next_to_use = first;
4127                 return NETDEV_TX_OK;
4128         }
4129
4130         igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4131
4132         /* Make sure there is space in the ring for the next send. */
4133         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4134
4135         return NETDEV_TX_OK;
4136 }
4137
4138 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4139                                       struct net_device *netdev)
4140 {
4141         struct igb_adapter *adapter = netdev_priv(netdev);
4142         struct igb_ring *tx_ring;
4143         int r_idx = 0;
4144
4145         if (test_bit(__IGB_DOWN, &adapter->state)) {
4146                 dev_kfree_skb_any(skb);
4147                 return NETDEV_TX_OK;
4148         }
4149
4150         if (skb->len <= 0) {
4151                 dev_kfree_skb_any(skb);
4152                 return NETDEV_TX_OK;
4153         }
4154
4155         r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4156         tx_ring = adapter->multi_tx_table[r_idx];
4157
4158         /* This goes back to the question of how to logically map a tx queue
4159          * to a flow.  Right now, performance is impacted slightly negatively
4160          * if using multiple tx queues.  If the stack breaks away from a
4161          * single qdisc implementation, we can look at this again. */
4162         return igb_xmit_frame_ring_adv(skb, tx_ring);
4163 }
4164
4165 /**
4166  * igb_tx_timeout - Respond to a Tx Hang
4167  * @netdev: network interface device structure
4168  **/
4169 static void igb_tx_timeout(struct net_device *netdev)
4170 {
4171         struct igb_adapter *adapter = netdev_priv(netdev);
4172         struct e1000_hw *hw = &adapter->hw;
4173
4174         /* Do the reset outside of interrupt context */
4175         adapter->tx_timeout_count++;
4176
4177         if (hw->mac.type == e1000_82580)
4178                 hw->dev_spec._82575.global_device_reset = true;
4179
4180         schedule_work(&adapter->reset_task);
4181         wr32(E1000_EICS,
4182              (adapter->eims_enable_mask & ~adapter->eims_other));
4183 }
4184
4185 static void igb_reset_task(struct work_struct *work)
4186 {
4187         struct igb_adapter *adapter;
4188         adapter = container_of(work, struct igb_adapter, reset_task);
4189
4190         igb_dump(adapter);
4191         netdev_err(adapter->netdev, "Reset adapter\n");
4192         igb_reinit_locked(adapter);
4193 }
4194
4195 /**
4196  * igb_get_stats - Get System Network Statistics
4197  * @netdev: network interface device structure
4198  *
4199  * Returns the address of the device statistics structure.
4200  * The statistics are actually updated from the timer callback.
4201  **/
4202 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4203 {
4204         /* only return the current stats */
4205         return &netdev->stats;
4206 }
4207
4208 /**
4209  * igb_change_mtu - Change the Maximum Transfer Unit
4210  * @netdev: network interface device structure
4211  * @new_mtu: new value for maximum frame size
4212  *
4213  * Returns 0 on success, negative on failure
4214  **/
4215 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4216 {
4217         struct igb_adapter *adapter = netdev_priv(netdev);
4218         struct pci_dev *pdev = adapter->pdev;
4219         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4220         u32 rx_buffer_len, i;
4221
4222         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4223                 dev_err(&pdev->dev, "Invalid MTU setting\n");
4224                 return -EINVAL;
4225         }
4226
4227         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4228                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4229                 return -EINVAL;
4230         }
4231
4232         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4233                 msleep(1);
4234
4235         /* igb_down has a dependency on max_frame_size */
4236         adapter->max_frame_size = max_frame;
4237
4238         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4239          * means we reserve 2 more, this pushes us to allocate from the next
4240          * larger slab size.
4241          * i.e. RXBUFFER_2048 --> size-4096 slab
4242          */
4243
4244         if (adapter->hw.mac.type == e1000_82580)
4245                 max_frame += IGB_TS_HDR_LEN;
4246
4247         if (max_frame <= IGB_RXBUFFER_1024)
4248                 rx_buffer_len = IGB_RXBUFFER_1024;
4249         else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4250                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4251         else
4252                 rx_buffer_len = IGB_RXBUFFER_128;
4253
4254         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4255              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4256                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4257
4258         if ((adapter->hw.mac.type == e1000_82580) &&
4259             (rx_buffer_len == IGB_RXBUFFER_128))
4260                 rx_buffer_len += IGB_RXBUFFER_64;
4261
4262         if (netif_running(netdev))
4263                 igb_down(adapter);
4264
4265         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4266                  netdev->mtu, new_mtu);
4267         netdev->mtu = new_mtu;
4268
4269         for (i = 0; i < adapter->num_rx_queues; i++)
4270                 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4271
4272         if (netif_running(netdev))
4273                 igb_up(adapter);
4274         else
4275                 igb_reset(adapter);
4276
4277         clear_bit(__IGB_RESETTING, &adapter->state);
4278
4279         return 0;
4280 }
4281
4282 /**
4283  * igb_update_stats - Update the board statistics counters
4284  * @adapter: board private structure
4285  **/
4286
4287 void igb_update_stats(struct igb_adapter *adapter)
4288 {
4289         struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
4290         struct e1000_hw *hw = &adapter->hw;
4291         struct pci_dev *pdev = adapter->pdev;
4292         u32 reg, mpc;
4293         u16 phy_tmp;
4294         int i;
4295         u64 bytes, packets;
4296
4297 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4298
4299         /*
4300          * Prevent stats update while adapter is being reset, or if the pci
4301          * connection is down.
4302          */
4303         if (adapter->link_speed == 0)
4304                 return;
4305         if (pci_channel_offline(pdev))
4306                 return;
4307
4308         bytes = 0;
4309         packets = 0;
4310         for (i = 0; i < adapter->num_rx_queues; i++) {
4311                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4312                 struct igb_ring *ring = adapter->rx_ring[i];
4313                 ring->rx_stats.drops += rqdpc_tmp;
4314                 net_stats->rx_fifo_errors += rqdpc_tmp;
4315                 bytes += ring->rx_stats.bytes;
4316                 packets += ring->rx_stats.packets;
4317         }
4318
4319         net_stats->rx_bytes = bytes;
4320         net_stats->rx_packets = packets;
4321
4322         bytes = 0;
4323         packets = 0;
4324         for (i = 0; i < adapter->num_tx_queues; i++) {
4325                 struct igb_ring *ring = adapter->tx_ring[i];
4326                 bytes += ring->tx_stats.bytes;
4327                 packets += ring->tx_stats.packets;
4328         }
4329         net_stats->tx_bytes = bytes;
4330         net_stats->tx_packets = packets;
4331
4332         /* read stats registers */
4333         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4334         adapter->stats.gprc += rd32(E1000_GPRC);
4335         adapter->stats.gorc += rd32(E1000_GORCL);
4336         rd32(E1000_GORCH); /* clear GORCL */
4337         adapter->stats.bprc += rd32(E1000_BPRC);
4338         adapter->stats.mprc += rd32(E1000_MPRC);
4339         adapter->stats.roc += rd32(E1000_ROC);
4340
4341         adapter->stats.prc64 += rd32(E1000_PRC64);
4342         adapter->stats.prc127 += rd32(E1000_PRC127);
4343         adapter->stats.prc255 += rd32(E1000_PRC255);
4344         adapter->stats.prc511 += rd32(E1000_PRC511);
4345         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4346         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4347         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4348         adapter->stats.sec += rd32(E1000_SEC);
4349
4350         mpc = rd32(E1000_MPC);
4351         adapter->stats.mpc += mpc;
4352         net_stats->rx_fifo_errors += mpc;
4353         adapter->stats.scc += rd32(E1000_SCC);
4354         adapter->stats.ecol += rd32(E1000_ECOL);
4355         adapter->stats.mcc += rd32(E1000_MCC);
4356         adapter->stats.latecol += rd32(E1000_LATECOL);
4357         adapter->stats.dc += rd32(E1000_DC);
4358         adapter->stats.rlec += rd32(E1000_RLEC);
4359         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4360         adapter->stats.xontxc += rd32(E1000_XONTXC);
4361         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4362         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4363         adapter->stats.fcruc += rd32(E1000_FCRUC);
4364         adapter->stats.gptc += rd32(E1000_GPTC);
4365         adapter->stats.gotc += rd32(E1000_GOTCL);
4366         rd32(E1000_GOTCH); /* clear GOTCL */
4367         adapter->stats.rnbc += rd32(E1000_RNBC);
4368         adapter->stats.ruc += rd32(E1000_RUC);
4369         adapter->stats.rfc += rd32(E1000_RFC);
4370         adapter->stats.rjc += rd32(E1000_RJC);
4371         adapter->stats.tor += rd32(E1000_TORH);
4372         adapter->stats.tot += rd32(E1000_TOTH);
4373         adapter->stats.tpr += rd32(E1000_TPR);
4374
4375         adapter->stats.ptc64 += rd32(E1000_PTC64);
4376         adapter->stats.ptc127 += rd32(E1000_PTC127);
4377         adapter->stats.ptc255 += rd32(E1000_PTC255);
4378         adapter->stats.ptc511 += rd32(E1000_PTC511);
4379         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4380         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4381
4382         adapter->stats.mptc += rd32(E1000_MPTC);
4383         adapter->stats.bptc += rd32(E1000_BPTC);
4384
4385         adapter->stats.tpt += rd32(E1000_TPT);
4386         adapter->stats.colc += rd32(E1000_COLC);
4387
4388         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4389         /* read internal phy specific stats */
4390         reg = rd32(E1000_CTRL_EXT);
4391         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4392                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4393                 adapter->stats.tncrs += rd32(E1000_TNCRS);
4394         }
4395
4396         adapter->stats.tsctc += rd32(E1000_TSCTC);
4397         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4398
4399         adapter->stats.iac += rd32(E1000_IAC);
4400         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4401         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4402         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4403         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4404         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4405         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4406         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4407         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4408
4409         /* Fill out the OS statistics structure */
4410         net_stats->multicast = adapter->stats.mprc;
4411         net_stats->collisions = adapter->stats.colc;
4412
4413         /* Rx Errors */
4414
4415         /* RLEC on some newer hardware can be incorrect so build
4416          * our own version based on RUC and ROC */
4417         net_stats->rx_errors = adapter->stats.rxerrc +
4418                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4419                 adapter->stats.ruc + adapter->stats.roc +
4420                 adapter->stats.cexterr;
4421         net_stats->rx_length_errors = adapter->stats.ruc +
4422                                       adapter->stats.roc;
4423         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4424         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4425         net_stats->rx_missed_errors = adapter->stats.mpc;
4426
4427         /* Tx Errors */
4428         net_stats->tx_errors = adapter->stats.ecol +
4429                                adapter->stats.latecol;
4430         net_stats->tx_aborted_errors = adapter->stats.ecol;
4431         net_stats->tx_window_errors = adapter->stats.latecol;
4432         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4433
4434         /* Tx Dropped needs to be maintained elsewhere */
4435
4436         /* Phy Stats */
4437         if (hw->phy.media_type == e1000_media_type_copper) {
4438                 if ((adapter->link_speed == SPEED_1000) &&
4439                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4440                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4441                         adapter->phy_stats.idle_errors += phy_tmp;
4442                 }
4443         }
4444
4445         /* Management Stats */
4446         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4447         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4448         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4449 }
4450
4451 static irqreturn_t igb_msix_other(int irq, void *data)
4452 {
4453         struct igb_adapter *adapter = data;
4454         struct e1000_hw *hw = &adapter->hw;
4455         u32 icr = rd32(E1000_ICR);
4456         /* reading ICR causes bit 31 of EICR to be cleared */
4457
4458         if (icr & E1000_ICR_DRSTA)
4459                 schedule_work(&adapter->reset_task);
4460
4461         if (icr & E1000_ICR_DOUTSYNC) {
4462                 /* HW is reporting DMA is out of sync */
4463                 adapter->stats.doosync++;
4464         }
4465
4466         /* Check for a mailbox event */
4467         if (icr & E1000_ICR_VMMB)
4468                 igb_msg_task(adapter);
4469
4470         if (icr & E1000_ICR_LSC) {
4471                 hw->mac.get_link_status = 1;
4472                 /* guard against interrupt when we're going down */
4473                 if (!test_bit(__IGB_DOWN, &adapter->state))
4474                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4475         }
4476
4477         if (adapter->vfs_allocated_count)
4478                 wr32(E1000_IMS, E1000_IMS_LSC |
4479                                 E1000_IMS_VMMB |
4480                                 E1000_IMS_DOUTSYNC);
4481         else
4482                 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4483         wr32(E1000_EIMS, adapter->eims_other);
4484
4485         return IRQ_HANDLED;
4486 }
4487
4488 static void igb_write_itr(struct igb_q_vector *q_vector)
4489 {
4490         struct igb_adapter *adapter = q_vector->adapter;
4491         u32 itr_val = q_vector->itr_val & 0x7FFC;
4492
4493         if (!q_vector->set_itr)
4494                 return;
4495
4496         if (!itr_val)
4497                 itr_val = 0x4;
4498
4499         if (adapter->hw.mac.type == e1000_82575)
4500                 itr_val |= itr_val << 16;
4501         else
4502                 itr_val |= 0x8000000;
4503
4504         writel(itr_val, q_vector->itr_register);
4505         q_vector->set_itr = 0;
4506 }
4507
4508 static irqreturn_t igb_msix_ring(int irq, void *data)
4509 {
4510         struct igb_q_vector *q_vector = data;
4511
4512         /* Write the ITR value calculated from the previous interrupt. */
4513         igb_write_itr(q_vector);
4514
4515         napi_schedule(&q_vector->napi);
4516
4517         return IRQ_HANDLED;
4518 }
4519
4520 #ifdef CONFIG_IGB_DCA
4521 static void igb_update_dca(struct igb_q_vector *q_vector)
4522 {
4523         struct igb_adapter *adapter = q_vector->adapter;
4524         struct e1000_hw *hw = &adapter->hw;
4525         int cpu = get_cpu();
4526
4527         if (q_vector->cpu == cpu)
4528                 goto out_no_update;
4529
4530         if (q_vector->tx_ring) {
4531                 int q = q_vector->tx_ring->reg_idx;
4532                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4533                 if (hw->mac.type == e1000_82575) {
4534                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4535                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4536                 } else {
4537                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4538                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4539                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4540                 }
4541                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4542                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4543         }
4544         if (q_vector->rx_ring) {
4545                 int q = q_vector->rx_ring->reg_idx;
4546                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4547                 if (hw->mac.type == e1000_82575) {
4548                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4549                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4550                 } else {
4551                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4552                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4553                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4554                 }
4555                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4556                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4557                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4558                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4559         }
4560         q_vector->cpu = cpu;
4561 out_no_update:
4562         put_cpu();
4563 }
4564
4565 static void igb_setup_dca(struct igb_adapter *adapter)
4566 {
4567         struct e1000_hw *hw = &adapter->hw;
4568         int i;
4569
4570         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4571                 return;
4572
4573         /* Always use CB2 mode, difference is masked in the CB driver. */
4574         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4575
4576         for (i = 0; i < adapter->num_q_vectors; i++) {
4577                 adapter->q_vector[i]->cpu = -1;
4578                 igb_update_dca(adapter->q_vector[i]);
4579         }
4580 }
4581
4582 static int __igb_notify_dca(struct device *dev, void *data)
4583 {
4584         struct net_device *netdev = dev_get_drvdata(dev);
4585         struct igb_adapter *adapter = netdev_priv(netdev);
4586         struct pci_dev *pdev = adapter->pdev;
4587         struct e1000_hw *hw = &adapter->hw;
4588         unsigned long event = *(unsigned long *)data;
4589
4590         switch (event) {
4591         case DCA_PROVIDER_ADD:
4592                 /* if already enabled, don't do it again */
4593                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4594                         break;
4595                 if (dca_add_requester(dev) == 0) {
4596                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4597                         dev_info(&pdev->dev, "DCA enabled\n");
4598                         igb_setup_dca(adapter);
4599                         break;
4600                 }
4601                 /* Fall Through since DCA is disabled. */
4602         case DCA_PROVIDER_REMOVE:
4603                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4604                         /* without this a class_device is left
4605                          * hanging around in the sysfs model */
4606                         dca_remove_requester(dev);
4607                         dev_info(&pdev->dev, "DCA disabled\n");
4608                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4609                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4610                 }
4611                 break;
4612         }
4613
4614         return 0;
4615 }
4616
4617 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4618                           void *p)
4619 {
4620         int ret_val;
4621
4622         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4623                                          __igb_notify_dca);
4624
4625         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4626 }
4627 #endif /* CONFIG_IGB_DCA */
4628
4629 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4630 {
4631         struct e1000_hw *hw = &adapter->hw;
4632         u32 ping;
4633         int i;
4634
4635         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4636                 ping = E1000_PF_CONTROL_MSG;
4637                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4638                         ping |= E1000_VT_MSGTYPE_CTS;
4639                 igb_write_mbx(hw, &ping, 1, i);
4640         }
4641 }
4642
4643 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4644 {
4645         struct e1000_hw *hw = &adapter->hw;
4646         u32 vmolr = rd32(E1000_VMOLR(vf));
4647         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4648
4649         vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4650                             IGB_VF_FLAG_MULTI_PROMISC);
4651         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4652
4653         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4654                 vmolr |= E1000_VMOLR_MPME;
4655                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4656         } else {
4657                 /*
4658                  * if we have hashes and we are clearing a multicast promisc
4659                  * flag we need to write the hashes to the MTA as this step
4660                  * was previously skipped
4661                  */
4662                 if (vf_data->num_vf_mc_hashes > 30) {
4663                         vmolr |= E1000_VMOLR_MPME;
4664                 } else if (vf_data->num_vf_mc_hashes) {
4665                         int j;
4666                         vmolr |= E1000_VMOLR_ROMPE;
4667                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4668                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4669                 }
4670         }
4671
4672         wr32(E1000_VMOLR(vf), vmolr);
4673
4674         /* there are flags left unprocessed, likely not supported */
4675         if (*msgbuf & E1000_VT_MSGINFO_MASK)
4676                 return -EINVAL;
4677
4678         return 0;
4679
4680 }
4681
4682 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4683                                   u32 *msgbuf, u32 vf)
4684 {
4685         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4686         u16 *hash_list = (u16 *)&msgbuf[1];
4687         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4688         int i;
4689
4690         /* salt away the number of multicast addresses assigned
4691          * to this VF for later use to restore when the PF multi cast
4692          * list changes
4693          */
4694         vf_data->num_vf_mc_hashes = n;
4695
4696         /* only up to 30 hash values supported */
4697         if (n > 30)
4698                 n = 30;
4699
4700         /* store the hashes for later use */
4701         for (i = 0; i < n; i++)
4702                 vf_data->vf_mc_hashes[i] = hash_list[i];
4703
4704         /* Flush and reset the mta with the new values */
4705         igb_set_rx_mode(adapter->netdev);
4706
4707         return 0;
4708 }
4709
4710 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4711 {
4712         struct e1000_hw *hw = &adapter->hw;
4713         struct vf_data_storage *vf_data;
4714         int i, j;
4715
4716         for (i = 0; i < adapter->vfs_allocated_count; i++) {
4717                 u32 vmolr = rd32(E1000_VMOLR(i));
4718                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4719
4720                 vf_data = &adapter->vf_data[i];
4721
4722                 if ((vf_data->num_vf_mc_hashes > 30) ||
4723                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4724                         vmolr |= E1000_VMOLR_MPME;
4725                 } else if (vf_data->num_vf_mc_hashes) {
4726                         vmolr |= E1000_VMOLR_ROMPE;
4727                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4728                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4729                 }
4730                 wr32(E1000_VMOLR(i), vmolr);
4731         }
4732 }
4733
4734 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4735 {
4736         struct e1000_hw *hw = &adapter->hw;
4737         u32 pool_mask, reg, vid;
4738         int i;
4739
4740         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4741
4742         /* Find the vlan filter for this id */
4743         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4744                 reg = rd32(E1000_VLVF(i));
4745
4746                 /* remove the vf from the pool */
4747                 reg &= ~pool_mask;
4748
4749                 /* if pool is empty then remove entry from vfta */
4750                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4751                     (reg & E1000_VLVF_VLANID_ENABLE)) {
4752                         reg = 0;
4753                         vid = reg & E1000_VLVF_VLANID_MASK;
4754                         igb_vfta_set(hw, vid, false);
4755                 }
4756
4757                 wr32(E1000_VLVF(i), reg);
4758         }
4759
4760         adapter->vf_data[vf].vlans_enabled = 0;
4761 }
4762
4763 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4764 {
4765         struct e1000_hw *hw = &adapter->hw;
4766         u32 reg, i;
4767
4768         /* The vlvf table only exists on 82576 hardware and newer */
4769         if (hw->mac.type < e1000_82576)
4770                 return -1;
4771
4772         /* we only need to do this if VMDq is enabled */
4773         if (!adapter->vfs_allocated_count)
4774                 return -1;
4775
4776         /* Find the vlan filter for this id */
4777         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4778                 reg = rd32(E1000_VLVF(i));
4779                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4780                     vid == (reg & E1000_VLVF_VLANID_MASK))
4781                         break;
4782         }
4783
4784         if (add) {
4785                 if (i == E1000_VLVF_ARRAY_SIZE) {
4786                         /* Did not find a matching VLAN ID entry that was
4787                          * enabled.  Search for a free filter entry, i.e.
4788                          * one without the enable bit set
4789                          */
4790                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4791                                 reg = rd32(E1000_VLVF(i));
4792                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4793                                         break;
4794                         }
4795                 }
4796                 if (i < E1000_VLVF_ARRAY_SIZE) {
4797                         /* Found an enabled/available entry */
4798                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4799
4800                         /* if !enabled we need to set this up in vfta */
4801                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4802                                 /* add VID to filter table */
4803                                 igb_vfta_set(hw, vid, true);
4804                                 reg |= E1000_VLVF_VLANID_ENABLE;
4805                         }
4806                         reg &= ~E1000_VLVF_VLANID_MASK;
4807                         reg |= vid;
4808                         wr32(E1000_VLVF(i), reg);
4809
4810                         /* do not modify RLPML for PF devices */
4811                         if (vf >= adapter->vfs_allocated_count)
4812                                 return 0;
4813
4814                         if (!adapter->vf_data[vf].vlans_enabled) {
4815                                 u32 size;
4816                                 reg = rd32(E1000_VMOLR(vf));
4817                                 size = reg & E1000_VMOLR_RLPML_MASK;
4818                                 size += 4;
4819                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4820                                 reg |= size;
4821                                 wr32(E1000_VMOLR(vf), reg);
4822                         }
4823
4824                         adapter->vf_data[vf].vlans_enabled++;
4825                         return 0;
4826                 }
4827         } else {
4828                 if (i < E1000_VLVF_ARRAY_SIZE) {
4829                         /* remove vf from the pool */
4830                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4831                         /* if pool is empty then remove entry from vfta */
4832                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4833                                 reg = 0;
4834                                 igb_vfta_set(hw, vid, false);
4835                         }
4836                         wr32(E1000_VLVF(i), reg);
4837
4838                         /* do not modify RLPML for PF devices */
4839                         if (vf >= adapter->vfs_allocated_count)
4840                                 return 0;
4841
4842                         adapter->vf_data[vf].vlans_enabled--;
4843                         if (!adapter->vf_data[vf].vlans_enabled) {
4844                                 u32 size;
4845                                 reg = rd32(E1000_VMOLR(vf));
4846                                 size = reg & E1000_VMOLR_RLPML_MASK;
4847                                 size -= 4;
4848                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4849                                 reg |= size;
4850                                 wr32(E1000_VMOLR(vf), reg);
4851                         }
4852                 }
4853         }
4854         return 0;
4855 }
4856
4857 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4858 {
4859         struct e1000_hw *hw = &adapter->hw;
4860
4861         if (vid)
4862                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4863         else
4864                 wr32(E1000_VMVIR(vf), 0);
4865 }
4866
4867 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4868                                int vf, u16 vlan, u8 qos)
4869 {
4870         int err = 0;
4871         struct igb_adapter *adapter = netdev_priv(netdev);
4872
4873         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4874                 return -EINVAL;
4875         if (vlan || qos) {
4876                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4877                 if (err)
4878                         goto out;
4879                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4880                 igb_set_vmolr(adapter, vf, !vlan);
4881                 adapter->vf_data[vf].pf_vlan = vlan;
4882                 adapter->vf_data[vf].pf_qos = qos;
4883                 dev_info(&adapter->pdev->dev,
4884                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4885                 if (test_bit(__IGB_DOWN, &adapter->state)) {
4886                         dev_warn(&adapter->pdev->dev,
4887                                  "The VF VLAN has been set,"
4888                                  " but the PF device is not up.\n");
4889                         dev_warn(&adapter->pdev->dev,
4890                                  "Bring the PF device up before"
4891                                  " attempting to use the VF device.\n");
4892                 }
4893         } else {
4894                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4895                                    false, vf);
4896                 igb_set_vmvir(adapter, vlan, vf);
4897                 igb_set_vmolr(adapter, vf, true);
4898                 adapter->vf_data[vf].pf_vlan = 0;
4899                 adapter->vf_data[vf].pf_qos = 0;
4900        }
4901 out:
4902        return err;
4903 }
4904
4905 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4906 {
4907         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4908         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4909
4910         return igb_vlvf_set(adapter, vid, add, vf);
4911 }
4912
4913 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4914 {
4915         /* clear flags */
4916         adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4917         adapter->vf_data[vf].last_nack = jiffies;
4918
4919         /* reset offloads to defaults */
4920         igb_set_vmolr(adapter, vf, true);
4921
4922         /* reset vlans for device */
4923         igb_clear_vf_vfta(adapter, vf);
4924         if (adapter->vf_data[vf].pf_vlan)
4925                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4926                                     adapter->vf_data[vf].pf_vlan,
4927                                     adapter->vf_data[vf].pf_qos);
4928         else
4929                 igb_clear_vf_vfta(adapter, vf);
4930
4931         /* reset multicast table array for vf */
4932         adapter->vf_data[vf].num_vf_mc_hashes = 0;
4933
4934         /* Flush and reset the mta with the new values */
4935         igb_set_rx_mode(adapter->netdev);
4936 }
4937
4938 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4939 {
4940         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4941
4942         /* generate a new mac address as we were hotplug removed/added */
4943         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4944                 random_ether_addr(vf_mac);
4945
4946         /* process remaining reset events */
4947         igb_vf_reset(adapter, vf);
4948 }
4949
4950 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4951 {
4952         struct e1000_hw *hw = &adapter->hw;
4953         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4954         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4955         u32 reg, msgbuf[3];
4956         u8 *addr = (u8 *)(&msgbuf[1]);
4957
4958         /* process all the same items cleared in a function level reset */
4959         igb_vf_reset(adapter, vf);
4960
4961         /* set vf mac address */
4962         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4963
4964         /* enable transmit and receive for vf */
4965         reg = rd32(E1000_VFTE);
4966         wr32(E1000_VFTE, reg | (1 << vf));
4967         reg = rd32(E1000_VFRE);
4968         wr32(E1000_VFRE, reg | (1 << vf));
4969
4970         adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4971
4972         /* reply to reset with ack and vf mac address */
4973         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4974         memcpy(addr, vf_mac, 6);
4975         igb_write_mbx(hw, msgbuf, 3, vf);
4976 }
4977
4978 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4979 {
4980         unsigned char *addr = (char *)&msg[1];
4981         int err = -1;
4982
4983         if (is_valid_ether_addr(addr))
4984                 err = igb_set_vf_mac(adapter, vf, addr);
4985
4986         return err;
4987 }
4988
4989 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4990 {
4991         struct e1000_hw *hw = &adapter->hw;
4992         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4993         u32 msg = E1000_VT_MSGTYPE_NACK;
4994
4995         /* if device isn't clear to send it shouldn't be reading either */
4996         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4997             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4998                 igb_write_mbx(hw, &msg, 1, vf);
4999                 vf_data->last_nack = jiffies;
5000         }
5001 }
5002
5003 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5004 {
5005         struct pci_dev *pdev = adapter->pdev;
5006         u32 msgbuf[E1000_VFMAILBOX_SIZE];
5007         struct e1000_hw *hw = &adapter->hw;
5008         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5009         s32 retval;
5010
5011         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5012
5013         if (retval) {
5014                 /* if receive failed revoke VF CTS stats and restart init */
5015                 dev_err(&pdev->dev, "Error receiving message from VF\n");
5016                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5017                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5018                         return;
5019                 goto out;
5020         }
5021
5022         /* this is a message we already processed, do nothing */
5023         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5024                 return;
5025
5026         /*
5027          * until the vf completes a reset it should not be
5028          * allowed to start any configuration.
5029          */
5030
5031         if (msgbuf[0] == E1000_VF_RESET) {
5032                 igb_vf_reset_msg(adapter, vf);
5033                 return;
5034         }
5035
5036         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5037                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5038                         return;
5039                 retval = -1;
5040                 goto out;
5041         }
5042
5043         switch ((msgbuf[0] & 0xFFFF)) {
5044         case E1000_VF_SET_MAC_ADDR:
5045                 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5046                 break;
5047         case E1000_VF_SET_PROMISC:
5048                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5049                 break;
5050         case E1000_VF_SET_MULTICAST:
5051                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5052                 break;
5053         case E1000_VF_SET_LPE:
5054                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5055                 break;
5056         case E1000_VF_SET_VLAN:
5057                 if (adapter->vf_data[vf].pf_vlan)
5058                         retval = -1;
5059                 else
5060                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5061                 break;
5062         default:
5063                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5064                 retval = -1;
5065                 break;
5066         }
5067
5068         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5069 out:
5070         /* notify the VF of the results of what it sent us */
5071         if (retval)
5072                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5073         else
5074                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5075
5076         igb_write_mbx(hw, msgbuf, 1, vf);
5077 }
5078
5079 static void igb_msg_task(struct igb_adapter *adapter)
5080 {
5081         struct e1000_hw *hw = &adapter->hw;
5082         u32 vf;
5083
5084         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5085                 /* process any reset requests */
5086                 if (!igb_check_for_rst(hw, vf))
5087                         igb_vf_reset_event(adapter, vf);
5088
5089                 /* process any messages pending */
5090                 if (!igb_check_for_msg(hw, vf))
5091                         igb_rcv_msg_from_vf(adapter, vf);
5092
5093                 /* process any acks */
5094                 if (!igb_check_for_ack(hw, vf))
5095                         igb_rcv_ack_from_vf(adapter, vf);
5096         }
5097 }
5098
5099 /**
5100  *  igb_set_uta - Set unicast filter table address
5101  *  @adapter: board private structure
5102  *
5103  *  The unicast table address is a register array of 32-bit registers.
5104  *  The table is meant to be used in a way similar to how the MTA is used
5105  *  however due to certain limitations in the hardware it is necessary to
5106  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5107  *  enable bit to allow vlan tag stripping when promiscous mode is enabled
5108  **/
5109 static void igb_set_uta(struct igb_adapter *adapter)
5110 {
5111         struct e1000_hw *hw = &adapter->hw;
5112         int i;
5113
5114         /* The UTA table only exists on 82576 hardware and newer */
5115         if (hw->mac.type < e1000_82576)
5116                 return;
5117
5118         /* we only need to do this if VMDq is enabled */
5119         if (!adapter->vfs_allocated_count)
5120                 return;
5121
5122         for (i = 0; i < hw->mac.uta_reg_count; i++)
5123                 array_wr32(E1000_UTA, i, ~0);
5124 }
5125
5126 /**
5127  * igb_intr_msi - Interrupt Handler
5128  * @irq: interrupt number
5129  * @data: pointer to a network interface device structure
5130  **/
5131 static irqreturn_t igb_intr_msi(int irq, void *data)
5132 {
5133         struct igb_adapter *adapter = data;
5134         struct igb_q_vector *q_vector = adapter->q_vector[0];
5135         struct e1000_hw *hw = &adapter->hw;
5136         /* read ICR disables interrupts using IAM */
5137         u32 icr = rd32(E1000_ICR);
5138
5139         igb_write_itr(q_vector);
5140
5141         if (icr & E1000_ICR_DRSTA)
5142                 schedule_work(&adapter->reset_task);
5143
5144         if (icr & E1000_ICR_DOUTSYNC) {
5145                 /* HW is reporting DMA is out of sync */
5146                 adapter->stats.doosync++;
5147         }
5148
5149         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5150                 hw->mac.get_link_status = 1;
5151                 if (!test_bit(__IGB_DOWN, &adapter->state))
5152                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5153         }
5154
5155         napi_schedule(&q_vector->napi);
5156
5157         return IRQ_HANDLED;
5158 }
5159
5160 /**
5161  * igb_intr - Legacy Interrupt Handler
5162  * @irq: interrupt number
5163  * @data: pointer to a network interface device structure
5164  **/
5165 static irqreturn_t igb_intr(int irq, void *data)
5166 {
5167         struct igb_adapter *adapter = data;
5168         struct igb_q_vector *q_vector = adapter->q_vector[0];
5169         struct e1000_hw *hw = &adapter->hw;
5170         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5171          * need for the IMC write */
5172         u32 icr = rd32(E1000_ICR);
5173         if (!icr)
5174                 return IRQ_NONE;  /* Not our interrupt */
5175
5176         igb_write_itr(q_vector);
5177
5178         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5179          * not set, then the adapter didn't send an interrupt */
5180         if (!(icr & E1000_ICR_INT_ASSERTED))
5181                 return IRQ_NONE;
5182
5183         if (icr & E1000_ICR_DRSTA)
5184                 schedule_work(&adapter->reset_task);
5185
5186         if (icr & E1000_ICR_DOUTSYNC) {
5187                 /* HW is reporting DMA is out of sync */
5188                 adapter->stats.doosync++;
5189         }
5190
5191         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5192                 hw->mac.get_link_status = 1;
5193                 /* guard against interrupt when we're going down */
5194                 if (!test_bit(__IGB_DOWN, &adapter->state))
5195                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5196         }
5197
5198         napi_schedule(&q_vector->napi);
5199
5200         return IRQ_HANDLED;
5201 }
5202
5203 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5204 {
5205         struct igb_adapter *adapter = q_vector->adapter;
5206         struct e1000_hw *hw = &adapter->hw;
5207
5208         if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5209             (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5210                 if (!adapter->msix_entries)
5211                         igb_set_itr(adapter);
5212                 else
5213                         igb_update_ring_itr(q_vector);
5214         }
5215
5216         if (!test_bit(__IGB_DOWN, &adapter->state)) {
5217                 if (adapter->msix_entries)
5218                         wr32(E1000_EIMS, q_vector->eims_value);
5219                 else
5220                         igb_irq_enable(adapter);
5221         }
5222 }
5223
5224 /**
5225  * igb_poll - NAPI Rx polling callback
5226  * @napi: napi polling structure
5227  * @budget: count of how many packets we should handle
5228  **/
5229 static int igb_poll(struct napi_struct *napi, int budget)
5230 {
5231         struct igb_q_vector *q_vector = container_of(napi,
5232                                                      struct igb_q_vector,
5233                                                      napi);
5234         int tx_clean_complete = 1, work_done = 0;
5235
5236 #ifdef CONFIG_IGB_DCA
5237         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5238                 igb_update_dca(q_vector);
5239 #endif
5240         if (q_vector->tx_ring)
5241                 tx_clean_complete = igb_clean_tx_irq(q_vector);
5242
5243         if (q_vector->rx_ring)
5244                 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5245
5246         if (!tx_clean_complete)
5247                 work_done = budget;
5248
5249         /* If not enough Rx work done, exit the polling mode */
5250         if (work_done < budget) {
5251                 napi_complete(napi);
5252                 igb_ring_irq_enable(q_vector);
5253         }
5254
5255         return work_done;
5256 }
5257
5258 /**
5259  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5260  * @adapter: board private structure
5261  * @shhwtstamps: timestamp structure to update
5262  * @regval: unsigned 64bit system time value.
5263  *
5264  * We need to convert the system time value stored in the RX/TXSTMP registers
5265  * into a hwtstamp which can be used by the upper level timestamping functions
5266  */
5267 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5268                                    struct skb_shared_hwtstamps *shhwtstamps,
5269                                    u64 regval)
5270 {
5271         u64 ns;
5272
5273         /*
5274          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5275          * 24 to match clock shift we setup earlier.
5276          */
5277         if (adapter->hw.mac.type == e1000_82580)
5278                 regval <<= IGB_82580_TSYNC_SHIFT;
5279
5280         ns = timecounter_cyc2time(&adapter->clock, regval);
5281         timecompare_update(&adapter->compare, ns);
5282         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5283         shhwtstamps->hwtstamp = ns_to_ktime(ns);
5284         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5285 }
5286
5287 /**
5288  * igb_tx_hwtstamp - utility function which checks for TX time stamp
5289  * @q_vector: pointer to q_vector containing needed info
5290  * @buffer: pointer to igb_buffer structure
5291  *
5292  * If we were asked to do hardware stamping and such a time stamp is
5293  * available, then it must have been for this skb here because we only
5294  * allow only one such packet into the queue.
5295  */
5296 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5297 {
5298         struct igb_adapter *adapter = q_vector->adapter;
5299         struct e1000_hw *hw = &adapter->hw;
5300         struct skb_shared_hwtstamps shhwtstamps;
5301         u64 regval;
5302
5303         /* if skb does not support hw timestamp or TX stamp not valid exit */
5304         if (likely(!buffer_info->shtx.hardware) ||
5305             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5306                 return;
5307
5308         regval = rd32(E1000_TXSTMPL);
5309         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5310
5311         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5312         skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5313 }
5314
5315 /**
5316  * igb_clean_tx_irq - Reclaim resources after transmit completes
5317  * @q_vector: pointer to q_vector containing needed info
5318  * returns true if ring is completely cleaned
5319  **/
5320 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5321 {
5322         struct igb_adapter *adapter = q_vector->adapter;
5323         struct igb_ring *tx_ring = q_vector->tx_ring;
5324         struct net_device *netdev = tx_ring->netdev;
5325         struct e1000_hw *hw = &adapter->hw;
5326         struct igb_buffer *buffer_info;
5327         union e1000_adv_tx_desc *tx_desc, *eop_desc;
5328         unsigned int total_bytes = 0, total_packets = 0;
5329         unsigned int i, eop, count = 0;
5330         bool cleaned = false;
5331
5332         i = tx_ring->next_to_clean;
5333         eop = tx_ring->buffer_info[i].next_to_watch;
5334         eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5335
5336         while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5337                (count < tx_ring->count)) {
5338                 for (cleaned = false; !cleaned; count++) {
5339                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5340                         buffer_info = &tx_ring->buffer_info[i];
5341                         cleaned = (i == eop);
5342
5343                         if (buffer_info->skb) {
5344                                 total_bytes += buffer_info->bytecount;
5345                                 /* gso_segs is currently only valid for tcp */
5346                                 total_packets += buffer_info->gso_segs;
5347                                 igb_tx_hwtstamp(q_vector, buffer_info);
5348                         }
5349
5350                         igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5351                         tx_desc->wb.status = 0;
5352
5353                         i++;
5354                         if (i == tx_ring->count)
5355                                 i = 0;
5356                 }
5357                 eop = tx_ring->buffer_info[i].next_to_watch;
5358                 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5359         }
5360
5361         tx_ring->next_to_clean = i;
5362
5363         if (unlikely(count &&
5364                      netif_carrier_ok(netdev) &&
5365                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5366                 /* Make sure that anybody stopping the queue after this
5367                  * sees the new next_to_clean.
5368                  */
5369                 smp_mb();
5370                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5371                     !(test_bit(__IGB_DOWN, &adapter->state))) {
5372                         netif_wake_subqueue(netdev, tx_ring->queue_index);
5373                         tx_ring->tx_stats.restart_queue++;
5374                 }
5375         }
5376
5377         if (tx_ring->detect_tx_hung) {
5378                 /* Detect a transmit hang in hardware, this serializes the
5379                  * check with the clearing of time_stamp and movement of i */
5380                 tx_ring->detect_tx_hung = false;
5381                 if (tx_ring->buffer_info[i].time_stamp &&
5382                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5383                                (adapter->tx_timeout_factor * HZ)) &&
5384                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5385
5386                         /* detected Tx unit hang */
5387                         dev_err(tx_ring->dev,
5388                                 "Detected Tx Unit Hang\n"
5389                                 "  Tx Queue             <%d>\n"
5390                                 "  TDH                  <%x>\n"
5391                                 "  TDT                  <%x>\n"
5392                                 "  next_to_use          <%x>\n"
5393                                 "  next_to_clean        <%x>\n"
5394                                 "buffer_info[next_to_clean]\n"
5395                                 "  time_stamp           <%lx>\n"
5396                                 "  next_to_watch        <%x>\n"
5397                                 "  jiffies              <%lx>\n"
5398                                 "  desc.status          <%x>\n",
5399                                 tx_ring->queue_index,
5400                                 readl(tx_ring->head),
5401                                 readl(tx_ring->tail),
5402                                 tx_ring->next_to_use,
5403                                 tx_ring->next_to_clean,
5404                                 tx_ring->buffer_info[eop].time_stamp,
5405                                 eop,
5406                                 jiffies,
5407                                 eop_desc->wb.status);
5408                         netif_stop_subqueue(netdev, tx_ring->queue_index);
5409                 }
5410         }
5411         tx_ring->total_bytes += total_bytes;
5412         tx_ring->total_packets += total_packets;
5413         tx_ring->tx_stats.bytes += total_bytes;
5414         tx_ring->tx_stats.packets += total_packets;
5415         return (count < tx_ring->count);
5416 }
5417
5418 /**
5419  * igb_receive_skb - helper function to handle rx indications
5420  * @q_vector: structure containing interrupt and ring information
5421  * @skb: packet to send up
5422  * @vlan_tag: vlan tag for packet
5423  **/
5424 static void igb_receive_skb(struct igb_q_vector *q_vector,
5425                             struct sk_buff *skb,
5426                             u16 vlan_tag)
5427 {
5428         struct igb_adapter *adapter = q_vector->adapter;
5429
5430         if (vlan_tag && adapter->vlgrp)
5431                 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5432                                  vlan_tag, skb);
5433         else
5434                 napi_gro_receive(&q_vector->napi, skb);
5435 }
5436
5437 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5438                                        u32 status_err, struct sk_buff *skb)
5439 {
5440         skb->ip_summed = CHECKSUM_NONE;
5441
5442         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5443         if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5444              (status_err & E1000_RXD_STAT_IXSM))
5445                 return;
5446
5447         /* TCP/UDP checksum error bit is set */
5448         if (status_err &
5449             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5450                 /*
5451                  * work around errata with sctp packets where the TCPE aka
5452                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5453                  * packets, (aka let the stack check the crc32c)
5454                  */
5455                 if ((skb->len == 60) &&
5456                     (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5457                         ring->rx_stats.csum_err++;
5458
5459                 /* let the stack verify checksum errors */
5460                 return;
5461         }
5462         /* It must be a TCP or UDP packet with a valid checksum */
5463         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5464                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5465
5466         dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5467 }
5468
5469 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5470                                    struct sk_buff *skb)
5471 {
5472         struct igb_adapter *adapter = q_vector->adapter;
5473         struct e1000_hw *hw = &adapter->hw;
5474         u64 regval;
5475
5476         /*
5477          * If this bit is set, then the RX registers contain the time stamp. No
5478          * other packet will be time stamped until we read these registers, so
5479          * read the registers to make them available again. Because only one
5480          * packet can be time stamped at a time, we know that the register
5481          * values must belong to this one here and therefore we don't need to
5482          * compare any of the additional attributes stored for it.
5483          *
5484          * If nothing went wrong, then it should have a skb_shared_tx that we
5485          * can turn into a skb_shared_hwtstamps.
5486          */
5487         if (staterr & E1000_RXDADV_STAT_TSIP) {
5488                 u32 *stamp = (u32 *)skb->data;
5489                 regval = le32_to_cpu(*(stamp + 2));
5490                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5491                 skb_pull(skb, IGB_TS_HDR_LEN);
5492         } else {
5493                 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5494                         return;
5495
5496                 regval = rd32(E1000_RXSTMPL);
5497                 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5498         }
5499
5500         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5501 }
5502 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5503                                union e1000_adv_rx_desc *rx_desc)
5504 {
5505         /* HW will not DMA in data larger than the given buffer, even if it
5506          * parses the (NFS, of course) header to be larger.  In that case, it
5507          * fills the header buffer and spills the rest into the page.
5508          */
5509         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5510                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5511         if (hlen > rx_ring->rx_buffer_len)
5512                 hlen = rx_ring->rx_buffer_len;
5513         return hlen;
5514 }
5515
5516 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5517                                  int *work_done, int budget)
5518 {
5519         struct igb_ring *rx_ring = q_vector->rx_ring;
5520         struct net_device *netdev = rx_ring->netdev;
5521         struct device *dev = rx_ring->dev;
5522         union e1000_adv_rx_desc *rx_desc , *next_rxd;
5523         struct igb_buffer *buffer_info , *next_buffer;
5524         struct sk_buff *skb;
5525         bool cleaned = false;
5526         int cleaned_count = 0;
5527         int current_node = numa_node_id();
5528         unsigned int total_bytes = 0, total_packets = 0;
5529         unsigned int i;
5530         u32 staterr;
5531         u16 length;
5532         u16 vlan_tag;
5533
5534         i = rx_ring->next_to_clean;
5535         buffer_info = &rx_ring->buffer_info[i];
5536         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5537         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5538
5539         while (staterr & E1000_RXD_STAT_DD) {
5540                 if (*work_done >= budget)
5541                         break;
5542                 (*work_done)++;
5543
5544                 skb = buffer_info->skb;
5545                 prefetch(skb->data - NET_IP_ALIGN);
5546                 buffer_info->skb = NULL;
5547
5548                 i++;
5549                 if (i == rx_ring->count)
5550                         i = 0;
5551
5552                 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5553                 prefetch(next_rxd);
5554                 next_buffer = &rx_ring->buffer_info[i];
5555
5556                 length = le16_to_cpu(rx_desc->wb.upper.length);
5557                 cleaned = true;
5558                 cleaned_count++;
5559
5560                 if (buffer_info->dma) {
5561                         dma_unmap_single(dev, buffer_info->dma,
5562                                          rx_ring->rx_buffer_len,
5563                                          DMA_FROM_DEVICE);
5564                         buffer_info->dma = 0;
5565                         if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5566                                 skb_put(skb, length);
5567                                 goto send_up;
5568                         }
5569                         skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5570                 }
5571
5572                 if (length) {
5573                         dma_unmap_page(dev, buffer_info->page_dma,
5574                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
5575                         buffer_info->page_dma = 0;
5576
5577                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5578                                                 buffer_info->page,
5579                                                 buffer_info->page_offset,
5580                                                 length);
5581
5582                         if ((page_count(buffer_info->page) != 1) ||
5583                             (page_to_nid(buffer_info->page) != current_node))
5584                                 buffer_info->page = NULL;
5585                         else
5586                                 get_page(buffer_info->page);
5587
5588                         skb->len += length;
5589                         skb->data_len += length;
5590                         skb->truesize += length;
5591                 }
5592
5593                 if (!(staterr & E1000_RXD_STAT_EOP)) {
5594                         buffer_info->skb = next_buffer->skb;
5595                         buffer_info->dma = next_buffer->dma;
5596                         next_buffer->skb = skb;
5597                         next_buffer->dma = 0;
5598                         goto next_desc;
5599                 }
5600 send_up:
5601                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5602                         dev_kfree_skb_irq(skb);
5603                         goto next_desc;
5604                 }
5605
5606                 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5607                         igb_rx_hwtstamp(q_vector, staterr, skb);
5608                 total_bytes += skb->len;
5609                 total_packets++;
5610
5611                 igb_rx_checksum_adv(rx_ring, staterr, skb);
5612
5613                 skb->protocol = eth_type_trans(skb, netdev);
5614                 skb_record_rx_queue(skb, rx_ring->queue_index);
5615
5616                 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5617                             le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5618
5619                 igb_receive_skb(q_vector, skb, vlan_tag);
5620
5621 next_desc:
5622                 rx_desc->wb.upper.status_error = 0;
5623
5624                 /* return some buffers to hardware, one at a time is too slow */
5625                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5626                         igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5627                         cleaned_count = 0;
5628                 }
5629
5630                 /* use prefetched values */
5631                 rx_desc = next_rxd;
5632                 buffer_info = next_buffer;
5633                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5634         }
5635
5636         rx_ring->next_to_clean = i;
5637         cleaned_count = igb_desc_unused(rx_ring);
5638
5639         if (cleaned_count)
5640                 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5641
5642         rx_ring->total_packets += total_packets;
5643         rx_ring->total_bytes += total_bytes;
5644         rx_ring->rx_stats.packets += total_packets;
5645         rx_ring->rx_stats.bytes += total_bytes;
5646         return cleaned;
5647 }
5648
5649 /**
5650  * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5651  * @adapter: address of board private structure
5652  **/
5653 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5654 {
5655         struct net_device *netdev = rx_ring->netdev;
5656         union e1000_adv_rx_desc *rx_desc;
5657         struct igb_buffer *buffer_info;
5658         struct sk_buff *skb;
5659         unsigned int i;
5660         int bufsz;
5661
5662         i = rx_ring->next_to_use;
5663         buffer_info = &rx_ring->buffer_info[i];
5664
5665         bufsz = rx_ring->rx_buffer_len;
5666
5667         while (cleaned_count--) {
5668                 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5669
5670                 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5671                         if (!buffer_info->page) {
5672                                 buffer_info->page = netdev_alloc_page(netdev);
5673                                 if (!buffer_info->page) {
5674                                         rx_ring->rx_stats.alloc_failed++;
5675                                         goto no_buffers;
5676                                 }
5677                                 buffer_info->page_offset = 0;
5678                         } else {
5679                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
5680                         }
5681                         buffer_info->page_dma =
5682                                 dma_map_page(rx_ring->dev, buffer_info->page,
5683                                              buffer_info->page_offset,
5684                                              PAGE_SIZE / 2,
5685                                              DMA_FROM_DEVICE);
5686                         if (dma_mapping_error(rx_ring->dev,
5687                                               buffer_info->page_dma)) {
5688                                 buffer_info->page_dma = 0;
5689                                 rx_ring->rx_stats.alloc_failed++;
5690                                 goto no_buffers;
5691                         }
5692                 }
5693
5694                 skb = buffer_info->skb;
5695                 if (!skb) {
5696                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5697                         if (!skb) {
5698                                 rx_ring->rx_stats.alloc_failed++;
5699                                 goto no_buffers;
5700                         }
5701
5702                         buffer_info->skb = skb;
5703                 }
5704                 if (!buffer_info->dma) {
5705                         buffer_info->dma = dma_map_single(rx_ring->dev,
5706                                                           skb->data,
5707                                                           bufsz,
5708                                                           DMA_FROM_DEVICE);
5709                         if (dma_mapping_error(rx_ring->dev,
5710                                               buffer_info->dma)) {
5711                                 buffer_info->dma = 0;
5712                                 rx_ring->rx_stats.alloc_failed++;
5713                                 goto no_buffers;
5714                         }
5715                 }
5716                 /* Refresh the desc even if buffer_addrs didn't change because
5717                  * each write-back erases this info. */
5718                 if (bufsz < IGB_RXBUFFER_1024) {
5719                         rx_desc->read.pkt_addr =
5720                              cpu_to_le64(buffer_info->page_dma);
5721                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5722                 } else {
5723                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5724                         rx_desc->read.hdr_addr = 0;
5725                 }
5726
5727                 i++;
5728                 if (i == rx_ring->count)
5729                         i = 0;
5730                 buffer_info = &rx_ring->buffer_info[i];
5731         }
5732
5733 no_buffers:
5734         if (rx_ring->next_to_use != i) {
5735                 rx_ring->next_to_use = i;
5736                 if (i == 0)
5737                         i = (rx_ring->count - 1);
5738                 else
5739                         i--;
5740
5741                 /* Force memory writes to complete before letting h/w
5742                  * know there are new descriptors to fetch.  (Only
5743                  * applicable for weak-ordered memory model archs,
5744                  * such as IA-64). */
5745                 wmb();
5746                 writel(i, rx_ring->tail);
5747         }
5748 }
5749
5750 /**
5751  * igb_mii_ioctl -
5752  * @netdev:
5753  * @ifreq:
5754  * @cmd:
5755  **/
5756 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5757 {
5758         struct igb_adapter *adapter = netdev_priv(netdev);
5759         struct mii_ioctl_data *data = if_mii(ifr);
5760
5761         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5762                 return -EOPNOTSUPP;
5763
5764         switch (cmd) {
5765         case SIOCGMIIPHY:
5766                 data->phy_id = adapter->hw.phy.addr;
5767                 break;
5768         case SIOCGMIIREG:
5769                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5770                                      &data->val_out))
5771                         return -EIO;
5772                 break;
5773         case SIOCSMIIREG:
5774         default:
5775                 return -EOPNOTSUPP;
5776         }
5777         return 0;
5778 }
5779
5780 /**
5781  * igb_hwtstamp_ioctl - control hardware time stamping
5782  * @netdev:
5783  * @ifreq:
5784  * @cmd:
5785  *
5786  * Outgoing time stamping can be enabled and disabled. Play nice and
5787  * disable it when requested, although it shouldn't case any overhead
5788  * when no packet needs it. At most one packet in the queue may be
5789  * marked for time stamping, otherwise it would be impossible to tell
5790  * for sure to which packet the hardware time stamp belongs.
5791  *
5792  * Incoming time stamping has to be configured via the hardware
5793  * filters. Not all combinations are supported, in particular event
5794  * type has to be specified. Matching the kind of event packet is
5795  * not supported, with the exception of "all V2 events regardless of
5796  * level 2 or 4".
5797  *
5798  **/
5799 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5800                               struct ifreq *ifr, int cmd)
5801 {
5802         struct igb_adapter *adapter = netdev_priv(netdev);
5803         struct e1000_hw *hw = &adapter->hw;
5804         struct hwtstamp_config config;
5805         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5806         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5807         u32 tsync_rx_cfg = 0;
5808         bool is_l4 = false;
5809         bool is_l2 = false;
5810         u32 regval;
5811
5812         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5813                 return -EFAULT;
5814
5815         /* reserved for future extensions */
5816         if (config.flags)
5817                 return -EINVAL;
5818
5819         switch (config.tx_type) {
5820         case HWTSTAMP_TX_OFF:
5821                 tsync_tx_ctl = 0;
5822         case HWTSTAMP_TX_ON:
5823                 break;
5824         default:
5825                 return -ERANGE;
5826         }
5827
5828         switch (config.rx_filter) {
5829         case HWTSTAMP_FILTER_NONE:
5830                 tsync_rx_ctl = 0;
5831                 break;
5832         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5833         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5834         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5835         case HWTSTAMP_FILTER_ALL:
5836                 /*
5837                  * register TSYNCRXCFG must be set, therefore it is not
5838                  * possible to time stamp both Sync and Delay_Req messages
5839                  * => fall back to time stamping all packets
5840                  */
5841                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5842                 config.rx_filter = HWTSTAMP_FILTER_ALL;
5843                 break;
5844         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5845                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5846                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5847                 is_l4 = true;
5848                 break;
5849         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5850                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5851                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5852                 is_l4 = true;
5853                 break;
5854         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5855         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5856                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5857                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5858                 is_l2 = true;
5859                 is_l4 = true;
5860                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5861                 break;
5862         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5863         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5864                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5865                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5866                 is_l2 = true;
5867                 is_l4 = true;
5868                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5869                 break;
5870         case HWTSTAMP_FILTER_PTP_V2_EVENT:
5871         case HWTSTAMP_FILTER_PTP_V2_SYNC:
5872         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5873                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5874                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5875                 is_l2 = true;
5876                 break;
5877         default:
5878                 return -ERANGE;
5879         }
5880
5881         if (hw->mac.type == e1000_82575) {
5882                 if (tsync_rx_ctl | tsync_tx_ctl)
5883                         return -EINVAL;
5884                 return 0;
5885         }
5886
5887         /*
5888          * Per-packet timestamping only works if all packets are
5889          * timestamped, so enable timestamping in all packets as
5890          * long as one rx filter was configured.
5891          */
5892         if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5893                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5894                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5895         }
5896
5897         /* enable/disable TX */
5898         regval = rd32(E1000_TSYNCTXCTL);
5899         regval &= ~E1000_TSYNCTXCTL_ENABLED;
5900         regval |= tsync_tx_ctl;
5901         wr32(E1000_TSYNCTXCTL, regval);
5902
5903         /* enable/disable RX */
5904         regval = rd32(E1000_TSYNCRXCTL);
5905         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5906         regval |= tsync_rx_ctl;
5907         wr32(E1000_TSYNCRXCTL, regval);
5908
5909         /* define which PTP packets are time stamped */
5910         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5911
5912         /* define ethertype filter for timestamped packets */
5913         if (is_l2)
5914                 wr32(E1000_ETQF(3),
5915                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5916                                  E1000_ETQF_1588 | /* enable timestamping */
5917                                  ETH_P_1588));     /* 1588 eth protocol type */
5918         else
5919                 wr32(E1000_ETQF(3), 0);
5920
5921 #define PTP_PORT 319
5922         /* L4 Queue Filter[3]: filter by destination port and protocol */
5923         if (is_l4) {
5924                 u32 ftqf = (IPPROTO_UDP /* UDP */
5925                         | E1000_FTQF_VF_BP /* VF not compared */
5926                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5927                         | E1000_FTQF_MASK); /* mask all inputs */
5928                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5929
5930                 wr32(E1000_IMIR(3), htons(PTP_PORT));
5931                 wr32(E1000_IMIREXT(3),
5932                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5933                 if (hw->mac.type == e1000_82576) {
5934                         /* enable source port check */
5935                         wr32(E1000_SPQF(3), htons(PTP_PORT));
5936                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5937                 }
5938                 wr32(E1000_FTQF(3), ftqf);
5939         } else {
5940                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5941         }
5942         wrfl();
5943
5944         adapter->hwtstamp_config = config;
5945
5946         /* clear TX/RX time stamp registers, just to be sure */
5947         regval = rd32(E1000_TXSTMPH);
5948         regval = rd32(E1000_RXSTMPH);
5949
5950         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5951                 -EFAULT : 0;
5952 }
5953
5954 /**
5955  * igb_ioctl -
5956  * @netdev:
5957  * @ifreq:
5958  * @cmd:
5959  **/
5960 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5961 {
5962         switch (cmd) {
5963         case SIOCGMIIPHY:
5964         case SIOCGMIIREG:
5965         case SIOCSMIIREG:
5966                 return igb_mii_ioctl(netdev, ifr, cmd);
5967         case SIOCSHWTSTAMP:
5968                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5969         default:
5970                 return -EOPNOTSUPP;
5971         }
5972 }
5973
5974 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5975 {
5976         struct igb_adapter *adapter = hw->back;
5977         u16 cap_offset;
5978
5979         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5980         if (!cap_offset)
5981                 return -E1000_ERR_CONFIG;
5982
5983         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5984
5985         return 0;
5986 }
5987
5988 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5989 {
5990         struct igb_adapter *adapter = hw->back;
5991         u16 cap_offset;
5992
5993         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5994         if (!cap_offset)
5995                 return -E1000_ERR_CONFIG;
5996
5997         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5998
5999         return 0;
6000 }
6001
6002 static void igb_vlan_rx_register(struct net_device *netdev,
6003                                  struct vlan_group *grp)
6004 {
6005         struct igb_adapter *adapter = netdev_priv(netdev);
6006         struct e1000_hw *hw = &adapter->hw;
6007         u32 ctrl, rctl;
6008
6009         igb_irq_disable(adapter);
6010         adapter->vlgrp = grp;
6011
6012         if (grp) {
6013                 /* enable VLAN tag insert/strip */
6014                 ctrl = rd32(E1000_CTRL);
6015                 ctrl |= E1000_CTRL_VME;
6016                 wr32(E1000_CTRL, ctrl);
6017
6018                 /* Disable CFI check */
6019                 rctl = rd32(E1000_RCTL);
6020                 rctl &= ~E1000_RCTL_CFIEN;
6021                 wr32(E1000_RCTL, rctl);
6022         } else {
6023                 /* disable VLAN tag insert/strip */
6024                 ctrl = rd32(E1000_CTRL);
6025                 ctrl &= ~E1000_CTRL_VME;
6026                 wr32(E1000_CTRL, ctrl);
6027         }
6028
6029         igb_rlpml_set(adapter);
6030
6031         if (!test_bit(__IGB_DOWN, &adapter->state))
6032                 igb_irq_enable(adapter);
6033 }
6034
6035 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6036 {
6037         struct igb_adapter *adapter = netdev_priv(netdev);
6038         struct e1000_hw *hw = &adapter->hw;
6039         int pf_id = adapter->vfs_allocated_count;
6040
6041         /* attempt to add filter to vlvf array */
6042         igb_vlvf_set(adapter, vid, true, pf_id);
6043
6044         /* add the filter since PF can receive vlans w/o entry in vlvf */
6045         igb_vfta_set(hw, vid, true);
6046 }
6047
6048 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6049 {
6050         struct igb_adapter *adapter = netdev_priv(netdev);
6051         struct e1000_hw *hw = &adapter->hw;
6052         int pf_id = adapter->vfs_allocated_count;
6053         s32 err;
6054
6055         igb_irq_disable(adapter);
6056         vlan_group_set_device(adapter->vlgrp, vid, NULL);
6057
6058         if (!test_bit(__IGB_DOWN, &adapter->state))
6059                 igb_irq_enable(adapter);
6060
6061         /* remove vlan from VLVF table array */
6062         err = igb_vlvf_set(adapter, vid, false, pf_id);
6063
6064         /* if vid was not present in VLVF just remove it from table */
6065         if (err)
6066                 igb_vfta_set(hw, vid, false);
6067 }
6068
6069 static void igb_restore_vlan(struct igb_adapter *adapter)
6070 {
6071         igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6072
6073         if (adapter->vlgrp) {
6074                 u16 vid;
6075                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6076                         if (!vlan_group_get_device(adapter->vlgrp, vid))
6077                                 continue;
6078                         igb_vlan_rx_add_vid(adapter->netdev, vid);
6079                 }
6080         }
6081 }
6082
6083 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6084 {
6085         struct pci_dev *pdev = adapter->pdev;
6086         struct e1000_mac_info *mac = &adapter->hw.mac;
6087
6088         mac->autoneg = 0;
6089
6090         switch (spddplx) {
6091         case SPEED_10 + DUPLEX_HALF:
6092                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6093                 break;
6094         case SPEED_10 + DUPLEX_FULL:
6095                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6096                 break;
6097         case SPEED_100 + DUPLEX_HALF:
6098                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6099                 break;
6100         case SPEED_100 + DUPLEX_FULL:
6101                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6102                 break;
6103         case SPEED_1000 + DUPLEX_FULL:
6104                 mac->autoneg = 1;
6105                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6106                 break;
6107         case SPEED_1000 + DUPLEX_HALF: /* not supported */
6108         default:
6109                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6110                 return -EINVAL;
6111         }
6112         return 0;
6113 }
6114
6115 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6116 {
6117         struct net_device *netdev = pci_get_drvdata(pdev);
6118         struct igb_adapter *adapter = netdev_priv(netdev);
6119         struct e1000_hw *hw = &adapter->hw;
6120         u32 ctrl, rctl, status;
6121         u32 wufc = adapter->wol;
6122 #ifdef CONFIG_PM
6123         int retval = 0;
6124 #endif
6125
6126         netif_device_detach(netdev);
6127
6128         if (netif_running(netdev))
6129                 igb_close(netdev);
6130
6131         igb_clear_interrupt_scheme(adapter);
6132
6133 #ifdef CONFIG_PM
6134         retval = pci_save_state(pdev);
6135         if (retval)
6136                 return retval;
6137 #endif
6138
6139         status = rd32(E1000_STATUS);
6140         if (status & E1000_STATUS_LU)
6141                 wufc &= ~E1000_WUFC_LNKC;
6142
6143         if (wufc) {
6144                 igb_setup_rctl(adapter);
6145                 igb_set_rx_mode(netdev);
6146
6147                 /* turn on all-multi mode if wake on multicast is enabled */
6148                 if (wufc & E1000_WUFC_MC) {
6149                         rctl = rd32(E1000_RCTL);
6150                         rctl |= E1000_RCTL_MPE;
6151                         wr32(E1000_RCTL, rctl);
6152                 }
6153
6154                 ctrl = rd32(E1000_CTRL);
6155                 /* advertise wake from D3Cold */
6156                 #define E1000_CTRL_ADVD3WUC 0x00100000
6157                 /* phy power management enable */
6158                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6159                 ctrl |= E1000_CTRL_ADVD3WUC;
6160                 wr32(E1000_CTRL, ctrl);
6161
6162                 /* Allow time for pending master requests to run */
6163                 igb_disable_pcie_master(hw);
6164
6165                 wr32(E1000_WUC, E1000_WUC_PME_EN);
6166                 wr32(E1000_WUFC, wufc);
6167         } else {
6168                 wr32(E1000_WUC, 0);
6169                 wr32(E1000_WUFC, 0);
6170         }
6171
6172         *enable_wake = wufc || adapter->en_mng_pt;
6173         if (!*enable_wake)
6174                 igb_power_down_link(adapter);
6175         else
6176                 igb_power_up_link(adapter);
6177
6178         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6179          * would have already happened in close and is redundant. */
6180         igb_release_hw_control(adapter);
6181
6182         pci_disable_device(pdev);
6183
6184         return 0;
6185 }
6186
6187 #ifdef CONFIG_PM
6188 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6189 {
6190         int retval;
6191         bool wake;
6192
6193         retval = __igb_shutdown(pdev, &wake);
6194         if (retval)
6195                 return retval;
6196
6197         if (wake) {
6198                 pci_prepare_to_sleep(pdev);
6199         } else {
6200                 pci_wake_from_d3(pdev, false);
6201                 pci_set_power_state(pdev, PCI_D3hot);
6202         }
6203
6204         return 0;
6205 }
6206
6207 static int igb_resume(struct pci_dev *pdev)
6208 {
6209         struct net_device *netdev = pci_get_drvdata(pdev);
6210         struct igb_adapter *adapter = netdev_priv(netdev);
6211         struct e1000_hw *hw = &adapter->hw;
6212         u32 err;
6213
6214         pci_set_power_state(pdev, PCI_D0);
6215         pci_restore_state(pdev);
6216         pci_save_state(pdev);
6217
6218         err = pci_enable_device_mem(pdev);
6219         if (err) {
6220                 dev_err(&pdev->dev,
6221                         "igb: Cannot enable PCI device from suspend\n");
6222                 return err;
6223         }
6224         pci_set_master(pdev);
6225
6226         pci_enable_wake(pdev, PCI_D3hot, 0);
6227         pci_enable_wake(pdev, PCI_D3cold, 0);
6228
6229         if (igb_init_interrupt_scheme(adapter)) {
6230                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6231                 return -ENOMEM;
6232         }
6233
6234         igb_reset(adapter);
6235
6236         /* let the f/w know that the h/w is now under the control of the
6237          * driver. */
6238         igb_get_hw_control(adapter);
6239
6240         wr32(E1000_WUS, ~0);
6241
6242         if (netif_running(netdev)) {
6243                 err = igb_open(netdev);
6244                 if (err)
6245                         return err;
6246         }
6247
6248         netif_device_attach(netdev);
6249
6250         return 0;
6251 }
6252 #endif
6253
6254 static void igb_shutdown(struct pci_dev *pdev)
6255 {
6256         bool wake;
6257
6258         __igb_shutdown(pdev, &wake);
6259
6260         if (system_state == SYSTEM_POWER_OFF) {
6261                 pci_wake_from_d3(pdev, wake);
6262                 pci_set_power_state(pdev, PCI_D3hot);
6263         }
6264 }
6265
6266 #ifdef CONFIG_NET_POLL_CONTROLLER
6267 /*
6268  * Polling 'interrupt' - used by things like netconsole to send skbs
6269  * without having to re-enable interrupts. It's not called while
6270  * the interrupt routine is executing.
6271  */
6272 static void igb_netpoll(struct net_device *netdev)
6273 {
6274         struct igb_adapter *adapter = netdev_priv(netdev);
6275         struct e1000_hw *hw = &adapter->hw;
6276         int i;
6277
6278         if (!adapter->msix_entries) {
6279                 struct igb_q_vector *q_vector = adapter->q_vector[0];
6280                 igb_irq_disable(adapter);
6281                 napi_schedule(&q_vector->napi);
6282                 return;
6283         }
6284
6285         for (i = 0; i < adapter->num_q_vectors; i++) {
6286                 struct igb_q_vector *q_vector = adapter->q_vector[i];
6287                 wr32(E1000_EIMC, q_vector->eims_value);
6288                 napi_schedule(&q_vector->napi);
6289         }
6290 }
6291 #endif /* CONFIG_NET_POLL_CONTROLLER */
6292
6293 /**
6294  * igb_io_error_detected - called when PCI error is detected
6295  * @pdev: Pointer to PCI device
6296  * @state: The current pci connection state
6297  *
6298  * This function is called after a PCI bus error affecting
6299  * this device has been detected.
6300  */
6301 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6302                                               pci_channel_state_t state)
6303 {
6304         struct net_device *netdev = pci_get_drvdata(pdev);
6305         struct igb_adapter *adapter = netdev_priv(netdev);
6306
6307         netif_device_detach(netdev);
6308
6309         if (state == pci_channel_io_perm_failure)
6310                 return PCI_ERS_RESULT_DISCONNECT;
6311
6312         if (netif_running(netdev))
6313                 igb_down(adapter);
6314         pci_disable_device(pdev);
6315
6316         /* Request a slot slot reset. */
6317         return PCI_ERS_RESULT_NEED_RESET;
6318 }
6319
6320 /**
6321  * igb_io_slot_reset - called after the pci bus has been reset.
6322  * @pdev: Pointer to PCI device
6323  *
6324  * Restart the card from scratch, as if from a cold-boot. Implementation
6325  * resembles the first-half of the igb_resume routine.
6326  */
6327 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6328 {
6329         struct net_device *netdev = pci_get_drvdata(pdev);
6330         struct igb_adapter *adapter = netdev_priv(netdev);
6331         struct e1000_hw *hw = &adapter->hw;
6332         pci_ers_result_t result;
6333         int err;
6334
6335         if (pci_enable_device_mem(pdev)) {
6336                 dev_err(&pdev->dev,
6337                         "Cannot re-enable PCI device after reset.\n");
6338                 result = PCI_ERS_RESULT_DISCONNECT;
6339         } else {
6340                 pci_set_master(pdev);
6341                 pci_restore_state(pdev);
6342                 pci_save_state(pdev);
6343
6344                 pci_enable_wake(pdev, PCI_D3hot, 0);
6345                 pci_enable_wake(pdev, PCI_D3cold, 0);
6346
6347                 igb_reset(adapter);
6348                 wr32(E1000_WUS, ~0);
6349                 result = PCI_ERS_RESULT_RECOVERED;
6350         }
6351
6352         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6353         if (err) {
6354                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6355                         "failed 0x%0x\n", err);
6356                 /* non-fatal, continue */
6357         }
6358
6359         return result;
6360 }
6361
6362 /**
6363  * igb_io_resume - called when traffic can start flowing again.
6364  * @pdev: Pointer to PCI device
6365  *
6366  * This callback is called when the error recovery driver tells us that
6367  * its OK to resume normal operation. Implementation resembles the
6368  * second-half of the igb_resume routine.
6369  */
6370 static void igb_io_resume(struct pci_dev *pdev)
6371 {
6372         struct net_device *netdev = pci_get_drvdata(pdev);
6373         struct igb_adapter *adapter = netdev_priv(netdev);
6374
6375         if (netif_running(netdev)) {
6376                 if (igb_up(adapter)) {
6377                         dev_err(&pdev->dev, "igb_up failed after reset\n");
6378                         return;
6379                 }
6380         }
6381
6382         netif_device_attach(netdev);
6383
6384         /* let the f/w know that the h/w is now under the control of the
6385          * driver. */
6386         igb_get_hw_control(adapter);
6387 }
6388
6389 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6390                              u8 qsel)
6391 {
6392         u32 rar_low, rar_high;
6393         struct e1000_hw *hw = &adapter->hw;
6394
6395         /* HW expects these in little endian so we reverse the byte order
6396          * from network order (big endian) to little endian
6397          */
6398         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6399                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6400         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6401
6402         /* Indicate to hardware the Address is Valid. */
6403         rar_high |= E1000_RAH_AV;
6404
6405         if (hw->mac.type == e1000_82575)
6406                 rar_high |= E1000_RAH_POOL_1 * qsel;
6407         else
6408                 rar_high |= E1000_RAH_POOL_1 << qsel;
6409
6410         wr32(E1000_RAL(index), rar_low);
6411         wrfl();
6412         wr32(E1000_RAH(index), rar_high);
6413         wrfl();
6414 }
6415
6416 static int igb_set_vf_mac(struct igb_adapter *adapter,
6417                           int vf, unsigned char *mac_addr)
6418 {
6419         struct e1000_hw *hw = &adapter->hw;
6420         /* VF MAC addresses start at end of receive addresses and moves
6421          * torwards the first, as a result a collision should not be possible */
6422         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6423
6424         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6425
6426         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6427
6428         return 0;
6429 }
6430
6431 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6432 {
6433         struct igb_adapter *adapter = netdev_priv(netdev);
6434         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6435                 return -EINVAL;
6436         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6437         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6438         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6439                                       " change effective.");
6440         if (test_bit(__IGB_DOWN, &adapter->state)) {
6441                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6442                          " but the PF device is not up.\n");
6443                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6444                          " attempting to use the VF device.\n");
6445         }
6446         return igb_set_vf_mac(adapter, vf, mac);
6447 }
6448
6449 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6450 {
6451         return -EOPNOTSUPP;
6452 }
6453
6454 static int igb_ndo_get_vf_config(struct net_device *netdev,
6455                                  int vf, struct ifla_vf_info *ivi)
6456 {
6457         struct igb_adapter *adapter = netdev_priv(netdev);
6458         if (vf >= adapter->vfs_allocated_count)
6459                 return -EINVAL;
6460         ivi->vf = vf;
6461         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6462         ivi->tx_rate = 0;
6463         ivi->vlan = adapter->vf_data[vf].pf_vlan;
6464         ivi->qos = adapter->vf_data[vf].pf_qos;
6465         return 0;
6466 }
6467
6468 static void igb_vmm_control(struct igb_adapter *adapter)
6469 {
6470         struct e1000_hw *hw = &adapter->hw;
6471         u32 reg;
6472
6473         switch (hw->mac.type) {
6474         case e1000_82575:
6475         default:
6476                 /* replication is not supported for 82575 */
6477                 return;
6478         case e1000_82576:
6479                 /* notify HW that the MAC is adding vlan tags */
6480                 reg = rd32(E1000_DTXCTL);
6481                 reg |= E1000_DTXCTL_VLAN_ADDED;
6482                 wr32(E1000_DTXCTL, reg);
6483         case e1000_82580:
6484                 /* enable replication vlan tag stripping */
6485                 reg = rd32(E1000_RPLOLR);
6486                 reg |= E1000_RPLOLR_STRVLAN;
6487                 wr32(E1000_RPLOLR, reg);
6488         case e1000_i350:
6489                 /* none of the above registers are supported by i350 */
6490                 break;
6491         }
6492
6493         if (adapter->vfs_allocated_count) {
6494                 igb_vmdq_set_loopback_pf(hw, true);
6495                 igb_vmdq_set_replication_pf(hw, true);
6496         } else {
6497                 igb_vmdq_set_loopback_pf(hw, false);
6498                 igb_vmdq_set_replication_pf(hw, false);
6499         }
6500 }
6501
6502 /* igb_main.c */