]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
vlan: Rename VLAN_GROUP_ARRAY_LEN to VLAN_N_VID.
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
2d064c06 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
87 /* required last entry */
88 {0, }
89};
90
91MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
92
93void igb_reset(struct igb_adapter *);
94static int igb_setup_all_tx_resources(struct igb_adapter *);
95static int igb_setup_all_rx_resources(struct igb_adapter *);
96static void igb_free_all_tx_resources(struct igb_adapter *);
97static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 98static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 99static int igb_probe(struct pci_dev *, const struct pci_device_id *);
100static void __devexit igb_remove(struct pci_dev *pdev);
101static int igb_sw_init(struct igb_adapter *);
102static int igb_open(struct net_device *);
103static int igb_close(struct net_device *);
104static void igb_configure_tx(struct igb_adapter *);
105static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 106static void igb_clean_all_tx_rings(struct igb_adapter *);
107static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 108static void igb_clean_tx_ring(struct igb_ring *);
109static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 110static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 111static void igb_update_phy_info(unsigned long);
112static void igb_watchdog(unsigned long);
113static void igb_watchdog_task(struct work_struct *);
b1a436c3 114static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 115static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
116 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 117static int igb_change_mtu(struct net_device *, int);
118static int igb_set_mac(struct net_device *, void *);
68d480c4 119static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 120static irqreturn_t igb_intr(int irq, void *);
121static irqreturn_t igb_intr_msi(int irq, void *);
122static irqreturn_t igb_msix_other(int irq, void *);
047e0030 123static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 124#ifdef CONFIG_IGB_DCA
047e0030 125static void igb_update_dca(struct igb_q_vector *);
fe4506b6 126static void igb_setup_dca(struct igb_adapter *);
421e02f0 127#endif /* CONFIG_IGB_DCA */
047e0030 128static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 129static int igb_poll(struct napi_struct *, int);
047e0030 130static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 131static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
132static void igb_tx_timeout(struct net_device *);
133static void igb_reset_task(struct work_struct *);
134static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
135static void igb_vlan_rx_add_vid(struct net_device *, u16);
136static void igb_vlan_rx_kill_vid(struct net_device *, u16);
137static void igb_restore_vlan(struct igb_adapter *);
26ad9178 138static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 139static void igb_ping_all_vfs(struct igb_adapter *);
140static void igb_msg_task(struct igb_adapter *);
4ae196df 141static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 142static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 143static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 144static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
145static int igb_ndo_set_vf_vlan(struct net_device *netdev,
146 int vf, u16 vlan, u8 qos);
147static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
148static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
149 struct ifla_vf_info *ivi);
9d5c8243 150
9d5c8243 151#ifdef CONFIG_PM
3fe7c4c9 152static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 153static int igb_resume(struct pci_dev *);
154#endif
155static void igb_shutdown(struct pci_dev *);
421e02f0 156#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 157static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
158static struct notifier_block dca_notifier = {
159 .notifier_call = igb_notify_dca,
160 .next = NULL,
161 .priority = 0
162};
163#endif
9d5c8243
AK		 164#ifdef CONFIG_NET_POLL_CONTROLLER
165/* for netdump / net console */
166static void igb_netpoll(struct net_device *);
167#endif
37680117 168#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 169static unsigned int max_vfs = 0;
170module_param(max_vfs, uint, 0);
171MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
172 "per physical function");
173#endif /* CONFIG_PCI_IOV */
174
9d5c8243
AK		 175static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
176 pci_channel_state_t);
177static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
178static void igb_io_resume(struct pci_dev *);
179
180static struct pci_error_handlers igb_err_handler = {
181 .error_detected = igb_io_error_detected,
182 .slot_reset = igb_io_slot_reset,
183 .resume = igb_io_resume,
184};
185
186
187static struct pci_driver igb_driver = {
188 .name = igb_driver_name,
189 .id_table = igb_pci_tbl,
190 .probe = igb_probe,
191 .remove = __devexit_p(igb_remove),
192#ifdef CONFIG_PM
193 /* Power Managment Hooks */
194 .suspend = igb_suspend,
195 .resume = igb_resume,
196#endif
197 .shutdown = igb_shutdown,
198 .err_handler = &igb_err_handler
199};
200
201MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
202MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
203MODULE_LICENSE("GPL");
204MODULE_VERSION(DRV_VERSION);
205
c97ec42a
TI		 206struct igb_reg_info {
207 u32 ofs;
208 char *name;
209};
210
211static const struct igb_reg_info igb_reg_info_tbl[] = {
212
213 /* General Registers */
214 {E1000_CTRL, "CTRL"},
215 {E1000_STATUS, "STATUS"},
216 {E1000_CTRL_EXT, "CTRL_EXT"},
217
218 /* Interrupt Registers */
219 {E1000_ICR, "ICR"},
220
221 /* RX Registers */
222 {E1000_RCTL, "RCTL"},
223 {E1000_RDLEN(0), "RDLEN"},
224 {E1000_RDH(0), "RDH"},
225 {E1000_RDT(0), "RDT"},
226 {E1000_RXDCTL(0), "RXDCTL"},
227 {E1000_RDBAL(0), "RDBAL"},
228 {E1000_RDBAH(0), "RDBAH"},
229
230 /* TX Registers */
231 {E1000_TCTL, "TCTL"},
232 {E1000_TDBAL(0), "TDBAL"},
233 {E1000_TDBAH(0), "TDBAH"},
234 {E1000_TDLEN(0), "TDLEN"},
235 {E1000_TDH(0), "TDH"},
236 {E1000_TDT(0), "TDT"},
237 {E1000_TXDCTL(0), "TXDCTL"},
238 {E1000_TDFH, "TDFH"},
239 {E1000_TDFT, "TDFT"},
240 {E1000_TDFHS, "TDFHS"},
241 {E1000_TDFPC, "TDFPC"},
242
243 /* List Terminator */
244 {}
245};
246
247/*
248 * igb_regdump - register printout routine
249 */
250static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
251{
252 int n = 0;
253 char rname[16];
254 u32 regs[8];
255
256 switch (reginfo->ofs) {
257 case E1000_RDLEN(0):
258 for (n = 0; n < 4; n++)
259 regs[n] = rd32(E1000_RDLEN(n));
260 break;
261 case E1000_RDH(0):
262 for (n = 0; n < 4; n++)
263 regs[n] = rd32(E1000_RDH(n));
264 break;
265 case E1000_RDT(0):
266 for (n = 0; n < 4; n++)
267 regs[n] = rd32(E1000_RDT(n));
268 break;
269 case E1000_RXDCTL(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RXDCTL(n));
272 break;
273 case E1000_RDBAL(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDBAL(n));
276 break;
277 case E1000_RDBAH(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDBAH(n));
280 break;
281 case E1000_TDBAL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RDBAL(n));
284 break;
285 case E1000_TDBAH(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_TDBAH(n));
288 break;
289 case E1000_TDLEN(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_TDLEN(n));
292 break;
293 case E1000_TDH(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_TDH(n));
296 break;
297 case E1000_TDT(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDT(n));
300 break;
301 case E1000_TXDCTL(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TXDCTL(n));
304 break;
305 default:
306 printk(KERN_INFO "%-15s %08x\n",
307 reginfo->name, rd32(reginfo->ofs));
308 return;
309 }
310
311 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
312 printk(KERN_INFO "%-15s ", rname);
313 for (n = 0; n < 4; n++)
314 printk(KERN_CONT "%08x ", regs[n]);
315 printk(KERN_CONT "\n");
316}
317
318/*
319 * igb_dump - Print registers, tx-rings and rx-rings
320 */
321static void igb_dump(struct igb_adapter *adapter)
322{
323 struct net_device *netdev = adapter->netdev;
324 struct e1000_hw *hw = &adapter->hw;
325 struct igb_reg_info *reginfo;
326 int n = 0;
327 struct igb_ring *tx_ring;
328 union e1000_adv_tx_desc *tx_desc;
329 struct my_u0 { u64 a; u64 b; } *u0;
330 struct igb_buffer *buffer_info;
331 struct igb_ring *rx_ring;
332 union e1000_adv_rx_desc *rx_desc;
333 u32 staterr;
334 int i = 0;
335
336 if (!netif_msg_hw(adapter))
337 return;
338
339 /* Print netdevice Info */
340 if (netdev) {
341 dev_info(&adapter->pdev->dev, "Net device Info\n");
342 printk(KERN_INFO "Device Name state "
343 "trans_start last_rx\n");
344 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
345 netdev->name,
346 netdev->state,
347 netdev->trans_start,
348 netdev->last_rx);
349 }
350
351 /* Print Registers */
352 dev_info(&adapter->pdev->dev, "Register Dump\n");
353 printk(KERN_INFO " Register Name Value\n");
354 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
355 reginfo->name; reginfo++) {
356 igb_regdump(hw, reginfo);
357 }
358
359 /* Print TX Ring Summary */
360 if (!netdev || !netif_running(netdev))
361 goto exit;
362
363 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
364 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
365 " leng ntw timestamp\n");
366 for (n = 0; n < adapter->num_tx_queues; n++) {
367 tx_ring = adapter->tx_ring[n];
368 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
369 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
370 n, tx_ring->next_to_use, tx_ring->next_to_clean,
371 (u64)buffer_info->dma,
372 buffer_info->length,
373 buffer_info->next_to_watch,
374 (u64)buffer_info->time_stamp);
375 }
376
377 /* Print TX Rings */
378 if (!netif_msg_tx_done(adapter))
379 goto rx_ring_summary;
380
381 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
382
383 /* Transmit Descriptor Formats
384 *
385 * Advanced Transmit Descriptor
386 * +--------------------------------------------------------------+
387 * 0 | Buffer Address [63:0] |
388 * +--------------------------------------------------------------+
389 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
390 * +--------------------------------------------------------------+
391 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 */
393
394 for (n = 0; n < adapter->num_tx_queues; n++) {
395 tx_ring = adapter->tx_ring[n];
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "T [desc] [address 63:0 ] "
400 "[PlPOCIStDDM Ln] [bi->dma ] "
401 "leng ntw timestamp bi->skb\n");
402
403 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
404 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
405 buffer_info = &tx_ring->buffer_info[i];
406 u0 = (struct my_u0 *)tx_desc;
407 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
408 " %04X %3X %016llX %p", i,
409 le64_to_cpu(u0->a),
410 le64_to_cpu(u0->b),
411 (u64)buffer_info->dma,
412 buffer_info->length,
413 buffer_info->next_to_watch,
414 (u64)buffer_info->time_stamp,
415 buffer_info->skb);
416 if (i == tx_ring->next_to_use &&
417 i == tx_ring->next_to_clean)
418 printk(KERN_CONT " NTC/U\n");
419 else if (i == tx_ring->next_to_use)
420 printk(KERN_CONT " NTU\n");
421 else if (i == tx_ring->next_to_clean)
422 printk(KERN_CONT " NTC\n");
423 else
424 printk(KERN_CONT "\n");
425
426 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
427 print_hex_dump(KERN_INFO, "",
428 DUMP_PREFIX_ADDRESS,
429 16, 1, phys_to_virt(buffer_info->dma),
430 buffer_info->length, true);
431 }
432 }
433
434 /* Print RX Rings Summary */
435rx_ring_summary:
436 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
437 printk(KERN_INFO "Queue [NTU] [NTC]\n");
438 for (n = 0; n < adapter->num_rx_queues; n++) {
439 rx_ring = adapter->rx_ring[n];
440 printk(KERN_INFO " %5d %5X %5X\n", n,
441 rx_ring->next_to_use, rx_ring->next_to_clean);
442 }
443
444 /* Print RX Rings */
445 if (!netif_msg_rx_status(adapter))
446 goto exit;
447
448 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
449
450 /* Advanced Receive Descriptor (Read) Format
451 * 63 1 0
452 * +-----------------------------------------------------+
453 * 0 | Packet Buffer Address [63:1] |A0/NSE|
454 * +----------------------------------------------+------+
455 * 8 | Header Buffer Address [63:1] | DD |
456 * +-----------------------------------------------------+
457 *
458 *
459 * Advanced Receive Descriptor (Write-Back) Format
460 *
461 * 63 48 47 32 31 30 21 20 17 16 4 3 0
462 * +------------------------------------------------------+
463 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
464 * | Checksum Ident | | | | Type | Type |
465 * +------------------------------------------------------+
466 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
467 * +------------------------------------------------------+
468 * 63 48 47 32 31 20 19 0
469 */
470
471 for (n = 0; n < adapter->num_rx_queues; n++) {
472 rx_ring = adapter->rx_ring[n];
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "R [desc] [ PktBuf A0] "
477 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
478 "<-- Adv Rx Read format\n");
479 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
480 "[vl er S cks ln] ---------------- [bi->skb] "
481 "<-- Adv Rx Write-Back format\n");
482
483 for (i = 0; i < rx_ring->count; i++) {
484 buffer_info = &rx_ring->buffer_info[i];
485 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
486 u0 = (struct my_u0 *)rx_desc;
487 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
488 if (staterr & E1000_RXD_STAT_DD) {
489 /* Descriptor Done */
490 printk(KERN_INFO "RWB[0x%03X] %016llX "
491 "%016llX ---------------- %p", i,
492 le64_to_cpu(u0->a),
493 le64_to_cpu(u0->b),
494 buffer_info->skb);
495 } else {
496 printk(KERN_INFO "R [0x%03X] %016llX "
497 "%016llX %016llX %p", i,
498 le64_to_cpu(u0->a),
499 le64_to_cpu(u0->b),
500 (u64)buffer_info->dma,
501 buffer_info->skb);
502
503 if (netif_msg_pktdata(adapter)) {
504 print_hex_dump(KERN_INFO, "",
505 DUMP_PREFIX_ADDRESS,
506 16, 1,
507 phys_to_virt(buffer_info->dma),
508 rx_ring->rx_buffer_len, true);
509 if (rx_ring->rx_buffer_len
510 < IGB_RXBUFFER_1024)
511 print_hex_dump(KERN_INFO, "",
512 DUMP_PREFIX_ADDRESS,
513 16, 1,
514 phys_to_virt(
515 buffer_info->page_dma +
516 buffer_info->page_offset),
517 PAGE_SIZE/2, true);
518 }
519 }
520
521 if (i == rx_ring->next_to_use)
522 printk(KERN_CONT " NTU\n");
523 else if (i == rx_ring->next_to_clean)
524 printk(KERN_CONT " NTC\n");
525 else
526 printk(KERN_CONT "\n");
527
528 }
529 }
530
531exit:
532 return;
533}
534
535
38c845c7
PO		 536/**
537 * igb_read_clock - read raw cycle counter (to be used by time counter)
538 */
539static cycle_t igb_read_clock(const struct cyclecounter *tc)
540{
541 struct igb_adapter *adapter =
542 container_of(tc, struct igb_adapter, cycles);
543 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 544 u64 stamp = 0;
545 int shift = 0;
38c845c7 546
55cac248
AD		 547 /*
548 * The timestamp latches on lowest register read. For the 82580
549 * the lowest register is SYSTIMR instead of SYSTIML. However we never
550 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
551 */
552 if (hw->mac.type == e1000_82580) {
553 stamp = rd32(E1000_SYSTIMR) >> 8;
554 shift = IGB_82580_TSYNC_SHIFT;
555 }
556
c5b9bd5e
AD		 557 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
558 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 559 return stamp;
560}
561
9d5c8243 562/**
c041076a 563 * igb_get_hw_dev - return device
9d5c8243
AK		 564 * used by hardware layer to print debugging information
565 **/
c041076a 566struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 567{
568 struct igb_adapter *adapter = hw->back;
c041076a 569 return adapter->netdev;
9d5c8243 570}
38c845c7 571
9d5c8243
AK		 572/**
573 * igb_init_module - Driver Registration Routine
574 *
575 * igb_init_module is the first routine called when the driver is
576 * loaded. All it does is register with the PCI subsystem.
577 **/
578static int __init igb_init_module(void)
579{
580 int ret;
581 printk(KERN_INFO "%s - version %s\n",
582 igb_driver_string, igb_driver_version);
583
584 printk(KERN_INFO "%s\n", igb_copyright);
585
421e02f0 586#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 587 dca_register_notify(&dca_notifier);
588#endif
bbd98fe4 589 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 590 return ret;
591}
592
593module_init(igb_init_module);
594
595/**
596 * igb_exit_module - Driver Exit Cleanup Routine
597 *
598 * igb_exit_module is called just before the driver is removed
599 * from memory.
600 **/
601static void __exit igb_exit_module(void)
602{
421e02f0 603#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 604 dca_unregister_notify(&dca_notifier);
605#endif
9d5c8243
AK		 606 pci_unregister_driver(&igb_driver);
607}
608
609module_exit(igb_exit_module);
610
26bc19ec
AD		 611#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
612/**
613 * igb_cache_ring_register - Descriptor ring to register mapping
614 * @adapter: board private structure to initialize
615 *
616 * Once we know the feature-set enabled for the device, we'll cache
617 * the register offset the descriptor ring is assigned to.
618 **/
619static void igb_cache_ring_register(struct igb_adapter *adapter)
620{
ee1b9f06 621 int i = 0, j = 0;
047e0030 622 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 623
624 switch (adapter->hw.mac.type) {
625 case e1000_82576:
626 /* The queues are allocated for virtualization such that VF 0
627 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
628 * In order to avoid collision we start at the first free queue
629 * and continue consuming queues in the same sequence
630 */
ee1b9f06 631 if (adapter->vfs_allocated_count) {
a99955fc 632 for (; i < adapter->rss_queues; i++)
3025a446
AD		 633 adapter->rx_ring[i]->reg_idx = rbase_offset +
634 Q_IDX_82576(i);
ee1b9f06 635 }
26bc19ec 636 case e1000_82575:
55cac248 637 case e1000_82580:
d2ba2ed8 638 case e1000_i350:
26bc19ec 639 default:
ee1b9f06 640 for (; i < adapter->num_rx_queues; i++)
3025a446 641 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 642 for (; j < adapter->num_tx_queues; j++)
3025a446 643 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 644 break;
645 }
646}
647
047e0030
AD		 648static void igb_free_queues(struct igb_adapter *adapter)
649{
3025a446 650 int i;
047e0030 651
3025a446
AD		 652 for (i = 0; i < adapter->num_tx_queues; i++) {
653 kfree(adapter->tx_ring[i]);
654 adapter->tx_ring[i] = NULL;
655 }
656 for (i = 0; i < adapter->num_rx_queues; i++) {
657 kfree(adapter->rx_ring[i]);
658 adapter->rx_ring[i] = NULL;
659 }
047e0030
AD		 660 adapter->num_rx_queues = 0;
661 adapter->num_tx_queues = 0;
662}
663
9d5c8243
AK		 664/**
665 * igb_alloc_queues - Allocate memory for all rings
666 * @adapter: board private structure to initialize
667 *
668 * We allocate one ring per queue at run-time since we don't know the
669 * number of queues at compile-time.
670 **/
671static int igb_alloc_queues(struct igb_adapter *adapter)
672{
3025a446 673 struct igb_ring *ring;
9d5c8243
AK		 674 int i;
675
661086df 676 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 677 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 if (!ring)
679 goto err;
68fd9910 680 ring->count = adapter->tx_ring_count;
661086df 681 ring->queue_index = i;
59d71989 682 ring->dev = &adapter->pdev->dev;
e694e964 683 ring->netdev = adapter->netdev;
85ad76b2
AD		 684 /* For 82575, context index must be unique per ring. */
685 if (adapter->hw.mac.type == e1000_82575)
686 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 687 adapter->tx_ring[i] = ring;
661086df 688 }
85ad76b2 689
9d5c8243 690 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 691 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 if (!ring)
693 goto err;
68fd9910 694 ring->count = adapter->rx_ring_count;
844290e5 695 ring->queue_index = i;
59d71989 696 ring->dev = &adapter->pdev->dev;
e694e964 697 ring->netdev = adapter->netdev;
4c844851 698 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 699 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
700 /* set flag indicating ring supports SCTP checksum offload */
701 if (adapter->hw.mac.type >= e1000_82576)
702 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 703 adapter->rx_ring[i] = ring;
9d5c8243 704 }
26bc19ec
AD		 705
706 igb_cache_ring_register(adapter);
9d5c8243 707
047e0030 708 return 0;
a88f10ec 709
047e0030
AD		 710err:
711 igb_free_queues(adapter);
d1a8c9e1 712
047e0030 713 return -ENOMEM;
a88f10ec
AD		 714}
715
9d5c8243 716#define IGB_N0_QUEUE -1
047e0030 717static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 718{
719 u32 msixbm = 0;
047e0030 720 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 721 struct e1000_hw *hw = &adapter->hw;
2d064c06 722 u32 ivar, index;
047e0030
AD		 723 int rx_queue = IGB_N0_QUEUE;
724 int tx_queue = IGB_N0_QUEUE;
725
726 if (q_vector->rx_ring)
727 rx_queue = q_vector->rx_ring->reg_idx;
728 if (q_vector->tx_ring)
729 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 730
731 switch (hw->mac.type) {
732 case e1000_82575:
9d5c8243
AK		 733 /* The 82575 assigns vectors using a bitmask, which matches the
734 bitmask for the EICR/EIMS/EIMC registers. To assign one
735 or more queues to a vector, we write the appropriate bits
736 into the MSIXBM register for that vector. */
047e0030 737 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 739 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 740 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 741 if (!adapter->msix_entries && msix_vector == 0)
742 msixbm |= E1000_EIMS_OTHER;
9d5c8243 743 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 744 q_vector->eims_value = msixbm;
2d064c06
AD		 745 break;
746 case e1000_82576:
26bc19ec 747 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 748 Each queue has a single entry in the table to which we write
749 a vector number along with a "valid" bit. Sadly, the layout
750 of the table is somewhat counterintuitive. */
751 if (rx_queue > IGB_N0_QUEUE) {
047e0030 752 index = (rx_queue & 0x7);
2d064c06 753 ivar = array_rd32(E1000_IVAR0, index);
047e0030 754 if (rx_queue < 8) {
26bc19ec
AD		 755 /* vector goes into low byte of register */
756 ivar = ivar & 0xFFFFFF00;
757 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 758 } else {
759 /* vector goes into third byte of register */
760 ivar = ivar & 0xFF00FFFF;
761 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 762 }
2d064c06
AD		 763 array_wr32(E1000_IVAR0, index, ivar);
764 }
765 if (tx_queue > IGB_N0_QUEUE) {
047e0030 766 index = (tx_queue & 0x7);
2d064c06 767 ivar = array_rd32(E1000_IVAR0, index);
047e0030 768 if (tx_queue < 8) {
26bc19ec
AD		 769 /* vector goes into second byte of register */
770 ivar = ivar & 0xFFFF00FF;
771 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 772 } else {
773 /* vector goes into high byte of register */
774 ivar = ivar & 0x00FFFFFF;
775 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 776 }
2d064c06
AD		 777 array_wr32(E1000_IVAR0, index, ivar);
778 }
047e0030 779 q_vector->eims_value = 1 << msix_vector;
2d064c06 780 break;
55cac248 781 case e1000_82580:
d2ba2ed8 782 case e1000_i350:
55cac248
AD		 783 /* 82580 uses the same table-based approach as 82576 but has fewer
784 entries as a result we carry over for queues greater than 4. */
785 if (rx_queue > IGB_N0_QUEUE) {
786 index = (rx_queue >> 1);
787 ivar = array_rd32(E1000_IVAR0, index);
788 if (rx_queue & 0x1) {
789 /* vector goes into third byte of register */
790 ivar = ivar & 0xFF00FFFF;
791 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
792 } else {
793 /* vector goes into low byte of register */
794 ivar = ivar & 0xFFFFFF00;
795 ivar |= msix_vector | E1000_IVAR_VALID;
796 }
797 array_wr32(E1000_IVAR0, index, ivar);
798 }
799 if (tx_queue > IGB_N0_QUEUE) {
800 index = (tx_queue >> 1);
801 ivar = array_rd32(E1000_IVAR0, index);
802 if (tx_queue & 0x1) {
803 /* vector goes into high byte of register */
804 ivar = ivar & 0x00FFFFFF;
805 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
806 } else {
807 /* vector goes into second byte of register */
808 ivar = ivar & 0xFFFF00FF;
809 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
810 }
811 array_wr32(E1000_IVAR0, index, ivar);
812 }
813 q_vector->eims_value = 1 << msix_vector;
814 break;
2d064c06
AD		 815 default:
816 BUG();
817 break;
818 }
26b39276
AD		 819
820 /* add q_vector eims value to global eims_enable_mask */
821 adapter->eims_enable_mask |= q_vector->eims_value;
822
823 /* configure q_vector to set itr on first interrupt */
824 q_vector->set_itr = 1;
9d5c8243
AK		 825}
826
827/**
828 * igb_configure_msix - Configure MSI-X hardware
829 *
830 * igb_configure_msix sets up the hardware to properly
831 * generate MSI-X interrupts.
832 **/
833static void igb_configure_msix(struct igb_adapter *adapter)
834{
835 u32 tmp;
836 int i, vector = 0;
837 struct e1000_hw *hw = &adapter->hw;
838
839 adapter->eims_enable_mask = 0;
9d5c8243
AK		 840
841 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 842 switch (hw->mac.type) {
843 case e1000_82575:
9d5c8243
AK		 844 tmp = rd32(E1000_CTRL_EXT);
845 /* enable MSI-X PBA support*/
846 tmp |= E1000_CTRL_EXT_PBA_CLR;
847
848 /* Auto-Mask interrupts upon ICR read. */
849 tmp |= E1000_CTRL_EXT_EIAME;
850 tmp |= E1000_CTRL_EXT_IRCA;
851
852 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 853
854 /* enable msix_other interrupt */
855 array_wr32(E1000_MSIXBM(0), vector++,
856 E1000_EIMS_OTHER);
844290e5 857 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 858
2d064c06
AD		 859 break;
860
861 case e1000_82576:
55cac248 862 case e1000_82580:
d2ba2ed8 863 case e1000_i350:
047e0030
AD		 864 /* Turn on MSI-X capability first, or our settings
865 * won't stick. And it will take days to debug. */
866 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
867 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 E1000_GPIE_NSICR);
869
870 /* enable msix_other interrupt */
871 adapter->eims_other = 1 << vector;
2d064c06 872 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 873
047e0030 874 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 875 break;
876 default:
877 /* do nothing, since nothing else supports MSI-X */
878 break;
879 } /* switch (hw->mac.type) */
047e0030
AD		 880
881 adapter->eims_enable_mask |= adapter->eims_other;
882
26b39276
AD		 883 for (i = 0; i < adapter->num_q_vectors; i++)
884 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 885
9d5c8243
AK		 886 wrfl();
887}
888
889/**
890 * igb_request_msix - Initialize MSI-X interrupts
891 *
892 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 * kernel.
894 **/
895static int igb_request_msix(struct igb_adapter *adapter)
896{
897 struct net_device *netdev = adapter->netdev;
047e0030 898 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 899 int i, err = 0, vector = 0;
900
047e0030 901 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 902 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 903 if (err)
904 goto out;
905 vector++;
906
907 for (i = 0; i < adapter->num_q_vectors; i++) {
908 struct igb_q_vector *q_vector = adapter->q_vector[i];
909
910 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
911
912 if (q_vector->rx_ring && q_vector->tx_ring)
913 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
914 q_vector->rx_ring->queue_index);
915 else if (q_vector->tx_ring)
916 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
917 q_vector->tx_ring->queue_index);
918 else if (q_vector->rx_ring)
919 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
920 q_vector->rx_ring->queue_index);
9d5c8243 921 else
047e0030
AD		 922 sprintf(q_vector->name, "%s-unused", netdev->name);
923
9d5c8243 924 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 925 igb_msix_ring, 0, q_vector->name,
047e0030 926 q_vector);
9d5c8243
AK		 927 if (err)
928 goto out;
9d5c8243
AK		 929 vector++;
930 }
931
9d5c8243
AK		 932 igb_configure_msix(adapter);
933 return 0;
934out:
935 return err;
936}
937
938static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
939{
940 if (adapter->msix_entries) {
941 pci_disable_msix(adapter->pdev);
942 kfree(adapter->msix_entries);
943 adapter->msix_entries = NULL;
047e0030 944 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 945 pci_disable_msi(adapter->pdev);
047e0030 946 }
9d5c8243
AK		 947}
948
047e0030
AD		 949/**
950 * igb_free_q_vectors - Free memory allocated for interrupt vectors
951 * @adapter: board private structure to initialize
952 *
953 * This function frees the memory allocated to the q_vectors. In addition if
954 * NAPI is enabled it will delete any references to the NAPI struct prior
955 * to freeing the q_vector.
956 **/
957static void igb_free_q_vectors(struct igb_adapter *adapter)
958{
959 int v_idx;
960
961 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
962 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
963 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 964 if (!q_vector)
965 continue;
047e0030
AD		 966 netif_napi_del(&q_vector->napi);
967 kfree(q_vector);
968 }
969 adapter->num_q_vectors = 0;
970}
971
972/**
973 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
974 *
975 * This function resets the device so that it has 0 rx queues, tx queues, and
976 * MSI-X interrupts allocated.
977 */
978static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
979{
980 igb_free_queues(adapter);
981 igb_free_q_vectors(adapter);
982 igb_reset_interrupt_capability(adapter);
983}
9d5c8243
AK		 984
985/**
986 * igb_set_interrupt_capability - set MSI or MSI-X if supported
987 *
988 * Attempt to configure interrupts using the best available
989 * capabilities of the hardware and kernel.
990 **/
21adef3e 991static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 992{
993 int err;
994 int numvecs, i;
995
83b7180d 996 /* Number of supported queues. */
a99955fc 997 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 998 if (adapter->vfs_allocated_count)
999 adapter->num_tx_queues = 1;
1000 else
1001 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1002
047e0030
AD		 1003 /* start with one vector for every rx queue */
1004 numvecs = adapter->num_rx_queues;
1005
3ad2f3fb 1006 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1007 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1008 numvecs += adapter->num_tx_queues;
047e0030
AD		 1009
1010 /* store the number of vectors reserved for queues */
1011 adapter->num_q_vectors = numvecs;
1012
1013 /* add 1 vector for link status interrupts */
1014 numvecs++;
9d5c8243
AK		 1015 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1016 GFP_KERNEL);
1017 if (!adapter->msix_entries)
1018 goto msi_only;
1019
1020 for (i = 0; i < numvecs; i++)
1021 adapter->msix_entries[i].entry = i;
1022
1023 err = pci_enable_msix(adapter->pdev,
1024 adapter->msix_entries,
1025 numvecs);
1026 if (err == 0)
34a20e89 1027 goto out;
9d5c8243
AK		 1028
1029 igb_reset_interrupt_capability(adapter);
1030
1031 /* If we can't do MSI-X, try MSI */
1032msi_only:
2a3abf6d
AD		 1033#ifdef CONFIG_PCI_IOV
1034 /* disable SR-IOV for non MSI-X configurations */
1035 if (adapter->vf_data) {
1036 struct e1000_hw *hw = &adapter->hw;
1037 /* disable iov and allow time for transactions to clear */
1038 pci_disable_sriov(adapter->pdev);
1039 msleep(500);
1040
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1044 msleep(100);
1045 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 }
1047#endif
4fc82adf 1048 adapter->vfs_allocated_count = 0;
a99955fc 1049 adapter->rss_queues = 1;
4fc82adf 1050 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1051 adapter->num_rx_queues = 1;
661086df 1052 adapter->num_tx_queues = 1;
047e0030 1053 adapter->num_q_vectors = 1;
9d5c8243 1054 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1055 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1056out:
21adef3e
BH		 1057 /* Notify the stack of the (possibly) reduced queue counts. */
1058 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1059 return netif_set_real_num_rx_queues(adapter->netdev,
1060 adapter->num_rx_queues);
9d5c8243
AK		 1061}
1062
047e0030
AD		 1063/**
1064 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1065 * @adapter: board private structure to initialize
1066 *
1067 * We allocate one q_vector per queue interrupt. If allocation fails we
1068 * return -ENOMEM.
1069 **/
1070static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1071{
1072 struct igb_q_vector *q_vector;
1073 struct e1000_hw *hw = &adapter->hw;
1074 int v_idx;
1075
1076 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1077 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1078 if (!q_vector)
1079 goto err_out;
1080 q_vector->adapter = adapter;
047e0030
AD		 1081 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1082 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1083 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1084 adapter->q_vector[v_idx] = q_vector;
1085 }
1086 return 0;
1087
1088err_out:
fe0592b4 1089 igb_free_q_vectors(adapter);
047e0030
AD		 1090 return -ENOMEM;
1091}
1092
1093static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1094 int ring_idx, int v_idx)
1095{
3025a446 1096 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1097
3025a446 1098 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1099 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1100 q_vector->itr_val = adapter->rx_itr_setting;
1101 if (q_vector->itr_val && q_vector->itr_val <= 3)
1102 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1103}
1104
1105static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1111 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->tx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117/**
1118 * igb_map_ring_to_vector - maps allocated queues to vectors
1119 *
1120 * This function maps the recently allocated queues to vectors.
1121 **/
1122static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123{
1124 int i;
1125 int v_idx = 0;
1126
1127 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1128 (adapter->num_q_vectors < adapter->num_tx_queues))
1129 return -ENOMEM;
1130
1131 if (adapter->num_q_vectors >=
1132 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1133 for (i = 0; i < adapter->num_rx_queues; i++)
1134 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1135 for (i = 0; i < adapter->num_tx_queues; i++)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1137 } else {
1138 for (i = 0; i < adapter->num_rx_queues; i++) {
1139 if (i < adapter->num_tx_queues)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1141 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1142 }
1143 for (; i < adapter->num_tx_queues; i++)
1144 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1145 }
1146 return 0;
1147}
1148
1149/**
1150 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1151 *
1152 * This function initializes the interrupts and allocates all of the queues.
1153 **/
1154static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1155{
1156 struct pci_dev *pdev = adapter->pdev;
1157 int err;
1158
21adef3e
BH		 1159 err = igb_set_interrupt_capability(adapter);
1160 if (err)
1161 return err;
047e0030
AD		 1162
1163 err = igb_alloc_q_vectors(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1166 goto err_alloc_q_vectors;
1167 }
1168
1169 err = igb_alloc_queues(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1172 goto err_alloc_queues;
1173 }
1174
1175 err = igb_map_ring_to_vector(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1178 goto err_map_queues;
1179 }
1180
1181
1182 return 0;
1183err_map_queues:
1184 igb_free_queues(adapter);
1185err_alloc_queues:
1186 igb_free_q_vectors(adapter);
1187err_alloc_q_vectors:
1188 igb_reset_interrupt_capability(adapter);
1189 return err;
1190}
1191
9d5c8243
AK		 1192/**
1193 * igb_request_irq - initialize interrupts
1194 *
1195 * Attempts to configure interrupts using the best available
1196 * capabilities of the hardware and kernel.
1197 **/
1198static int igb_request_irq(struct igb_adapter *adapter)
1199{
1200 struct net_device *netdev = adapter->netdev;
047e0030 1201 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1202 int err = 0;
1203
1204 if (adapter->msix_entries) {
1205 err = igb_request_msix(adapter);
844290e5 1206 if (!err)
9d5c8243 1207 goto request_done;
9d5c8243 1208 /* fall back to MSI */
047e0030 1209 igb_clear_interrupt_scheme(adapter);
9d5c8243 1210 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1211 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1212 igb_free_all_tx_resources(adapter);
1213 igb_free_all_rx_resources(adapter);
047e0030 1214 adapter->num_tx_queues = 1;
9d5c8243 1215 adapter->num_rx_queues = 1;
047e0030
AD		 1216 adapter->num_q_vectors = 1;
1217 err = igb_alloc_q_vectors(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for vectors\n");
1221 goto request_done;
1222 }
1223 err = igb_alloc_queues(adapter);
1224 if (err) {
1225 dev_err(&pdev->dev,
1226 "Unable to allocate memory for queues\n");
1227 igb_free_q_vectors(adapter);
1228 goto request_done;
1229 }
1230 igb_setup_all_tx_resources(adapter);
1231 igb_setup_all_rx_resources(adapter);
844290e5 1232 } else {
feeb2721 1233 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1234 }
844290e5 1235
7dfc16fa 1236 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1237 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1238 netdev->name, adapter);
9d5c8243
AK		 1239 if (!err)
1240 goto request_done;
047e0030 1241
9d5c8243
AK		 1242 /* fall back to legacy interrupts */
1243 igb_reset_interrupt_capability(adapter);
7dfc16fa 1244 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1245 }
1246
a0607fd3 1247 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1248 netdev->name, adapter);
9d5c8243 1249
6cb5e577 1250 if (err)
9d5c8243
AK		 1251 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 err);
9d5c8243
AK		 1253
1254request_done:
1255 return err;
1256}
1257
1258static void igb_free_irq(struct igb_adapter *adapter)
1259{
9d5c8243
AK		 1260 if (adapter->msix_entries) {
1261 int vector = 0, i;
1262
047e0030 1263 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1264
047e0030
AD		 1265 for (i = 0; i < adapter->num_q_vectors; i++) {
1266 struct igb_q_vector *q_vector = adapter->q_vector[i];
1267 free_irq(adapter->msix_entries[vector++].vector,
1268 q_vector);
1269 }
1270 } else {
1271 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1272 }
9d5c8243
AK		 1273}
1274
1275/**
1276 * igb_irq_disable - Mask off interrupt generation on the NIC
1277 * @adapter: board private structure
1278 **/
1279static void igb_irq_disable(struct igb_adapter *adapter)
1280{
1281 struct e1000_hw *hw = &adapter->hw;
1282
25568a53
AD		 1283 /*
1284 * we need to be careful when disabling interrupts. The VFs are also
1285 * mapped into these registers and so clearing the bits can cause
1286 * issues on the VF drivers so we only need to clear what we set
1287 */
9d5c8243 1288 if (adapter->msix_entries) {
2dfd1212
AD		 1289 u32 regval = rd32(E1000_EIAM);
1290 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_EIMC, adapter->eims_enable_mask);
1292 regval = rd32(E1000_EIAC);
1293 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1294 }
844290e5
PW		 1295
1296 wr32(E1000_IAM, 0);
9d5c8243
AK		 1297 wr32(E1000_IMC, ~0);
1298 wrfl();
81a61859
ET		 1299 if (adapter->msix_entries) {
1300 int i;
1301 for (i = 0; i < adapter->num_q_vectors; i++)
1302 synchronize_irq(adapter->msix_entries[i].vector);
1303 } else {
1304 synchronize_irq(adapter->pdev->irq);
1305 }
9d5c8243
AK		 1306}
1307
1308/**
1309 * igb_irq_enable - Enable default interrupt generation settings
1310 * @adapter: board private structure
1311 **/
1312static void igb_irq_enable(struct igb_adapter *adapter)
1313{
1314 struct e1000_hw *hw = &adapter->hw;
1315
1316 if (adapter->msix_entries) {
25568a53 1317 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1318 u32 regval = rd32(E1000_EIAC);
1319 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1320 regval = rd32(E1000_EIAM);
1321 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1322 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1323 if (adapter->vfs_allocated_count) {
4ae196df 1324 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1325 ims |= E1000_IMS_VMMB;
1326 }
55cac248
AD		 1327 if (adapter->hw.mac.type == e1000_82580)
1328 ims |= E1000_IMS_DRSTA;
1329
25568a53 1330 wr32(E1000_IMS, ims);
844290e5 1331 } else {
55cac248
AD		 1332 wr32(E1000_IMS, IMS_ENABLE_MASK |
1333 E1000_IMS_DRSTA);
1334 wr32(E1000_IAM, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
844290e5 1336 }
9d5c8243
AK		 1337}
1338
1339static void igb_update_mng_vlan(struct igb_adapter *adapter)
1340{
51466239 1341 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1342 u16 vid = adapter->hw.mng_cookie.vlan_id;
1343 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1344
1345 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1346 /* add VID to filter table */
1347 igb_vfta_set(hw, vid, true);
1348 adapter->mng_vlan_id = vid;
1349 } else {
1350 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1351 }
1352
1353 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1354 (vid != old_vid) &&
1355 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1356 /* remove VID from filter table */
1357 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1358 }
1359}
1360
1361/**
1362 * igb_release_hw_control - release control of the h/w to f/w
1363 * @adapter: address of board private structure
1364 *
1365 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1366 * For ASF and Pass Through versions of f/w this means that the
1367 * driver is no longer loaded.
1368 *
1369 **/
1370static void igb_release_hw_control(struct igb_adapter *adapter)
1371{
1372 struct e1000_hw *hw = &adapter->hw;
1373 u32 ctrl_ext;
1374
1375 /* Let firmware take over control of h/w */
1376 ctrl_ext = rd32(E1000_CTRL_EXT);
1377 wr32(E1000_CTRL_EXT,
1378 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1379}
1380
9d5c8243
AK		 1381/**
1382 * igb_get_hw_control - get control of the h/w from f/w
1383 * @adapter: address of board private structure
1384 *
1385 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1386 * For ASF and Pass Through versions of f/w this means that
1387 * the driver is loaded.
1388 *
1389 **/
1390static void igb_get_hw_control(struct igb_adapter *adapter)
1391{
1392 struct e1000_hw *hw = &adapter->hw;
1393 u32 ctrl_ext;
1394
1395 /* Let firmware know the driver has taken over */
1396 ctrl_ext = rd32(E1000_CTRL_EXT);
1397 wr32(E1000_CTRL_EXT,
1398 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1399}
1400
9d5c8243
AK		 1401/**
1402 * igb_configure - configure the hardware for RX and TX
1403 * @adapter: private board structure
1404 **/
1405static void igb_configure(struct igb_adapter *adapter)
1406{
1407 struct net_device *netdev = adapter->netdev;
1408 int i;
1409
1410 igb_get_hw_control(adapter);
ff41f8dc 1411 igb_set_rx_mode(netdev);
9d5c8243
AK		 1412
1413 igb_restore_vlan(adapter);
9d5c8243 1414
85b430b4 1415 igb_setup_tctl(adapter);
06cf2666 1416 igb_setup_mrqc(adapter);
9d5c8243 1417 igb_setup_rctl(adapter);
85b430b4
AD		 1418
1419 igb_configure_tx(adapter);
9d5c8243 1420 igb_configure_rx(adapter);
662d7205
AD		 1421
1422 igb_rx_fifo_flush_82575(&adapter->hw);
1423
c493ea45 1424 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1425 * at least 1 descriptor unused to make sure
1426 * next_to_use != next_to_clean */
1427 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1428 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1429 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1430 }
9d5c8243
AK		 1431}
1432
88a268c1
NN		 1433/**
1434 * igb_power_up_link - Power up the phy/serdes link
1435 * @adapter: address of board private structure
1436 **/
1437void igb_power_up_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_up_phy_copper(&adapter->hw);
1441 else
1442 igb_power_up_serdes_link_82575(&adapter->hw);
1443}
1444
1445/**
1446 * igb_power_down_link - Power down the phy/serdes link
1447 * @adapter: address of board private structure
1448 */
1449static void igb_power_down_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_down_phy_copper_82575(&adapter->hw);
1453 else
1454 igb_shutdown_serdes_link_82575(&adapter->hw);
1455}
9d5c8243
AK		 1456
1457/**
1458 * igb_up - Open the interface and prepare it to handle traffic
1459 * @adapter: board private structure
1460 **/
9d5c8243
AK		 1461int igb_up(struct igb_adapter *adapter)
1462{
1463 struct e1000_hw *hw = &adapter->hw;
1464 int i;
1465
1466 /* hardware has been reset, we need to reload some things */
1467 igb_configure(adapter);
1468
1469 clear_bit(__IGB_DOWN, &adapter->state);
1470
047e0030
AD		 1471 for (i = 0; i < adapter->num_q_vectors; i++) {
1472 struct igb_q_vector *q_vector = adapter->q_vector[i];
1473 napi_enable(&q_vector->napi);
1474 }
844290e5 1475 if (adapter->msix_entries)
9d5c8243 1476 igb_configure_msix(adapter);
feeb2721
AD		 1477 else
1478 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1479
1480 /* Clear any pending interrupts. */
1481 rd32(E1000_ICR);
1482 igb_irq_enable(adapter);
1483
d4960307
AD		 1484 /* notify VFs that reset has been completed */
1485 if (adapter->vfs_allocated_count) {
1486 u32 reg_data = rd32(E1000_CTRL_EXT);
1487 reg_data |= E1000_CTRL_EXT_PFRSTD;
1488 wr32(E1000_CTRL_EXT, reg_data);
1489 }
1490
4cb9be7a
JB		 1491 netif_tx_start_all_queues(adapter->netdev);
1492
25568a53
AD		 1493 /* start the watchdog. */
1494 hw->mac.get_link_status = 1;
1495 schedule_work(&adapter->watchdog_task);
1496
9d5c8243
AK		 1497 return 0;
1498}
1499
1500void igb_down(struct igb_adapter *adapter)
1501{
9d5c8243 1502 struct net_device *netdev = adapter->netdev;
330a6d6a 1503 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1504 u32 tctl, rctl;
1505 int i;
1506
1507 /* signal that we're down so the interrupt handler does not
1508 * reschedule our watchdog timer */
1509 set_bit(__IGB_DOWN, &adapter->state);
1510
1511 /* disable receives in the hardware */
1512 rctl = rd32(E1000_RCTL);
1513 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1514 /* flush and sleep below */
1515
fd2ea0a7 1516 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1517
1518 /* disable transmits in the hardware */
1519 tctl = rd32(E1000_TCTL);
1520 tctl &= ~E1000_TCTL_EN;
1521 wr32(E1000_TCTL, tctl);
1522 /* flush both disables and wait for them to finish */
1523 wrfl();
1524 msleep(10);
1525
047e0030
AD		 1526 for (i = 0; i < adapter->num_q_vectors; i++) {
1527 struct igb_q_vector *q_vector = adapter->q_vector[i];
1528 napi_disable(&q_vector->napi);
1529 }
9d5c8243 1530
9d5c8243
AK		 1531 igb_irq_disable(adapter);
1532
1533 del_timer_sync(&adapter->watchdog_timer);
1534 del_timer_sync(&adapter->phy_info_timer);
1535
9d5c8243 1536 netif_carrier_off(netdev);
04fe6358
AD		 1537
1538 /* record the stats before reset*/
12dcd86b
ED		 1539 spin_lock(&adapter->stats64_lock);
1540 igb_update_stats(adapter, &adapter->stats64);
1541 spin_unlock(&adapter->stats64_lock);
04fe6358 1542
9d5c8243
AK		 1543 adapter->link_speed = 0;
1544 adapter->link_duplex = 0;
1545
3023682e
JK		 1546 if (!pci_channel_offline(adapter->pdev))
1547 igb_reset(adapter);
9d5c8243
AK		 1548 igb_clean_all_tx_rings(adapter);
1549 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1550#ifdef CONFIG_IGB_DCA
1551
1552 /* since we reset the hardware DCA settings were cleared */
1553 igb_setup_dca(adapter);
1554#endif
9d5c8243
AK		 1555}
1556
1557void igb_reinit_locked(struct igb_adapter *adapter)
1558{
1559 WARN_ON(in_interrupt());
1560 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1561 msleep(1);
1562 igb_down(adapter);
1563 igb_up(adapter);
1564 clear_bit(__IGB_RESETTING, &adapter->state);
1565}
1566
1567void igb_reset(struct igb_adapter *adapter)
1568{
090b1795 1569 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1570 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1571 struct e1000_mac_info *mac = &hw->mac;
1572 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1573 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1574 u16 hwm;
1575
1576 /* Repartition Pba for greater than 9k mtu
1577 * To take effect CTRL.RST is required.
1578 */
fa4dfae0 1579 switch (mac->type) {
d2ba2ed8 1580 case e1000_i350:
55cac248
AD		 1581 case e1000_82580:
1582 pba = rd32(E1000_RXPBS);
1583 pba = igb_rxpbs_adjust_82580(pba);
1584 break;
fa4dfae0 1585 case e1000_82576:
d249be54
AD		 1586 pba = rd32(E1000_RXPBS);
1587 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1588 break;
1589 case e1000_82575:
1590 default:
1591 pba = E1000_PBA_34K;
1592 break;
2d064c06 1593 }
9d5c8243 1594
2d064c06
AD		 1595 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1596 (mac->type < e1000_82576)) {
9d5c8243
AK		 1597 /* adjust PBA for jumbo frames */
1598 wr32(E1000_PBA, pba);
1599
1600 /* To maintain wire speed transmits, the Tx FIFO should be
1601 * large enough to accommodate two full transmit packets,
1602 * rounded up to the next 1KB and expressed in KB. Likewise,
1603 * the Rx FIFO should be large enough to accommodate at least
1604 * one full receive packet and is similarly rounded up and
1605 * expressed in KB. */
1606 pba = rd32(E1000_PBA);
1607 /* upper 16 bits has Tx packet buffer allocation size in KB */
1608 tx_space = pba >> 16;
1609 /* lower 16 bits has Rx packet buffer allocation size in KB */
1610 pba &= 0xffff;
1611 /* the tx fifo also stores 16 bytes of information about the tx
1612 * but don't include ethernet FCS because hardware appends it */
1613 min_tx_space = (adapter->max_frame_size +
85e8d004 1614 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1615 ETH_FCS_LEN) * 2;
1616 min_tx_space = ALIGN(min_tx_space, 1024);
1617 min_tx_space >>= 10;
1618 /* software strips receive CRC, so leave room for it */
1619 min_rx_space = adapter->max_frame_size;
1620 min_rx_space = ALIGN(min_rx_space, 1024);
1621 min_rx_space >>= 10;
1622
1623 /* If current Tx allocation is less than the min Tx FIFO size,
1624 * and the min Tx FIFO size is less than the current Rx FIFO
1625 * allocation, take space away from current Rx allocation */
1626 if (tx_space < min_tx_space &&
1627 ((min_tx_space - tx_space) < pba)) {
1628 pba = pba - (min_tx_space - tx_space);
1629
1630 /* if short on rx space, rx wins and must trump tx
1631 * adjustment */
1632 if (pba < min_rx_space)
1633 pba = min_rx_space;
1634 }
2d064c06 1635 wr32(E1000_PBA, pba);
9d5c8243 1636 }
9d5c8243
AK		 1637
1638 /* flow control settings */
1639 /* The high water mark must be low enough to fit one full frame
1640 * (or the size used for early receive) above it in the Rx FIFO.
1641 * Set it to the lower of:
1642 * - 90% of the Rx FIFO size, or
1643 * - the full Rx FIFO size minus one full frame */
1644 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1645 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1646
d405ea3e
AD		 1647 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1648 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1649 fc->pause_time = 0xFFFF;
1650 fc->send_xon = 1;
0cce119a 1651 fc->current_mode = fc->requested_mode;
9d5c8243 1652
4ae196df
AD		 1653 /* disable receive for all VFs and wait one second */
1654 if (adapter->vfs_allocated_count) {
1655 int i;
1656 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1657 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1658
1659 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1660 igb_ping_all_vfs(adapter);
4ae196df
AD		 1661
1662 /* disable transmits and receives */
1663 wr32(E1000_VFRE, 0);
1664 wr32(E1000_VFTE, 0);
1665 }
1666
9d5c8243 1667 /* Allow time for pending master requests to run */
330a6d6a 1668 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1669 wr32(E1000_WUC, 0);
1670
330a6d6a 1671 if (hw->mac.ops.init_hw(hw))
090b1795 1672 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1673
55cac248
AD		 1674 if (hw->mac.type == e1000_82580) {
1675 u32 reg = rd32(E1000_PCIEMISC);
1676 wr32(E1000_PCIEMISC,
1677 reg & ~E1000_PCIEMISC_LX_DECISION);
1678 }
88a268c1
NN		 1679 if (!netif_running(adapter->netdev))
1680 igb_power_down_link(adapter);
1681
9d5c8243
AK		 1682 igb_update_mng_vlan(adapter);
1683
1684 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1685 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1686
330a6d6a 1687 igb_get_phy_info(hw);
9d5c8243
AK		 1688}
1689
2e5c6922 1690static const struct net_device_ops igb_netdev_ops = {
559e9c49 1691 .ndo_open = igb_open,
2e5c6922 1692 .ndo_stop = igb_close,
00829823 1693 .ndo_start_xmit = igb_xmit_frame_adv,
12dcd86b 1694 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc
AD		 1695 .ndo_set_rx_mode = igb_set_rx_mode,
1696 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1697 .ndo_set_mac_address = igb_set_mac,
1698 .ndo_change_mtu = igb_change_mtu,
1699 .ndo_do_ioctl = igb_ioctl,
1700 .ndo_tx_timeout = igb_tx_timeout,
1701 .ndo_validate_addr = eth_validate_addr,
1702 .ndo_vlan_rx_register = igb_vlan_rx_register,
1703 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1704 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1705 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1706 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1707 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1708 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1709#ifdef CONFIG_NET_POLL_CONTROLLER
1710 .ndo_poll_controller = igb_netpoll,
1711#endif
1712};
1713
9d5c8243
AK		 1714/**
1715 * igb_probe - Device Initialization Routine
1716 * @pdev: PCI device information struct
1717 * @ent: entry in igb_pci_tbl
1718 *
1719 * Returns 0 on success, negative on failure
1720 *
1721 * igb_probe initializes an adapter identified by a pci_dev structure.
1722 * The OS initialization, configuring of the adapter private structure,
1723 * and a hardware reset occur.
1724 **/
1725static int __devinit igb_probe(struct pci_dev *pdev,
1726 const struct pci_device_id *ent)
1727{
1728 struct net_device *netdev;
1729 struct igb_adapter *adapter;
1730 struct e1000_hw *hw;
4337e993
AD		 1731 u16 eeprom_data = 0;
1732 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1733 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1734 unsigned long mmio_start, mmio_len;
2d6a5e95 1735 int err, pci_using_dac;
9d5c8243
AK		 1736 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1737 u32 part_num;
1738
bded64a7
AG		 1739 /* Catch broken hardware that put the wrong VF device ID in
1740 * the PCIe SR-IOV capability.
1741 */
1742 if (pdev->is_virtfn) {
1743 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1744 pci_name(pdev), pdev->vendor, pdev->device);
1745 return -EINVAL;
1746 }
1747
aed5dec3 1748 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1749 if (err)
1750 return err;
1751
1752 pci_using_dac = 0;
59d71989 1753 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1754 if (!err) {
59d71989 1755 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1756 if (!err)
1757 pci_using_dac = 1;
1758 } else {
59d71989 1759 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1760 if (err) {
59d71989 1761 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1762 if (err) {
1763 dev_err(&pdev->dev, "No usable DMA "
1764 "configuration, aborting\n");
1765 goto err_dma;
1766 }
1767 }
1768 }
1769
aed5dec3
AD		 1770 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1771 IORESOURCE_MEM),
1772 igb_driver_name);
9d5c8243
AK		 1773 if (err)
1774 goto err_pci_reg;
1775
19d5afd4 1776 pci_enable_pcie_error_reporting(pdev);
40a914fa 1777
9d5c8243 1778 pci_set_master(pdev);
c682fc23 1779 pci_save_state(pdev);
9d5c8243
AK		 1780
1781 err = -ENOMEM;
1bfaf07b
AD		 1782 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1783 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1784 if (!netdev)
1785 goto err_alloc_etherdev;
1786
1787 SET_NETDEV_DEV(netdev, &pdev->dev);
1788
1789 pci_set_drvdata(pdev, netdev);
1790 adapter = netdev_priv(netdev);
1791 adapter->netdev = netdev;
1792 adapter->pdev = pdev;
1793 hw = &adapter->hw;
1794 hw->back = adapter;
1795 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1796
1797 mmio_start = pci_resource_start(pdev, 0);
1798 mmio_len = pci_resource_len(pdev, 0);
1799
1800 err = -EIO;
28b0759c
AD		 1801 hw->hw_addr = ioremap(mmio_start, mmio_len);
1802 if (!hw->hw_addr)
9d5c8243
AK		 1803 goto err_ioremap;
1804
2e5c6922 1805 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1806 igb_set_ethtool_ops(netdev);
9d5c8243 1807 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1808
1809 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1810
1811 netdev->mem_start = mmio_start;
1812 netdev->mem_end = mmio_start + mmio_len;
1813
9d5c8243
AK		 1814 /* PCI config space info */
1815 hw->vendor_id = pdev->vendor;
1816 hw->device_id = pdev->device;
1817 hw->revision_id = pdev->revision;
1818 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1819 hw->subsystem_device_id = pdev->subsystem_device;
1820
9d5c8243
AK		 1821 /* Copy the default MAC, PHY and NVM function pointers */
1822 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1823 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1824 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1825 /* Initialize skew-specific constants */
1826 err = ei->get_invariants(hw);
1827 if (err)
450c87c8 1828 goto err_sw_init;
9d5c8243 1829
450c87c8 1830 /* setup the private structure */
9d5c8243
AK		 1831 err = igb_sw_init(adapter);
1832 if (err)
1833 goto err_sw_init;
1834
1835 igb_get_bus_info_pcie(hw);
1836
1837 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1838
1839 /* Copper options */
1840 if (hw->phy.media_type == e1000_media_type_copper) {
1841 hw->phy.mdix = AUTO_ALL_MODES;
1842 hw->phy.disable_polarity_correction = false;
1843 hw->phy.ms_type = e1000_ms_hw_default;
1844 }
1845
1846 if (igb_check_reset_block(hw))
1847 dev_info(&pdev->dev,
1848 "PHY reset is blocked due to SOL/IDER session.\n");
1849
1850 netdev->features = NETIF_F_SG |
7d8eb29e 1851 NETIF_F_IP_CSUM |
9d5c8243
AK		 1852 NETIF_F_HW_VLAN_TX |
1853 NETIF_F_HW_VLAN_RX |
1854 NETIF_F_HW_VLAN_FILTER;
1855
7d8eb29e 1856 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1857 netdev->features |= NETIF_F_TSO;
9d5c8243 1858 netdev->features |= NETIF_F_TSO6;
5c0999b7 1859 netdev->features |= NETIF_F_GRO;
d3352520 1860
48f29ffc
JK		 1861 netdev->vlan_features |= NETIF_F_TSO;
1862 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1863 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1864 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1865 netdev->vlan_features |= NETIF_F_SG;
1866
7b872a55 1867 if (pci_using_dac) {
9d5c8243 1868 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1869 netdev->vlan_features |= NETIF_F_HIGHDMA;
1870 }
9d5c8243 1871
5b043fb0 1872 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1873 netdev->features |= NETIF_F_SCTP_CSUM;
1874
330a6d6a 1875 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1876
1877 /* before reading the NVM, reset the controller to put the device in a
1878 * known good starting state */
1879 hw->mac.ops.reset_hw(hw);
1880
1881 /* make sure the NVM is good */
1882 if (igb_validate_nvm_checksum(hw) < 0) {
1883 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1884 err = -EIO;
1885 goto err_eeprom;
1886 }
1887
1888 /* copy the MAC address out of the NVM */
1889 if (hw->mac.ops.read_mac_addr(hw))
1890 dev_err(&pdev->dev, "NVM Read Error\n");
1891
1892 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1893 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1894
1895 if (!is_valid_ether_addr(netdev->perm_addr)) {
1896 dev_err(&pdev->dev, "Invalid MAC Address\n");
1897 err = -EIO;
1898 goto err_eeprom;
1899 }
1900
c061b18d 1901 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 1902 (unsigned long) adapter);
c061b18d 1903 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 1904 (unsigned long) adapter);
9d5c8243
AK		 1905
1906 INIT_WORK(&adapter->reset_task, igb_reset_task);
1907 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1908
450c87c8 1909 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1910 adapter->fc_autoneg = true;
1911 hw->mac.autoneg = true;
1912 hw->phy.autoneg_advertised = 0x2f;
1913
0cce119a
AD		 1914 hw->fc.requested_mode = e1000_fc_default;
1915 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1916
9d5c8243
AK		 1917 igb_validate_mdi_setting(hw);
1918
9d5c8243
AK		 1919 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1920 * enable the ACPI Magic Packet filter
1921 */
1922
a2cf8b6c 1923 if (hw->bus.func == 0)
312c75ae 1924 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1925 else if (hw->mac.type == e1000_82580)
1926 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1927 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1928 &eeprom_data);
a2cf8b6c
AD		 1929 else if (hw->bus.func == 1)
1930 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1931
1932 if (eeprom_data & eeprom_apme_mask)
1933 adapter->eeprom_wol |= E1000_WUFC_MAG;
1934
1935 /* now that we have the eeprom settings, apply the special cases where
1936 * the eeprom may be wrong or the board simply won't support wake on
1937 * lan on a particular port */
1938 switch (pdev->device) {
1939 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1940 adapter->eeprom_wol = 0;
1941 break;
1942 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1943 case E1000_DEV_ID_82576_FIBER:
1944 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1945 /* Wake events only supported on port A for dual fiber
1946 * regardless of eeprom setting */
1947 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1948 adapter->eeprom_wol = 0;
1949 break;
c8ea5ea9 1950 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1951 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1952 /* if quad port adapter, disable WoL on all but port A */
1953 if (global_quad_port_a != 0)
1954 adapter->eeprom_wol = 0;
1955 else
1956 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1957 /* Reset for multiple quad port adapters */
1958 if (++global_quad_port_a == 4)
1959 global_quad_port_a = 0;
1960 break;
9d5c8243
AK		 1961 }
1962
1963 /* initialize the wol settings based on the eeprom settings */
1964 adapter->wol = adapter->eeprom_wol;
e1b86d84 1965 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1966
1967 /* reset the hardware with the new settings */
1968 igb_reset(adapter);
1969
1970 /* let the f/w know that the h/w is now under the control of the
1971 * driver. */
1972 igb_get_hw_control(adapter);
1973
9d5c8243
AK		 1974 strcpy(netdev->name, "eth%d");
1975 err = register_netdev(netdev);
1976 if (err)
1977 goto err_register;
1978
b168dfc5
JB		 1979 /* carrier off reporting is important to ethtool even BEFORE open */
1980 netif_carrier_off(netdev);
1981
421e02f0 1982#ifdef CONFIG_IGB_DCA
bbd98fe4 1983 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1984 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1985 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1986 igb_setup_dca(adapter);
1987 }
fe4506b6 1988
38c845c7 1989#endif
9d5c8243
AK		 1990 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1991 /* print bus type/speed/width info */
7c510e4b 1992 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1993 netdev->name,
559e9c49 1994 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1995 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1996 "unknown"),
59c3de89
AD		 1997 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1998 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1999 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2000 "unknown"),
7c510e4b 2001 netdev->dev_addr);
9d5c8243
AK		 2002
2003 igb_read_part_num(hw, &part_num);
2004 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
2005 (part_num >> 8), (part_num & 0xff));
2006
2007 dev_info(&pdev->dev,
2008 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2009 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2010 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 2011 adapter->num_rx_queues, adapter->num_tx_queues);
2012
9d5c8243
AK		 2013 return 0;
2014
2015err_register:
2016 igb_release_hw_control(adapter);
2017err_eeprom:
2018 if (!igb_check_reset_block(hw))
f5f4cf08 2019 igb_reset_phy(hw);
9d5c8243
AK		 2020
2021 if (hw->flash_address)
2022 iounmap(hw->flash_address);
9d5c8243 2023err_sw_init:
047e0030 2024 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2025 iounmap(hw->hw_addr);
2026err_ioremap:
2027 free_netdev(netdev);
2028err_alloc_etherdev:
559e9c49
AD		 2029 pci_release_selected_regions(pdev,
2030 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2031err_pci_reg:
2032err_dma:
2033 pci_disable_device(pdev);
2034 return err;
2035}
2036
2037/**
2038 * igb_remove - Device Removal Routine
2039 * @pdev: PCI device information struct
2040 *
2041 * igb_remove is called by the PCI subsystem to alert the driver
2042 * that it should release a PCI device. The could be caused by a
2043 * Hot-Plug event, or because the driver is going to be removed from
2044 * memory.
2045 **/
2046static void __devexit igb_remove(struct pci_dev *pdev)
2047{
2048 struct net_device *netdev = pci_get_drvdata(pdev);
2049 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2050 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2051
2052 /* flush_scheduled work may reschedule our watchdog task, so
2053 * explicitly disable watchdog tasks from being rescheduled */
2054 set_bit(__IGB_DOWN, &adapter->state);
2055 del_timer_sync(&adapter->watchdog_timer);
2056 del_timer_sync(&adapter->phy_info_timer);
2057
2058 flush_scheduled_work();
2059
421e02f0 2060#ifdef CONFIG_IGB_DCA
7dfc16fa 2061 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2062 dev_info(&pdev->dev, "DCA disabled\n");
2063 dca_remove_requester(&pdev->dev);
7dfc16fa 2064 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2065 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2066 }
2067#endif
2068
9d5c8243
AK		 2069 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2070 * would have already happened in close and is redundant. */
2071 igb_release_hw_control(adapter);
2072
2073 unregister_netdev(netdev);
2074
047e0030 2075 igb_clear_interrupt_scheme(adapter);
9d5c8243 2076
37680117
AD		 2077#ifdef CONFIG_PCI_IOV
2078 /* reclaim resources allocated to VFs */
2079 if (adapter->vf_data) {
2080 /* disable iov and allow time for transactions to clear */
2081 pci_disable_sriov(pdev);
2082 msleep(500);
2083
2084 kfree(adapter->vf_data);
2085 adapter->vf_data = NULL;
2086 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2087 msleep(100);
2088 dev_info(&pdev->dev, "IOV Disabled\n");
2089 }
2090#endif
559e9c49 2091
28b0759c
AD		 2092 iounmap(hw->hw_addr);
2093 if (hw->flash_address)
2094 iounmap(hw->flash_address);
559e9c49
AD		 2095 pci_release_selected_regions(pdev,
2096 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2097
2098 free_netdev(netdev);
2099
19d5afd4 2100 pci_disable_pcie_error_reporting(pdev);
40a914fa 2101
9d5c8243
AK		 2102 pci_disable_device(pdev);
2103}
2104
a6b623e0
AD		 2105/**
2106 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2107 * @adapter: board private structure to initialize
2108 *
2109 * This function initializes the vf specific data storage and then attempts to
2110 * allocate the VFs. The reason for ordering it this way is because it is much
2111 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2112 * the memory for the VFs.
2113 **/
2114static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2115{
2116#ifdef CONFIG_PCI_IOV
2117 struct pci_dev *pdev = adapter->pdev;
2118
a6b623e0
AD		 2119 if (adapter->vfs_allocated_count) {
2120 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2121 sizeof(struct vf_data_storage),
2122 GFP_KERNEL);
2123 /* if allocation failed then we do not support SR-IOV */
2124 if (!adapter->vf_data) {
2125 adapter->vfs_allocated_count = 0;
2126 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2127 "Data Storage\n");
2128 }
2129 }
2130
2131 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2132 kfree(adapter->vf_data);
2133 adapter->vf_data = NULL;
2134#endif /* CONFIG_PCI_IOV */
2135 adapter->vfs_allocated_count = 0;
2136#ifdef CONFIG_PCI_IOV
2137 } else {
2138 unsigned char mac_addr[ETH_ALEN];
2139 int i;
2140 dev_info(&pdev->dev, "%d vfs allocated\n",
2141 adapter->vfs_allocated_count);
2142 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2143 random_ether_addr(mac_addr);
2144 igb_set_vf_mac(adapter, i, mac_addr);
2145 }
2146 }
2147#endif /* CONFIG_PCI_IOV */
2148}
2149
115f459a
AD		 2150
2151/**
2152 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2153 * @adapter: board private structure to initialize
2154 *
2155 * igb_init_hw_timer initializes the function pointer and values for the hw
2156 * timer found in hardware.
2157 **/
2158static void igb_init_hw_timer(struct igb_adapter *adapter)
2159{
2160 struct e1000_hw *hw = &adapter->hw;
2161
2162 switch (hw->mac.type) {
d2ba2ed8 2163 case e1000_i350:
55cac248
AD		 2164 case e1000_82580:
2165 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2166 adapter->cycles.read = igb_read_clock;
2167 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2168 adapter->cycles.mult = 1;
2169 /*
2170 * The 82580 timesync updates the system timer every 8ns by 8ns
2171 * and the value cannot be shifted. Instead we need to shift
2172 * the registers to generate a 64bit timer value. As a result
2173 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2174 * 24 in order to generate a larger value for synchronization.
2175 */
2176 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2177 /* disable system timer temporarily by setting bit 31 */
2178 wr32(E1000_TSAUXC, 0x80000000);
2179 wrfl();
2180
2181 /* Set registers so that rollover occurs soon to test this. */
2182 wr32(E1000_SYSTIMR, 0x00000000);
2183 wr32(E1000_SYSTIML, 0x80000000);
2184 wr32(E1000_SYSTIMH, 0x000000FF);
2185 wrfl();
2186
2187 /* enable system timer by clearing bit 31 */
2188 wr32(E1000_TSAUXC, 0x0);
2189 wrfl();
2190
2191 timecounter_init(&adapter->clock,
2192 &adapter->cycles,
2193 ktime_to_ns(ktime_get_real()));
2194 /*
2195 * Synchronize our NIC clock against system wall clock. NIC
2196 * time stamp reading requires ~3us per sample, each sample
2197 * was pretty stable even under load => only require 10
2198 * samples for each offset comparison.
2199 */
2200 memset(&adapter->compare, 0, sizeof(adapter->compare));
2201 adapter->compare.source = &adapter->clock;
2202 adapter->compare.target = ktime_get_real;
2203 adapter->compare.num_samples = 10;
2204 timecompare_update(&adapter->compare, 0);
2205 break;
115f459a
AD		 2206 case e1000_82576:
2207 /*
2208 * Initialize hardware timer: we keep it running just in case
2209 * that some program needs it later on.
2210 */
2211 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2212 adapter->cycles.read = igb_read_clock;
2213 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2214 adapter->cycles.mult = 1;
2215 /**
2216 * Scale the NIC clock cycle by a large factor so that
2217 * relatively small clock corrections can be added or
2218 * substracted at each clock tick. The drawbacks of a large
2219 * factor are a) that the clock register overflows more quickly
2220 * (not such a big deal) and b) that the increment per tick has
2221 * to fit into 24 bits. As a result we need to use a shift of
2222 * 19 so we can fit a value of 16 into the TIMINCA register.
2223 */
2224 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2225 wr32(E1000_TIMINCA,
2226 (1 << E1000_TIMINCA_16NS_SHIFT) |
2227 (16 << IGB_82576_TSYNC_SHIFT));
2228
2229 /* Set registers so that rollover occurs soon to test this. */
2230 wr32(E1000_SYSTIML, 0x00000000);
2231 wr32(E1000_SYSTIMH, 0xFF800000);
2232 wrfl();
2233
2234 timecounter_init(&adapter->clock,
2235 &adapter->cycles,
2236 ktime_to_ns(ktime_get_real()));
2237 /*
2238 * Synchronize our NIC clock against system wall clock. NIC
2239 * time stamp reading requires ~3us per sample, each sample
2240 * was pretty stable even under load => only require 10
2241 * samples for each offset comparison.
2242 */
2243 memset(&adapter->compare, 0, sizeof(adapter->compare));
2244 adapter->compare.source = &adapter->clock;
2245 adapter->compare.target = ktime_get_real;
2246 adapter->compare.num_samples = 10;
2247 timecompare_update(&adapter->compare, 0);
2248 break;
2249 case e1000_82575:
2250 /* 82575 does not support timesync */
2251 default:
2252 break;
2253 }
2254
2255}
2256
9d5c8243
AK		 2257/**
2258 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2259 * @adapter: board private structure to initialize
2260 *
2261 * igb_sw_init initializes the Adapter private data structure.
2262 * Fields are initialized based on PCI device information and
2263 * OS network device settings (MTU size).
2264 **/
2265static int __devinit igb_sw_init(struct igb_adapter *adapter)
2266{
2267 struct e1000_hw *hw = &adapter->hw;
2268 struct net_device *netdev = adapter->netdev;
2269 struct pci_dev *pdev = adapter->pdev;
2270
2271 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2272
68fd9910
AD		 2273 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2274 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2275 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2276 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2277
9d5c8243
AK		 2278 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2279 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2280
12dcd86b 2281 spin_lock_init(&adapter->stats64_lock);
a6b623e0
AD		 2282#ifdef CONFIG_PCI_IOV
2283 if (hw->mac.type == e1000_82576)
c0f2276f 2284 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2285
2286#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2287 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2288
2289 /*
2290 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2291 * then we should combine the queues into a queue pair in order to
2292 * conserve interrupts due to limited supply
2293 */
2294 if ((adapter->rss_queues > 4) ||
2295 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2296 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2297
a6b623e0 2298 /* This call may decrease the number of queues */
047e0030 2299 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2300 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2301 return -ENOMEM;
2302 }
2303
115f459a 2304 igb_init_hw_timer(adapter);
a6b623e0
AD		 2305 igb_probe_vfs(adapter);
2306
9d5c8243
AK		 2307 /* Explicitly disable IRQ since the NIC can be in any state. */
2308 igb_irq_disable(adapter);
2309
2310 set_bit(__IGB_DOWN, &adapter->state);
2311 return 0;
2312}
2313
2314/**
2315 * igb_open - Called when a network interface is made active
2316 * @netdev: network interface device structure
2317 *
2318 * Returns 0 on success, negative value on failure
2319 *
2320 * The open entry point is called when a network interface is made
2321 * active by the system (IFF_UP). At this point all resources needed
2322 * for transmit and receive operations are allocated, the interrupt
2323 * handler is registered with the OS, the watchdog timer is started,
2324 * and the stack is notified that the interface is ready.
2325 **/
2326static int igb_open(struct net_device *netdev)
2327{
2328 struct igb_adapter *adapter = netdev_priv(netdev);
2329 struct e1000_hw *hw = &adapter->hw;
2330 int err;
2331 int i;
2332
2333 /* disallow open during test */
2334 if (test_bit(__IGB_TESTING, &adapter->state))
2335 return -EBUSY;
2336
b168dfc5
JB		 2337 netif_carrier_off(netdev);
2338
9d5c8243
AK		 2339 /* allocate transmit descriptors */
2340 err = igb_setup_all_tx_resources(adapter);
2341 if (err)
2342 goto err_setup_tx;
2343
2344 /* allocate receive descriptors */
2345 err = igb_setup_all_rx_resources(adapter);
2346 if (err)
2347 goto err_setup_rx;
2348
88a268c1 2349 igb_power_up_link(adapter);
9d5c8243 2350
9d5c8243
AK		 2351 /* before we allocate an interrupt, we must be ready to handle it.
2352 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2353 * as soon as we call pci_request_irq, so we have to setup our
2354 * clean_rx handler before we do so. */
2355 igb_configure(adapter);
2356
2357 err = igb_request_irq(adapter);
2358 if (err)
2359 goto err_req_irq;
2360
2361 /* From here on the code is the same as igb_up() */
2362 clear_bit(__IGB_DOWN, &adapter->state);
2363
047e0030
AD		 2364 for (i = 0; i < adapter->num_q_vectors; i++) {
2365 struct igb_q_vector *q_vector = adapter->q_vector[i];
2366 napi_enable(&q_vector->napi);
2367 }
9d5c8243
AK		 2368
2369 /* Clear any pending interrupts. */
2370 rd32(E1000_ICR);
844290e5
PW		 2371
2372 igb_irq_enable(adapter);
2373
d4960307
AD		 2374 /* notify VFs that reset has been completed */
2375 if (adapter->vfs_allocated_count) {
2376 u32 reg_data = rd32(E1000_CTRL_EXT);
2377 reg_data |= E1000_CTRL_EXT_PFRSTD;
2378 wr32(E1000_CTRL_EXT, reg_data);
2379 }
2380
d55b53ff
JK		 2381 netif_tx_start_all_queues(netdev);
2382
25568a53
AD		 2383 /* start the watchdog. */
2384 hw->mac.get_link_status = 1;
2385 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2386
2387 return 0;
2388
2389err_req_irq:
2390 igb_release_hw_control(adapter);
88a268c1 2391 igb_power_down_link(adapter);
9d5c8243
AK		 2392 igb_free_all_rx_resources(adapter);
2393err_setup_rx:
2394 igb_free_all_tx_resources(adapter);
2395err_setup_tx:
2396 igb_reset(adapter);
2397
2398 return err;
2399}
2400
2401/**
2402 * igb_close - Disables a network interface
2403 * @netdev: network interface device structure
2404 *
2405 * Returns 0, this is not allowed to fail
2406 *
2407 * The close entry point is called when an interface is de-activated
2408 * by the OS. The hardware is still under the driver's control, but
2409 * needs to be disabled. A global MAC reset is issued to stop the
2410 * hardware, and all transmit and receive resources are freed.
2411 **/
2412static int igb_close(struct net_device *netdev)
2413{
2414 struct igb_adapter *adapter = netdev_priv(netdev);
2415
2416 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2417 igb_down(adapter);
2418
2419 igb_free_irq(adapter);
2420
2421 igb_free_all_tx_resources(adapter);
2422 igb_free_all_rx_resources(adapter);
2423
9d5c8243
AK		 2424 return 0;
2425}
2426
2427/**
2428 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2429 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2430 *
2431 * Return 0 on success, negative on failure
2432 **/
80785298 2433int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2434{
59d71989 2435 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2436 int size;
2437
2438 size = sizeof(struct igb_buffer) * tx_ring->count;
2439 tx_ring->buffer_info = vmalloc(size);
2440 if (!tx_ring->buffer_info)
2441 goto err;
2442 memset(tx_ring->buffer_info, 0, size);
2443
2444 /* round up to nearest 4K */
85e8d004 2445 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2446 tx_ring->size = ALIGN(tx_ring->size, 4096);
2447
59d71989
AD		 2448 tx_ring->desc = dma_alloc_coherent(dev,
2449 tx_ring->size,
2450 &tx_ring->dma,
2451 GFP_KERNEL);
9d5c8243
AK		 2452
2453 if (!tx_ring->desc)
2454 goto err;
2455
9d5c8243
AK		 2456 tx_ring->next_to_use = 0;
2457 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2458 return 0;
2459
2460err:
2461 vfree(tx_ring->buffer_info);
59d71989 2462 dev_err(dev,
9d5c8243
AK		 2463 "Unable to allocate memory for the transmit descriptor ring\n");
2464 return -ENOMEM;
2465}
2466
2467/**
2468 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2469 * (Descriptors) for all queues
2470 * @adapter: board private structure
2471 *
2472 * Return 0 on success, negative on failure
2473 **/
2474static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2475{
439705e1 2476 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2477 int i, err = 0;
2478
2479 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2480 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2481 if (err) {
439705e1 2482 dev_err(&pdev->dev,
9d5c8243
AK		 2483 "Allocation for Tx Queue %u failed\n", i);
2484 for (i--; i >= 0; i--)
3025a446 2485 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2486 break;
2487 }
2488 }
2489
a99955fc 2490 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2491 int r_idx = i % adapter->num_tx_queues;
3025a446 2492 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2493 }
9d5c8243
AK		 2494 return err;
2495}
2496
2497/**
85b430b4
AD		 2498 * igb_setup_tctl - configure the transmit control registers
2499 * @adapter: Board private structure
9d5c8243 2500 **/
d7ee5b3a 2501void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2502{
9d5c8243
AK		 2503 struct e1000_hw *hw = &adapter->hw;
2504 u32 tctl;
9d5c8243 2505
85b430b4
AD		 2506 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2507 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2508
2509 /* Program the Transmit Control Register */
9d5c8243
AK		 2510 tctl = rd32(E1000_TCTL);
2511 tctl &= ~E1000_TCTL_CT;
2512 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2513 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2514
2515 igb_config_collision_dist(hw);
2516
9d5c8243
AK		 2517 /* Enable transmits */
2518 tctl |= E1000_TCTL_EN;
2519
2520 wr32(E1000_TCTL, tctl);
2521}
2522
85b430b4
AD		 2523/**
2524 * igb_configure_tx_ring - Configure transmit ring after Reset
2525 * @adapter: board private structure
2526 * @ring: tx ring to configure
2527 *
2528 * Configure a transmit ring after a reset.
2529 **/
d7ee5b3a
AD		 2530void igb_configure_tx_ring(struct igb_adapter *adapter,
2531 struct igb_ring *ring)
85b430b4
AD		 2532{
2533 struct e1000_hw *hw = &adapter->hw;
2534 u32 txdctl;
2535 u64 tdba = ring->dma;
2536 int reg_idx = ring->reg_idx;
2537
2538 /* disable the queue */
2539 txdctl = rd32(E1000_TXDCTL(reg_idx));
2540 wr32(E1000_TXDCTL(reg_idx),
2541 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2542 wrfl();
2543 mdelay(10);
2544
2545 wr32(E1000_TDLEN(reg_idx),
2546 ring->count * sizeof(union e1000_adv_tx_desc));
2547 wr32(E1000_TDBAL(reg_idx),
2548 tdba & 0x00000000ffffffffULL);
2549 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2550
fce99e34
AD		 2551 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2552 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2553 writel(0, ring->head);
2554 writel(0, ring->tail);
85b430b4
AD		 2555
2556 txdctl |= IGB_TX_PTHRESH;
2557 txdctl |= IGB_TX_HTHRESH << 8;
2558 txdctl |= IGB_TX_WTHRESH << 16;
2559
2560 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2561 wr32(E1000_TXDCTL(reg_idx), txdctl);
2562}
2563
2564/**
2565 * igb_configure_tx - Configure transmit Unit after Reset
2566 * @adapter: board private structure
2567 *
2568 * Configure the Tx unit of the MAC after a reset.
2569 **/
2570static void igb_configure_tx(struct igb_adapter *adapter)
2571{
2572 int i;
2573
2574 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2575 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2576}
2577
9d5c8243
AK		 2578/**
2579 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2580 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2581 *
2582 * Returns 0 on success, negative on failure
2583 **/
80785298 2584int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2585{
59d71989 2586 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2587 int size, desc_len;
2588
2589 size = sizeof(struct igb_buffer) * rx_ring->count;
2590 rx_ring->buffer_info = vmalloc(size);
2591 if (!rx_ring->buffer_info)
2592 goto err;
2593 memset(rx_ring->buffer_info, 0, size);
2594
2595 desc_len = sizeof(union e1000_adv_rx_desc);
2596
2597 /* Round up to nearest 4K */
2598 rx_ring->size = rx_ring->count * desc_len;
2599 rx_ring->size = ALIGN(rx_ring->size, 4096);
2600
59d71989
AD		 2601 rx_ring->desc = dma_alloc_coherent(dev,
2602 rx_ring->size,
2603 &rx_ring->dma,
2604 GFP_KERNEL);
9d5c8243
AK		 2605
2606 if (!rx_ring->desc)
2607 goto err;
2608
2609 rx_ring->next_to_clean = 0;
2610 rx_ring->next_to_use = 0;
9d5c8243 2611
9d5c8243
AK		 2612 return 0;
2613
2614err:
2615 vfree(rx_ring->buffer_info);
439705e1 2616 rx_ring->buffer_info = NULL;
59d71989
AD		 2617 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2618 " ring\n");
9d5c8243
AK		 2619 return -ENOMEM;
2620}
2621
2622/**
2623 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2624 * (Descriptors) for all queues
2625 * @adapter: board private structure
2626 *
2627 * Return 0 on success, negative on failure
2628 **/
2629static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2630{
439705e1 2631 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2632 int i, err = 0;
2633
2634 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2635 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2636 if (err) {
439705e1 2637 dev_err(&pdev->dev,
9d5c8243
AK		 2638 "Allocation for Rx Queue %u failed\n", i);
2639 for (i--; i >= 0; i--)
3025a446 2640 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2641 break;
2642 }
2643 }
2644
2645 return err;
2646}
2647
06cf2666
AD		 2648/**
2649 * igb_setup_mrqc - configure the multiple receive queue control registers
2650 * @adapter: Board private structure
2651 **/
2652static void igb_setup_mrqc(struct igb_adapter *adapter)
2653{
2654 struct e1000_hw *hw = &adapter->hw;
2655 u32 mrqc, rxcsum;
2656 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2657 union e1000_reta {
2658 u32 dword;
2659 u8 bytes[4];
2660 } reta;
2661 static const u8 rsshash[40] = {
2662 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2663 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2664 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2665 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2666
2667 /* Fill out hash function seeds */
2668 for (j = 0; j < 10; j++) {
2669 u32 rsskey = rsshash[(j * 4)];
2670 rsskey |= rsshash[(j * 4) + 1] << 8;
2671 rsskey |= rsshash[(j * 4) + 2] << 16;
2672 rsskey |= rsshash[(j * 4) + 3] << 24;
2673 array_wr32(E1000_RSSRK(0), j, rsskey);
2674 }
2675
a99955fc 2676 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2677
2678 if (adapter->vfs_allocated_count) {
2679 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2680 switch (hw->mac.type) {
d2ba2ed8 2681 case e1000_i350:
55cac248
AD		 2682 case e1000_82580:
2683 num_rx_queues = 1;
2684 shift = 0;
2685 break;
06cf2666
AD		 2686 case e1000_82576:
2687 shift = 3;
2688 num_rx_queues = 2;
2689 break;
2690 case e1000_82575:
2691 shift = 2;
2692 shift2 = 6;
2693 default:
2694 break;
2695 }
2696 } else {
2697 if (hw->mac.type == e1000_82575)
2698 shift = 6;
2699 }
2700
2701 for (j = 0; j < (32 * 4); j++) {
2702 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2703 if (shift2)
2704 reta.bytes[j & 3] |= num_rx_queues << shift2;
2705 if ((j & 3) == 3)
2706 wr32(E1000_RETA(j >> 2), reta.dword);
2707 }
2708
2709 /*
2710 * Disable raw packet checksumming so that RSS hash is placed in
2711 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2712 * offloads as they are enabled by default
2713 */
2714 rxcsum = rd32(E1000_RXCSUM);
2715 rxcsum |= E1000_RXCSUM_PCSD;
2716
2717 if (adapter->hw.mac.type >= e1000_82576)
2718 /* Enable Receive Checksum Offload for SCTP */
2719 rxcsum |= E1000_RXCSUM_CRCOFL;
2720
2721 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2722 wr32(E1000_RXCSUM, rxcsum);
2723
2724 /* If VMDq is enabled then we set the appropriate mode for that, else
2725 * we default to RSS so that an RSS hash is calculated per packet even
2726 * if we are only using one queue */
2727 if (adapter->vfs_allocated_count) {
2728 if (hw->mac.type > e1000_82575) {
2729 /* Set the default pool for the PF's first queue */
2730 u32 vtctl = rd32(E1000_VT_CTL);
2731 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2732 E1000_VT_CTL_DISABLE_DEF_POOL);
2733 vtctl |= adapter->vfs_allocated_count <<
2734 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2735 wr32(E1000_VT_CTL, vtctl);
2736 }
a99955fc 2737 if (adapter->rss_queues > 1)
06cf2666
AD		 2738 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2739 else
2740 mrqc = E1000_MRQC_ENABLE_VMDQ;
2741 } else {
2742 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2743 }
2744 igb_vmm_control(adapter);
2745
4478a9cd
AD		 2746 /*
2747 * Generate RSS hash based on TCP port numbers and/or
2748 * IPv4/v6 src and dst addresses since UDP cannot be
2749 * hashed reliably due to IP fragmentation
2750 */
2751 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2752 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2753 E1000_MRQC_RSS_FIELD_IPV6 |
2754 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2755 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2756
2757 wr32(E1000_MRQC, mrqc);
2758}
2759
9d5c8243
AK		 2760/**
2761 * igb_setup_rctl - configure the receive control registers
2762 * @adapter: Board private structure
2763 **/
d7ee5b3a 2764void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2765{
2766 struct e1000_hw *hw = &adapter->hw;
2767 u32 rctl;
9d5c8243
AK		 2768
2769 rctl = rd32(E1000_RCTL);
2770
2771 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2772 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2773
69d728ba 2774 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2775 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2776
87cb7e8c
AK		 2777 /*
2778 * enable stripping of CRC. It's unlikely this will break BMC
2779 * redirection as it did with e1000. Newer features require
2780 * that the HW strips the CRC.
73cd78f1 2781 */
87cb7e8c 2782 rctl |= E1000_RCTL_SECRC;
9d5c8243 2783
559e9c49 2784 /* disable store bad packets and clear size bits. */
ec54d7d6 2785 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2786
6ec43fe6
AD		 2787 /* enable LPE to prevent packets larger than max_frame_size */
2788 rctl |= E1000_RCTL_LPE;
9d5c8243 2789
952f72a8
AD		 2790 /* disable queue 0 to prevent tail write w/o re-config */
2791 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2792
e1739522
AD		 2793 /* Attention!!! For SR-IOV PF driver operations you must enable
2794 * queue drop for all VF and PF queues to prevent head of line blocking
2795 * if an un-trusted VF does not provide descriptors to hardware.
2796 */
2797 if (adapter->vfs_allocated_count) {
e1739522
AD		 2798 /* set all queue drop enable bits */
2799 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2800 }
2801
9d5c8243
AK		 2802 wr32(E1000_RCTL, rctl);
2803}
2804
7d5753f0
AD		 2805static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2806 int vfn)
2807{
2808 struct e1000_hw *hw = &adapter->hw;
2809 u32 vmolr;
2810
2811 /* if it isn't the PF check to see if VFs are enabled and
2812 * increase the size to support vlan tags */
2813 if (vfn < adapter->vfs_allocated_count &&
2814 adapter->vf_data[vfn].vlans_enabled)
2815 size += VLAN_TAG_SIZE;
2816
2817 vmolr = rd32(E1000_VMOLR(vfn));
2818 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2819 vmolr |= size | E1000_VMOLR_LPE;
2820 wr32(E1000_VMOLR(vfn), vmolr);
2821
2822 return 0;
2823}
2824
e1739522
AD		 2825/**
2826 * igb_rlpml_set - set maximum receive packet size
2827 * @adapter: board private structure
2828 *
2829 * Configure maximum receivable packet size.
2830 **/
2831static void igb_rlpml_set(struct igb_adapter *adapter)
2832{
2833 u32 max_frame_size = adapter->max_frame_size;
2834 struct e1000_hw *hw = &adapter->hw;
2835 u16 pf_id = adapter->vfs_allocated_count;
2836
2837 if (adapter->vlgrp)
2838 max_frame_size += VLAN_TAG_SIZE;
2839
2840 /* if vfs are enabled we set RLPML to the largest possible request
2841 * size and set the VMOLR RLPML to the size we need */
2842 if (pf_id) {
2843 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2844 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2845 }
2846
2847 wr32(E1000_RLPML, max_frame_size);
2848}
2849
8151d294
WM		 2850static inline void igb_set_vmolr(struct igb_adapter *adapter,
2851 int vfn, bool aupe)
7d5753f0
AD		 2852{
2853 struct e1000_hw *hw = &adapter->hw;
2854 u32 vmolr;
2855
2856 /*
2857 * This register exists only on 82576 and newer so if we are older then
2858 * we should exit and do nothing
2859 */
2860 if (hw->mac.type < e1000_82576)
2861 return;
2862
2863 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2864 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2865 if (aupe)
2866 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2867 else
2868 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2869
2870 /* clear all bits that might not be set */
2871 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2872
a99955fc 2873 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2874 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2875 /*
2876 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2877 * multicast packets
2878 */
2879 if (vfn <= adapter->vfs_allocated_count)
2880 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2881
2882 wr32(E1000_VMOLR(vfn), vmolr);
2883}
2884
85b430b4
AD		 2885/**
2886 * igb_configure_rx_ring - Configure a receive ring after Reset
2887 * @adapter: board private structure
2888 * @ring: receive ring to be configured
2889 *
2890 * Configure the Rx unit of the MAC after a reset.
2891 **/
d7ee5b3a
AD		 2892void igb_configure_rx_ring(struct igb_adapter *adapter,
2893 struct igb_ring *ring)
85b430b4
AD		 2894{
2895 struct e1000_hw *hw = &adapter->hw;
2896 u64 rdba = ring->dma;
2897 int reg_idx = ring->reg_idx;
952f72a8 2898 u32 srrctl, rxdctl;
85b430b4
AD		 2899
2900 /* disable the queue */
2901 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2902 wr32(E1000_RXDCTL(reg_idx),
2903 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2904
2905 /* Set DMA base address registers */
2906 wr32(E1000_RDBAL(reg_idx),
2907 rdba & 0x00000000ffffffffULL);
2908 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2909 wr32(E1000_RDLEN(reg_idx),
2910 ring->count * sizeof(union e1000_adv_rx_desc));
2911
2912 /* initialize head and tail */
fce99e34
AD		 2913 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2914 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2915 writel(0, ring->head);
2916 writel(0, ring->tail);
85b430b4 2917
952f72a8 2918 /* set descriptor configuration */
4c844851
AD		 2919 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2920 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2921 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2922#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2923 srrctl |= IGB_RXBUFFER_16384 >>
2924 E1000_SRRCTL_BSIZEPKT_SHIFT;
2925#else
2926 srrctl |= (PAGE_SIZE / 2) >>
2927 E1000_SRRCTL_BSIZEPKT_SHIFT;
2928#endif
2929 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2930 } else {
4c844851 2931 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2932 E1000_SRRCTL_BSIZEPKT_SHIFT;
2933 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2934 }
757b77e2
NN		 2935 if (hw->mac.type == e1000_82580)
2936 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2937 /* Only set Drop Enable if we are supporting multiple queues */
2938 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2939 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2940
2941 wr32(E1000_SRRCTL(reg_idx), srrctl);
2942
7d5753f0 2943 /* set filtering for VMDQ pools */
8151d294 2944 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2945
85b430b4
AD		 2946 /* enable receive descriptor fetching */
2947 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2948 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2949 rxdctl &= 0xFFF00000;
2950 rxdctl |= IGB_RX_PTHRESH;
2951 rxdctl |= IGB_RX_HTHRESH << 8;
2952 rxdctl |= IGB_RX_WTHRESH << 16;
2953 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2954}
2955
9d5c8243
AK		 2956/**
2957 * igb_configure_rx - Configure receive Unit after Reset
2958 * @adapter: board private structure
2959 *
2960 * Configure the Rx unit of the MAC after a reset.
2961 **/
2962static void igb_configure_rx(struct igb_adapter *adapter)
2963{
9107584e 2964 int i;
9d5c8243 2965
68d480c4
AD		 2966 /* set UTA to appropriate mode */
2967 igb_set_uta(adapter);
2968
26ad9178
AD		 2969 /* set the correct pool for the PF default MAC address in entry 0 */
2970 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2971 adapter->vfs_allocated_count);
2972
06cf2666
AD		 2973 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2974 * the Base and Length of the Rx Descriptor Ring */
2975 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2976 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2977}
2978
2979/**
2980 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2981 * @tx_ring: Tx descriptor ring for a specific queue
2982 *
2983 * Free all transmit software resources
2984 **/
68fd9910 2985void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2986{
3b644cf6 2987 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2988
2989 vfree(tx_ring->buffer_info);
2990 tx_ring->buffer_info = NULL;
2991
439705e1
AD		 2992 /* if not set, then don't free */
2993 if (!tx_ring->desc)
2994 return;
2995
59d71989
AD		 2996 dma_free_coherent(tx_ring->dev, tx_ring->size,
2997 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2998
2999 tx_ring->desc = NULL;
3000}
3001
3002/**
3003 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3004 * @adapter: board private structure
3005 *
3006 * Free all transmit software resources
3007 **/
3008static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3009{
3010 int i;
3011
3012 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3013 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3014}
3015
b1a436c3
AD		 3016void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3017 struct igb_buffer *buffer_info)
9d5c8243 3018{
6366ad33
AD		 3019 if (buffer_info->dma) {
3020 if (buffer_info->mapped_as_page)
59d71989 3021 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3022 buffer_info->dma,
3023 buffer_info->length,
59d71989 3024 DMA_TO_DEVICE);
6366ad33 3025 else
59d71989 3026 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3027 buffer_info->dma,
3028 buffer_info->length,
59d71989 3029 DMA_TO_DEVICE);
6366ad33
AD		 3030 buffer_info->dma = 0;
3031 }
9d5c8243
AK		 3032 if (buffer_info->skb) {
3033 dev_kfree_skb_any(buffer_info->skb);
3034 buffer_info->skb = NULL;
3035 }
3036 buffer_info->time_stamp = 0;
6366ad33
AD		 3037 buffer_info->length = 0;
3038 buffer_info->next_to_watch = 0;
3039 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3040}
3041
3042/**
3043 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3044 * @tx_ring: ring to be cleaned
3045 **/
3b644cf6 3046static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3047{
3048 struct igb_buffer *buffer_info;
3049 unsigned long size;
3050 unsigned int i;
3051
3052 if (!tx_ring->buffer_info)
3053 return;
3054 /* Free all the Tx ring sk_buffs */
3055
3056 for (i = 0; i < tx_ring->count; i++) {
3057 buffer_info = &tx_ring->buffer_info[i];
80785298 3058 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3059 }
3060
3061 size = sizeof(struct igb_buffer) * tx_ring->count;
3062 memset(tx_ring->buffer_info, 0, size);
3063
3064 /* Zero out the descriptor ring */
9d5c8243
AK		 3065 memset(tx_ring->desc, 0, tx_ring->size);
3066
3067 tx_ring->next_to_use = 0;
3068 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3069}
3070
3071/**
3072 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3073 * @adapter: board private structure
3074 **/
3075static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3076{
3077 int i;
3078
3079 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3080 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3081}
3082
3083/**
3084 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3085 * @rx_ring: ring to clean the resources from
3086 *
3087 * Free all receive software resources
3088 **/
68fd9910 3089void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3090{
3b644cf6 3091 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3092
3093 vfree(rx_ring->buffer_info);
3094 rx_ring->buffer_info = NULL;
3095
439705e1
AD		 3096 /* if not set, then don't free */
3097 if (!rx_ring->desc)
3098 return;
3099
59d71989
AD		 3100 dma_free_coherent(rx_ring->dev, rx_ring->size,
3101 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3102
3103 rx_ring->desc = NULL;
3104}
3105
3106/**
3107 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3108 * @adapter: board private structure
3109 *
3110 * Free all receive software resources
3111 **/
3112static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3113{
3114 int i;
3115
3116 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3117 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3118}
3119
3120/**
3121 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3122 * @rx_ring: ring to free buffers from
3123 **/
3b644cf6 3124static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3125{
3126 struct igb_buffer *buffer_info;
9d5c8243
AK		 3127 unsigned long size;
3128 unsigned int i;
3129
3130 if (!rx_ring->buffer_info)
3131 return;
439705e1 3132
9d5c8243
AK		 3133 /* Free all the Rx ring sk_buffs */
3134 for (i = 0; i < rx_ring->count; i++) {
3135 buffer_info = &rx_ring->buffer_info[i];
3136 if (buffer_info->dma) {
59d71989 3137 dma_unmap_single(rx_ring->dev,
80785298 3138 buffer_info->dma,
4c844851 3139 rx_ring->rx_buffer_len,
59d71989 3140 DMA_FROM_DEVICE);
9d5c8243
AK		 3141 buffer_info->dma = 0;
3142 }
3143
3144 if (buffer_info->skb) {
3145 dev_kfree_skb(buffer_info->skb);
3146 buffer_info->skb = NULL;
3147 }
6ec43fe6 3148 if (buffer_info->page_dma) {
59d71989 3149 dma_unmap_page(rx_ring->dev,
80785298 3150 buffer_info->page_dma,
6ec43fe6 3151 PAGE_SIZE / 2,
59d71989 3152 DMA_FROM_DEVICE);
6ec43fe6
AD		 3153 buffer_info->page_dma = 0;
3154 }
9d5c8243 3155 if (buffer_info->page) {
9d5c8243
AK		 3156 put_page(buffer_info->page);
3157 buffer_info->page = NULL;
bf36c1a0 3158 buffer_info->page_offset = 0;
9d5c8243
AK		 3159 }
3160 }
3161
9d5c8243
AK		 3162 size = sizeof(struct igb_buffer) * rx_ring->count;
3163 memset(rx_ring->buffer_info, 0, size);
3164
3165 /* Zero out the descriptor ring */
3166 memset(rx_ring->desc, 0, rx_ring->size);
3167
3168 rx_ring->next_to_clean = 0;
3169 rx_ring->next_to_use = 0;
9d5c8243
AK		 3170}
3171
3172/**
3173 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3174 * @adapter: board private structure
3175 **/
3176static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3177{
3178 int i;
3179
3180 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3181 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3182}
3183
3184/**
3185 * igb_set_mac - Change the Ethernet Address of the NIC
3186 * @netdev: network interface device structure
3187 * @p: pointer to an address structure
3188 *
3189 * Returns 0 on success, negative on failure
3190 **/
3191static int igb_set_mac(struct net_device *netdev, void *p)
3192{
3193 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3194 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3195 struct sockaddr *addr = p;
3196
3197 if (!is_valid_ether_addr(addr->sa_data))
3198 return -EADDRNOTAVAIL;
3199
3200 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3201 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3202
26ad9178
AD		 3203 /* set the correct pool for the new PF MAC address in entry 0 */
3204 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3205 adapter->vfs_allocated_count);
e1739522 3206
9d5c8243
AK		 3207 return 0;
3208}
3209
3210/**
68d480c4 3211 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3212 * @netdev: network interface device structure
3213 *
68d480c4
AD		 3214 * Writes multicast address list to the MTA hash table.
3215 * Returns: -ENOMEM on failure
3216 * 0 on no addresses written
3217 * X on writing X addresses to MTA
9d5c8243 3218 **/
68d480c4 3219static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3220{
3221 struct igb_adapter *adapter = netdev_priv(netdev);
3222 struct e1000_hw *hw = &adapter->hw;
22bedad3 3223 struct netdev_hw_addr *ha;
68d480c4 3224 u8 *mta_list;
9d5c8243
AK		 3225 int i;
3226
4cd24eaf 3227 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3228 /* nothing to program, so clear mc list */
3229 igb_update_mc_addr_list(hw, NULL, 0);
3230 igb_restore_vf_multicasts(adapter);
3231 return 0;
3232 }
9d5c8243 3233
4cd24eaf 3234 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3235 if (!mta_list)
3236 return -ENOMEM;
ff41f8dc 3237
68d480c4 3238 /* The shared function expects a packed array of only addresses. */
48e2f183 3239 i = 0;
22bedad3
JP		 3240 netdev_for_each_mc_addr(ha, netdev)
3241 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3242
68d480c4
AD		 3243 igb_update_mc_addr_list(hw, mta_list, i);
3244 kfree(mta_list);
3245
4cd24eaf 3246 return netdev_mc_count(netdev);
68d480c4
AD		 3247}
3248
3249/**
3250 * igb_write_uc_addr_list - write unicast addresses to RAR table
3251 * @netdev: network interface device structure
3252 *
3253 * Writes unicast address list to the RAR table.
3254 * Returns: -ENOMEM on failure/insufficient address space
3255 * 0 on no addresses written
3256 * X on writing X addresses to the RAR table
3257 **/
3258static int igb_write_uc_addr_list(struct net_device *netdev)
3259{
3260 struct igb_adapter *adapter = netdev_priv(netdev);
3261 struct e1000_hw *hw = &adapter->hw;
3262 unsigned int vfn = adapter->vfs_allocated_count;
3263 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3264 int count = 0;
3265
3266 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3267 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3268 return -ENOMEM;
9d5c8243 3269
32e7bfc4 3270 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3271 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3272
3273 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3274 if (!rar_entries)
3275 break;
26ad9178
AD		 3276 igb_rar_set_qsel(adapter, ha->addr,
3277 rar_entries--,
68d480c4
AD		 3278 vfn);
3279 count++;
ff41f8dc
AD		 3280 }
3281 }
3282 /* write the addresses in reverse order to avoid write combining */
3283 for (; rar_entries > 0 ; rar_entries--) {
3284 wr32(E1000_RAH(rar_entries), 0);
3285 wr32(E1000_RAL(rar_entries), 0);
3286 }
3287 wrfl();
3288
68d480c4
AD		 3289 return count;
3290}
3291
3292/**
3293 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3294 * @netdev: network interface device structure
3295 *
3296 * The set_rx_mode entry point is called whenever the unicast or multicast
3297 * address lists or the network interface flags are updated. This routine is
3298 * responsible for configuring the hardware for proper unicast, multicast,
3299 * promiscuous mode, and all-multi behavior.
3300 **/
3301static void igb_set_rx_mode(struct net_device *netdev)
3302{
3303 struct igb_adapter *adapter = netdev_priv(netdev);
3304 struct e1000_hw *hw = &adapter->hw;
3305 unsigned int vfn = adapter->vfs_allocated_count;
3306 u32 rctl, vmolr = 0;
3307 int count;
3308
3309 /* Check for Promiscuous and All Multicast modes */
3310 rctl = rd32(E1000_RCTL);
3311
3312 /* clear the effected bits */
3313 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3314
3315 if (netdev->flags & IFF_PROMISC) {
3316 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3317 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3318 } else {
3319 if (netdev->flags & IFF_ALLMULTI) {
3320 rctl |= E1000_RCTL_MPE;
3321 vmolr |= E1000_VMOLR_MPME;
3322 } else {
3323 /*
3324 * Write addresses to the MTA, if the attempt fails
3325 * then we should just turn on promiscous mode so
3326 * that we can at least receive multicast traffic
3327 */
3328 count = igb_write_mc_addr_list(netdev);
3329 if (count < 0) {
3330 rctl |= E1000_RCTL_MPE;
3331 vmolr |= E1000_VMOLR_MPME;
3332 } else if (count) {
3333 vmolr |= E1000_VMOLR_ROMPE;
3334 }
3335 }
3336 /*
3337 * Write addresses to available RAR registers, if there is not
3338 * sufficient space to store all the addresses then enable
3339 * unicast promiscous mode
3340 */
3341 count = igb_write_uc_addr_list(netdev);
3342 if (count < 0) {
3343 rctl |= E1000_RCTL_UPE;
3344 vmolr |= E1000_VMOLR_ROPE;
3345 }
3346 rctl |= E1000_RCTL_VFE;
28fc06f5 3347 }
68d480c4 3348 wr32(E1000_RCTL, rctl);
28fc06f5 3349
68d480c4
AD		 3350 /*
3351 * In order to support SR-IOV and eventually VMDq it is necessary to set
3352 * the VMOLR to enable the appropriate modes. Without this workaround
3353 * we will have issues with VLAN tag stripping not being done for frames
3354 * that are only arriving because we are the default pool
3355 */
3356 if (hw->mac.type < e1000_82576)
28fc06f5 3357 return;
9d5c8243 3358
68d480c4
AD		 3359 vmolr |= rd32(E1000_VMOLR(vfn)) &
3360 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3361 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3362 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3363}
3364
3365/* Need to wait a few seconds after link up to get diagnostic information from
3366 * the phy */
3367static void igb_update_phy_info(unsigned long data)
3368{
3369 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3370 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3371}
3372
4d6b725e
AD		 3373/**
3374 * igb_has_link - check shared code for link and determine up/down
3375 * @adapter: pointer to driver private info
3376 **/
3145535a 3377bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3378{
3379 struct e1000_hw *hw = &adapter->hw;
3380 bool link_active = false;
3381 s32 ret_val = 0;
3382
3383 /* get_link_status is set on LSC (link status) interrupt or
3384 * rx sequence error interrupt. get_link_status will stay
3385 * false until the e1000_check_for_link establishes link
3386 * for copper adapters ONLY
3387 */
3388 switch (hw->phy.media_type) {
3389 case e1000_media_type_copper:
3390 if (hw->mac.get_link_status) {
3391 ret_val = hw->mac.ops.check_for_link(hw);
3392 link_active = !hw->mac.get_link_status;
3393 } else {
3394 link_active = true;
3395 }
3396 break;
4d6b725e
AD		 3397 case e1000_media_type_internal_serdes:
3398 ret_val = hw->mac.ops.check_for_link(hw);
3399 link_active = hw->mac.serdes_has_link;
3400 break;
3401 default:
3402 case e1000_media_type_unknown:
3403 break;
3404 }
3405
3406 return link_active;
3407}
3408
9d5c8243
AK		 3409/**
3410 * igb_watchdog - Timer Call-back
3411 * @data: pointer to adapter cast into an unsigned long
3412 **/
3413static void igb_watchdog(unsigned long data)
3414{
3415 struct igb_adapter *adapter = (struct igb_adapter *)data;
3416 /* Do the rest outside of interrupt context */
3417 schedule_work(&adapter->watchdog_task);
3418}
3419
3420static void igb_watchdog_task(struct work_struct *work)
3421{
3422 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3423 struct igb_adapter,
3424 watchdog_task);
9d5c8243 3425 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3426 struct net_device *netdev = adapter->netdev;
9d5c8243 3427 u32 link;
7a6ea550 3428 int i;
9d5c8243 3429
4d6b725e 3430 link = igb_has_link(adapter);
9d5c8243
AK		 3431 if (link) {
3432 if (!netif_carrier_ok(netdev)) {
3433 u32 ctrl;
330a6d6a
AD		 3434 hw->mac.ops.get_speed_and_duplex(hw,
3435 &adapter->link_speed,
3436 &adapter->link_duplex);
9d5c8243
AK		 3437
3438 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3439 /* Links status message must follow this format */
3440 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3441 "Flow Control: %s\n",
559e9c49
AD		 3442 netdev->name,
3443 adapter->link_speed,
3444 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3445 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3446 ((ctrl & E1000_CTRL_TFCE) &&
3447 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3448 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3449 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3450
d07f3e37 3451 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3452 adapter->tx_timeout_factor = 1;
3453 switch (adapter->link_speed) {
3454 case SPEED_10:
9d5c8243
AK		 3455 adapter->tx_timeout_factor = 14;
3456 break;
3457 case SPEED_100:
9d5c8243
AK		 3458 /* maybe add some timeout factor ? */
3459 break;
3460 }
3461
3462 netif_carrier_on(netdev);
9d5c8243 3463
4ae196df
AD		 3464 igb_ping_all_vfs(adapter);
3465
4b1a9877 3466 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3470 }
3471 } else {
3472 if (netif_carrier_ok(netdev)) {
3473 adapter->link_speed = 0;
3474 adapter->link_duplex = 0;
527d47c1
AD		 3475 /* Links status message must follow this format */
3476 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3477 netdev->name);
9d5c8243 3478 netif_carrier_off(netdev);
4b1a9877 3479
4ae196df
AD		 3480 igb_ping_all_vfs(adapter);
3481
4b1a9877 3482 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3483 if (!test_bit(__IGB_DOWN, &adapter->state))
3484 mod_timer(&adapter->phy_info_timer,
3485 round_jiffies(jiffies + 2 * HZ));
3486 }
3487 }
3488
12dcd86b
ED		 3489 spin_lock(&adapter->stats64_lock);
3490 igb_update_stats(adapter, &adapter->stats64);
3491 spin_unlock(&adapter->stats64_lock);
9d5c8243 3492
dbabb065 3493 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3494 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3495 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3496 /* We've lost link, so the controller stops DMA,
3497 * but we've got queued Tx work that's never going
3498 * to get done, so reset controller to flush Tx.
3499 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3500 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3501 adapter->tx_timeout_count++;
3502 schedule_work(&adapter->reset_task);
3503 /* return immediately since reset is imminent */
3504 return;
3505 }
9d5c8243 3506 }
9d5c8243 3507
dbabb065
AD		 3508 /* Force detection of hung controller every watchdog period */
3509 tx_ring->detect_tx_hung = true;
3510 }
f7ba205e 3511
9d5c8243 3512 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3513 if (adapter->msix_entries) {
047e0030
AD		 3514 u32 eics = 0;
3515 for (i = 0; i < adapter->num_q_vectors; i++) {
3516 struct igb_q_vector *q_vector = adapter->q_vector[i];
3517 eics |= q_vector->eims_value;
3518 }
7a6ea550
AD		 3519 wr32(E1000_EICS, eics);
3520 } else {
3521 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3522 }
9d5c8243 3523
9d5c8243
AK		 3524 /* Reset the timer */
3525 if (!test_bit(__IGB_DOWN, &adapter->state))
3526 mod_timer(&adapter->watchdog_timer,
3527 round_jiffies(jiffies + 2 * HZ));
3528}
3529
3530enum latency_range {
3531 lowest_latency = 0,
3532 low_latency = 1,
3533 bulk_latency = 2,
3534 latency_invalid = 255
3535};
3536
6eb5a7f1
AD		 3537/**
3538 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3539 *
3540 * Stores a new ITR value based on strictly on packet size. This
3541 * algorithm is less sophisticated than that used in igb_update_itr,
3542 * due to the difficulty of synchronizing statistics across multiple
3543 * receive rings. The divisors and thresholds used by this fuction
3544 * were determined based on theoretical maximum wire speed and testing
3545 * data, in order to minimize response time while increasing bulk
3546 * throughput.
3547 * This functionality is controlled by the InterruptThrottleRate module
3548 * parameter (see igb_param.c)
3549 * NOTE: This function is called only when operating in a multiqueue
3550 * receive environment.
047e0030 3551 * @q_vector: pointer to q_vector
6eb5a7f1 3552 **/
047e0030 3553static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3554{
047e0030 3555 int new_val = q_vector->itr_val;
6eb5a7f1 3556 int avg_wire_size = 0;
047e0030 3557 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3558 struct igb_ring *ring;
3559 unsigned int packets;
9d5c8243 3560
6eb5a7f1
AD		 3561 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3562 * ints/sec - ITR timer value of 120 ticks.
3563 */
3564 if (adapter->link_speed != SPEED_1000) {
047e0030 3565 new_val = 976;
6eb5a7f1 3566 goto set_itr_val;
9d5c8243 3567 }
047e0030 3568
12dcd86b
ED		 3569 ring = q_vector->rx_ring;
3570 if (ring) {
3571 packets = ACCESS_ONCE(ring->total_packets);
3572
3573 if (packets)
3574 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3575 }
3576
12dcd86b
ED		 3577 ring = q_vector->tx_ring;
3578 if (ring) {
3579 packets = ACCESS_ONCE(ring->total_packets);
3580
3581 if (packets)
3582 avg_wire_size = max_t(u32, avg_wire_size,
3583 ring->total_bytes / packets);
047e0030
AD		 3584 }
3585
3586 /* if avg_wire_size isn't set no work was done */
3587 if (!avg_wire_size)
3588 goto clear_counts;
9d5c8243 3589
6eb5a7f1
AD		 3590 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3591 avg_wire_size += 24;
3592
3593 /* Don't starve jumbo frames */
3594 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3595
6eb5a7f1
AD		 3596 /* Give a little boost to mid-size frames */
3597 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3598 new_val = avg_wire_size / 3;
3599 else
3600 new_val = avg_wire_size / 2;
9d5c8243 3601
abe1c363
NN		 3602 /* when in itr mode 3 do not exceed 20K ints/sec */
3603 if (adapter->rx_itr_setting == 3 && new_val < 196)
3604 new_val = 196;
3605
6eb5a7f1 3606set_itr_val:
047e0030
AD		 3607 if (new_val != q_vector->itr_val) {
3608 q_vector->itr_val = new_val;
3609 q_vector->set_itr = 1;
9d5c8243 3610 }
6eb5a7f1 3611clear_counts:
047e0030
AD		 3612 if (q_vector->rx_ring) {
3613 q_vector->rx_ring->total_bytes = 0;
3614 q_vector->rx_ring->total_packets = 0;
3615 }
3616 if (q_vector->tx_ring) {
3617 q_vector->tx_ring->total_bytes = 0;
3618 q_vector->tx_ring->total_packets = 0;
3619 }
9d5c8243
AK		 3620}
3621
3622/**
3623 * igb_update_itr - update the dynamic ITR value based on statistics
3624 * Stores a new ITR value based on packets and byte
3625 * counts during the last interrupt. The advantage of per interrupt
3626 * computation is faster updates and more accurate ITR for the current
3627 * traffic pattern. Constants in this function were computed
3628 * based on theoretical maximum wire speed and thresholds were set based
3629 * on testing data as well as attempting to minimize response time
3630 * while increasing bulk throughput.
3631 * this functionality is controlled by the InterruptThrottleRate module
3632 * parameter (see igb_param.c)
3633 * NOTE: These calculations are only valid when operating in a single-
3634 * queue environment.
3635 * @adapter: pointer to adapter
047e0030 3636 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3637 * @packets: the number of packets during this measurement interval
3638 * @bytes: the number of bytes during this measurement interval
3639 **/
3640static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3641 int packets, int bytes)
3642{
3643 unsigned int retval = itr_setting;
3644
3645 if (packets == 0)
3646 goto update_itr_done;
3647
3648 switch (itr_setting) {
3649 case lowest_latency:
3650 /* handle TSO and jumbo frames */
3651 if (bytes/packets > 8000)
3652 retval = bulk_latency;
3653 else if ((packets < 5) && (bytes > 512))
3654 retval = low_latency;
3655 break;
3656 case low_latency: /* 50 usec aka 20000 ints/s */
3657 if (bytes > 10000) {
3658 /* this if handles the TSO accounting */
3659 if (bytes/packets > 8000) {
3660 retval = bulk_latency;
3661 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3662 retval = bulk_latency;
3663 } else if ((packets > 35)) {
3664 retval = lowest_latency;
3665 }
3666 } else if (bytes/packets > 2000) {
3667 retval = bulk_latency;
3668 } else if (packets <= 2 && bytes < 512) {
3669 retval = lowest_latency;
3670 }
3671 break;
3672 case bulk_latency: /* 250 usec aka 4000 ints/s */
3673 if (bytes > 25000) {
3674 if (packets > 35)
3675 retval = low_latency;
1e5c3d21 3676 } else if (bytes < 1500) {
9d5c8243
AK		 3677 retval = low_latency;
3678 }
3679 break;
3680 }
3681
3682update_itr_done:
3683 return retval;
3684}
3685
6eb5a7f1 3686static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3687{
047e0030 3688 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3689 u16 current_itr;
047e0030 3690 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3691
3692 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3693 if (adapter->link_speed != SPEED_1000) {
3694 current_itr = 0;
3695 new_itr = 4000;
3696 goto set_itr_now;
3697 }
3698
3699 adapter->rx_itr = igb_update_itr(adapter,
3700 adapter->rx_itr,
3025a446
AD		 3701 q_vector->rx_ring->total_packets,
3702 q_vector->rx_ring->total_bytes);
9d5c8243 3703
047e0030
AD		 3704 adapter->tx_itr = igb_update_itr(adapter,
3705 adapter->tx_itr,
3025a446
AD		 3706 q_vector->tx_ring->total_packets,
3707 q_vector->tx_ring->total_bytes);
047e0030 3708 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3709
6eb5a7f1 3710 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3711 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3712 current_itr = low_latency;
3713
9d5c8243
AK		 3714 switch (current_itr) {
3715 /* counts and packets in update_itr are dependent on these numbers */
3716 case lowest_latency:
78b1f607 3717 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3718 break;
3719 case low_latency:
78b1f607 3720 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3721 break;
3722 case bulk_latency:
78b1f607 3723 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3724 break;
3725 default:
3726 break;
3727 }
3728
3729set_itr_now:
3025a446
AD		 3730 q_vector->rx_ring->total_bytes = 0;
3731 q_vector->rx_ring->total_packets = 0;
3732 q_vector->tx_ring->total_bytes = 0;
3733 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3734
047e0030 3735 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3736 /* this attempts to bias the interrupt rate towards Bulk
3737 * by adding intermediate steps when interrupt rate is
3738 * increasing */
047e0030
AD		 3739 new_itr = new_itr > q_vector->itr_val ?
3740 max((new_itr * q_vector->itr_val) /
3741 (new_itr + (q_vector->itr_val >> 2)),
3742 new_itr) :
9d5c8243
AK		 3743 new_itr;
3744 /* Don't write the value here; it resets the adapter's
3745 * internal timer, and causes us to delay far longer than
3746 * we should between interrupts. Instead, we write the ITR
3747 * value at the beginning of the next interrupt so the timing
3748 * ends up being correct.
3749 */
047e0030
AD		 3750 q_vector->itr_val = new_itr;
3751 q_vector->set_itr = 1;
9d5c8243 3752 }
9d5c8243
AK		 3753}
3754
9d5c8243
AK		 3755#define IGB_TX_FLAGS_CSUM 0x00000001
3756#define IGB_TX_FLAGS_VLAN 0x00000002
3757#define IGB_TX_FLAGS_TSO 0x00000004
3758#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3759#define IGB_TX_FLAGS_TSTAMP 0x00000010
3760#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3761#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3762
85ad76b2 3763static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3764 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3765{
3766 struct e1000_adv_tx_context_desc *context_desc;
3767 unsigned int i;
3768 int err;
3769 struct igb_buffer *buffer_info;
3770 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3771 u32 mss_l4len_idx;
3772 u8 l4len;
9d5c8243
AK		 3773
3774 if (skb_header_cloned(skb)) {
3775 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3776 if (err)
3777 return err;
3778 }
3779
3780 l4len = tcp_hdrlen(skb);
3781 *hdr_len += l4len;
3782
3783 if (skb->protocol == htons(ETH_P_IP)) {
3784 struct iphdr *iph = ip_hdr(skb);
3785 iph->tot_len = 0;
3786 iph->check = 0;
3787 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3788 iph->daddr, 0,
3789 IPPROTO_TCP,
3790 0);
8e1e8a47 3791 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3792 ipv6_hdr(skb)->payload_len = 0;
3793 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3794 &ipv6_hdr(skb)->daddr,
3795 0, IPPROTO_TCP, 0);
3796 }
3797
3798 i = tx_ring->next_to_use;
3799
3800 buffer_info = &tx_ring->buffer_info[i];
3801 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3802 /* VLAN MACLEN IPLEN */
3803 if (tx_flags & IGB_TX_FLAGS_VLAN)
3804 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3805 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3806 *hdr_len += skb_network_offset(skb);
3807 info |= skb_network_header_len(skb);
3808 *hdr_len += skb_network_header_len(skb);
3809 context_desc->vlan_macip_lens = cpu_to_le32(info);
3810
3811 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3812 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3813
3814 if (skb->protocol == htons(ETH_P_IP))
3815 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3816 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3817
3818 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3819
3820 /* MSS L4LEN IDX */
3821 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3822 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3823
73cd78f1 3824 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3825 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3826 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3827
3828 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3829 context_desc->seqnum_seed = 0;
3830
3831 buffer_info->time_stamp = jiffies;
0e014cb1 3832 buffer_info->next_to_watch = i;
9d5c8243
AK		 3833 buffer_info->dma = 0;
3834 i++;
3835 if (i == tx_ring->count)
3836 i = 0;
3837
3838 tx_ring->next_to_use = i;
3839
3840 return true;
3841}
3842
85ad76b2
AD		 3843static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3844 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3845{
3846 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3847 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3848 struct igb_buffer *buffer_info;
3849 u32 info = 0, tu_cmd = 0;
80785298 3850 unsigned int i;
9d5c8243
AK		 3851
3852 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3853 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3854 i = tx_ring->next_to_use;
3855 buffer_info = &tx_ring->buffer_info[i];
3856 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3857
3858 if (tx_flags & IGB_TX_FLAGS_VLAN)
3859 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3860
9d5c8243
AK		 3861 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3862 if (skb->ip_summed == CHECKSUM_PARTIAL)
3863 info |= skb_network_header_len(skb);
3864
3865 context_desc->vlan_macip_lens = cpu_to_le32(info);
3866
3867 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3868
3869 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3870 __be16 protocol;
3871
3872 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3873 const struct vlan_ethhdr *vhdr =
3874 (const struct vlan_ethhdr*)skb->data;
3875
3876 protocol = vhdr->h_vlan_encapsulated_proto;
3877 } else {
3878 protocol = skb->protocol;
3879 }
3880
3881 switch (protocol) {
09640e63 3882 case cpu_to_be16(ETH_P_IP):
9d5c8243 3883 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3884 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3885 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3886 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3887 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3888 break;
09640e63 3889 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3890 /* XXX what about other V6 headers?? */
3891 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3892 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3893 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3894 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3895 break;
3896 default:
3897 if (unlikely(net_ratelimit()))
59d71989 3898 dev_warn(dev,
44b0cda3
MW		 3899 "partial checksum but proto=%x!\n",
3900 skb->protocol);
3901 break;
3902 }
9d5c8243
AK		 3903 }
3904
3905 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3906 context_desc->seqnum_seed = 0;
85ad76b2 3907 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3908 context_desc->mss_l4len_idx =
85ad76b2 3909 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3910
3911 buffer_info->time_stamp = jiffies;
0e014cb1 3912 buffer_info->next_to_watch = i;
9d5c8243
AK		 3913 buffer_info->dma = 0;
3914
3915 i++;
3916 if (i == tx_ring->count)
3917 i = 0;
3918 tx_ring->next_to_use = i;
3919
3920 return true;
3921 }
9d5c8243
AK		 3922 return false;
3923}
3924
3925#define IGB_MAX_TXD_PWR 16
3926#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3927
80785298 3928static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3929 unsigned int first)
9d5c8243
AK		 3930{
3931 struct igb_buffer *buffer_info;
59d71989 3932 struct device *dev = tx_ring->dev;
2873957d 3933 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3934 unsigned int count = 0, i;
3935 unsigned int f;
2873957d 3936 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3937
3938 i = tx_ring->next_to_use;
3939
3940 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3941 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3942 buffer_info->length = hlen;
9d5c8243
AK		 3943 /* set time_stamp *before* dma to help avoid a possible race */
3944 buffer_info->time_stamp = jiffies;
0e014cb1 3945 buffer_info->next_to_watch = i;
2873957d 3946 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3947 DMA_TO_DEVICE);
3948 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3949 goto dma_error;
9d5c8243
AK		 3950
3951 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3952 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3953 unsigned int len = frag->size;
9d5c8243 3954
8581145f 3955 count++;
65689fef
AD		 3956 i++;
3957 if (i == tx_ring->count)
3958 i = 0;
3959
9d5c8243
AK		 3960 buffer_info = &tx_ring->buffer_info[i];
3961 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3962 buffer_info->length = len;
3963 buffer_info->time_stamp = jiffies;
0e014cb1 3964 buffer_info->next_to_watch = i;
6366ad33 3965 buffer_info->mapped_as_page = true;
59d71989 3966 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3967 frag->page,
3968 frag->page_offset,
3969 len,
59d71989
AD		 3970 DMA_TO_DEVICE);
3971 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3972 goto dma_error;
3973
9d5c8243
AK		 3974 }
3975
9d5c8243 3976 tx_ring->buffer_info[i].skb = skb;
2244d07b 3977 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 3978 /* multiply data chunks by size of headers */
3979 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3980 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3981 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3982
cdfd01fc 3983 return ++count;
6366ad33
AD		 3984
3985dma_error:
59d71989 3986 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3987
3988 /* clear timestamp and dma mappings for failed buffer_info mapping */
3989 buffer_info->dma = 0;
3990 buffer_info->time_stamp = 0;
3991 buffer_info->length = 0;
3992 buffer_info->next_to_watch = 0;
3993 buffer_info->mapped_as_page = false;
6366ad33
AD		 3994
3995 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3996 while (count--) {
3997 if (i == 0)
3998 i = tx_ring->count;
6366ad33 3999 i--;
6366ad33
AD		 4000 buffer_info = &tx_ring->buffer_info[i];
4001 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4002 }
4003
4004 return 0;
9d5c8243
AK		 4005}
4006
85ad76b2 4007static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 4008 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 4009 u8 hdr_len)
4010{
cdfd01fc 4011 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4012 struct igb_buffer *buffer_info;
4013 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4014 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4015
4016 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4017 E1000_ADVTXD_DCMD_DEXT);
4018
4019 if (tx_flags & IGB_TX_FLAGS_VLAN)
4020 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4021
33af6bcc
PO		 4022 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4023 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4024
9d5c8243
AK		 4025 if (tx_flags & IGB_TX_FLAGS_TSO) {
4026 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4027
4028 /* insert tcp checksum */
4029 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4030
4031 /* insert ip checksum */
4032 if (tx_flags & IGB_TX_FLAGS_IPV4)
4033 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4034
4035 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4036 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4037 }
4038
85ad76b2
AD		 4039 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4040 (tx_flags & (IGB_TX_FLAGS_CSUM |
4041 IGB_TX_FLAGS_TSO |
7dfc16fa 4042 IGB_TX_FLAGS_VLAN)))
85ad76b2 4043 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4044
4045 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4046
cdfd01fc 4047 do {
9d5c8243
AK		 4048 buffer_info = &tx_ring->buffer_info[i];
4049 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4050 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4051 tx_desc->read.cmd_type_len =
4052 cpu_to_le32(cmd_type_len | buffer_info->length);
4053 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4054 count--;
9d5c8243
AK		 4055 i++;
4056 if (i == tx_ring->count)
4057 i = 0;
cdfd01fc 4058 } while (count > 0);
9d5c8243 4059
85ad76b2 4060 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4061 /* Force memory writes to complete before letting h/w
4062 * know there are new descriptors to fetch. (Only
4063 * applicable for weak-ordered memory model archs,
4064 * such as IA-64). */
4065 wmb();
4066
4067 tx_ring->next_to_use = i;
fce99e34 4068 writel(i, tx_ring->tail);
9d5c8243
AK		 4069 /* we need this if more than one processor can write to our tail
4070 * at a time, it syncronizes IO on IA64/Altix systems */
4071 mmiowb();
4072}
4073
e694e964 4074static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4075{
e694e964
AD		 4076 struct net_device *netdev = tx_ring->netdev;
4077
661086df 4078 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4079
9d5c8243
AK		 4080 /* Herbert's original patch had:
4081 * smp_mb__after_netif_stop_queue();
4082 * but since that doesn't exist yet, just open code it. */
4083 smp_mb();
4084
4085 /* We need to check again in a case another CPU has just
4086 * made room available. */
c493ea45 4087 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4088 return -EBUSY;
4089
4090 /* A reprieve! */
661086df 4091 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4092
4093 u64_stats_update_begin(&tx_ring->tx_syncp2);
4094 tx_ring->tx_stats.restart_queue2++;
4095 u64_stats_update_end(&tx_ring->tx_syncp2);
4096
9d5c8243
AK		 4097 return 0;
4098}
4099
717ba089 4100static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4101{
c493ea45 4102 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4103 return 0;
e694e964 4104 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4105}
4106
b1a436c3
AD		 4107netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4108 struct igb_ring *tx_ring)
9d5c8243 4109{
e694e964 4110 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 4111 int tso = 0, count;
91d4ee33
NN		 4112 u32 tx_flags = 0;
4113 u16 first;
4114 u8 hdr_len = 0;
9d5c8243 4115
9d5c8243
AK		 4116 /* need: 1 descriptor per page,
4117 * + 2 desc gap to keep tail from touching head,
4118 * + 1 desc for skb->data,
4119 * + 1 desc for context descriptor,
4120 * otherwise try next time */
e694e964 4121 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4122 /* this is a hard error */
9d5c8243
AK		 4123 return NETDEV_TX_BUSY;
4124 }
33af6bcc 4125
2244d07b
OH		 4126 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4127 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4128 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4129 }
9d5c8243 4130
cdfd01fc 4131 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 4132 tx_flags |= IGB_TX_FLAGS_VLAN;
4133 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4134 }
4135
661086df
PWJ		 4136 if (skb->protocol == htons(ETH_P_IP))
4137 tx_flags |= IGB_TX_FLAGS_IPV4;
4138
0e014cb1 4139 first = tx_ring->next_to_use;
85ad76b2
AD		 4140 if (skb_is_gso(skb)) {
4141 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4142
85ad76b2
AD		 4143 if (tso < 0) {
4144 dev_kfree_skb_any(skb);
4145 return NETDEV_TX_OK;
4146 }
9d5c8243
AK		 4147 }
4148
4149 if (tso)
4150 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4151 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4152 (skb->ip_summed == CHECKSUM_PARTIAL))
4153 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4154
65689fef 4155 /*
cdfd01fc 4156 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4157 * has occured and we need to rewind the descriptor queue
4158 */
80785298 4159 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4160 if (!count) {
65689fef
AD		 4161 dev_kfree_skb_any(skb);
4162 tx_ring->buffer_info[first].time_stamp = 0;
4163 tx_ring->next_to_use = first;
85ad76b2 4164 return NETDEV_TX_OK;
65689fef 4165 }
9d5c8243 4166
85ad76b2
AD		 4167 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4168
4169 /* Make sure there is space in the ring for the next send. */
e694e964 4170 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4171
9d5c8243
AK		 4172 return NETDEV_TX_OK;
4173}
4174
3b29a56d
SH		 4175static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4176 struct net_device *netdev)
9d5c8243
AK		 4177{
4178 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4179 struct igb_ring *tx_ring;
661086df 4180 int r_idx = 0;
b1a436c3
AD		 4181
4182 if (test_bit(__IGB_DOWN, &adapter->state)) {
4183 dev_kfree_skb_any(skb);
4184 return NETDEV_TX_OK;
4185 }
4186
4187 if (skb->len <= 0) {
4188 dev_kfree_skb_any(skb);
4189 return NETDEV_TX_OK;
4190 }
4191
1bfaf07b 4192 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4193 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4194
4195 /* This goes back to the question of how to logically map a tx queue
4196 * to a flow. Right now, performance is impacted slightly negatively
4197 * if using multiple tx queues. If the stack breaks away from a
4198 * single qdisc implementation, we can look at this again. */
e694e964 4199 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4200}
4201
4202/**
4203 * igb_tx_timeout - Respond to a Tx Hang
4204 * @netdev: network interface device structure
4205 **/
4206static void igb_tx_timeout(struct net_device *netdev)
4207{
4208 struct igb_adapter *adapter = netdev_priv(netdev);
4209 struct e1000_hw *hw = &adapter->hw;
4210
4211 /* Do the reset outside of interrupt context */
4212 adapter->tx_timeout_count++;
f7ba205e 4213
55cac248
AD		 4214 if (hw->mac.type == e1000_82580)
4215 hw->dev_spec._82575.global_device_reset = true;
4216
9d5c8243 4217 schedule_work(&adapter->reset_task);
265de409
AD		 4218 wr32(E1000_EICS,
4219 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4220}
4221
4222static void igb_reset_task(struct work_struct *work)
4223{
4224 struct igb_adapter *adapter;
4225 adapter = container_of(work, struct igb_adapter, reset_task);
4226
c97ec42a
TI		 4227 igb_dump(adapter);
4228 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4229 igb_reinit_locked(adapter);
4230}
4231
4232/**
12dcd86b 4233 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4234 * @netdev: network interface device structure
12dcd86b 4235 * @stats: rtnl_link_stats64 pointer
9d5c8243 4236 *
9d5c8243 4237 **/
12dcd86b
ED		 4238static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4239 struct rtnl_link_stats64 *stats)
9d5c8243 4240{
12dcd86b
ED		 4241 struct igb_adapter *adapter = netdev_priv(netdev);
4242
4243 spin_lock(&adapter->stats64_lock);
4244 igb_update_stats(adapter, &adapter->stats64);
4245 memcpy(stats, &adapter->stats64, sizeof(*stats));
4246 spin_unlock(&adapter->stats64_lock);
4247
4248 return stats;
9d5c8243
AK		 4249}
4250
4251/**
4252 * igb_change_mtu - Change the Maximum Transfer Unit
4253 * @netdev: network interface device structure
4254 * @new_mtu: new value for maximum frame size
4255 *
4256 * Returns 0 on success, negative on failure
4257 **/
4258static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4259{
4260 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4261 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4262 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4263 u32 rx_buffer_len, i;
9d5c8243 4264
c809d227 4265 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4266 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4267 return -EINVAL;
4268 }
4269
9d5c8243 4270 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4271 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4272 return -EINVAL;
4273 }
4274
4275 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4276 msleep(1);
73cd78f1 4277
9d5c8243
AK		 4278 /* igb_down has a dependency on max_frame_size */
4279 adapter->max_frame_size = max_frame;
559e9c49 4280
9d5c8243
AK		 4281 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4282 * means we reserve 2 more, this pushes us to allocate from the next
4283 * larger slab size.
4284 * i.e. RXBUFFER_2048 --> size-4096 slab
4285 */
4286
757b77e2
NN		 4287 if (adapter->hw.mac.type == e1000_82580)
4288 max_frame += IGB_TS_HDR_LEN;
4289
7d95b717 4290 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4291 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4292 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4293 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4294 else
4c844851
AD		 4295 rx_buffer_len = IGB_RXBUFFER_128;
4296
757b77e2
NN		 4297 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4298 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4299 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4300
4301 if ((adapter->hw.mac.type == e1000_82580) &&
4302 (rx_buffer_len == IGB_RXBUFFER_128))
4303 rx_buffer_len += IGB_RXBUFFER_64;
4304
4c844851
AD		 4305 if (netif_running(netdev))
4306 igb_down(adapter);
9d5c8243 4307
090b1795 4308 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4309 netdev->mtu, new_mtu);
4310 netdev->mtu = new_mtu;
4311
4c844851 4312 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4313 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4314
9d5c8243
AK		 4315 if (netif_running(netdev))
4316 igb_up(adapter);
4317 else
4318 igb_reset(adapter);
4319
4320 clear_bit(__IGB_RESETTING, &adapter->state);
4321
4322 return 0;
4323}
4324
4325/**
4326 * igb_update_stats - Update the board statistics counters
4327 * @adapter: board private structure
4328 **/
4329
12dcd86b
ED		 4330void igb_update_stats(struct igb_adapter *adapter,
4331 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4332{
4333 struct e1000_hw *hw = &adapter->hw;
4334 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4335 u32 reg, mpc;
9d5c8243 4336 u16 phy_tmp;
3f9c0164
AD		 4337 int i;
4338 u64 bytes, packets;
12dcd86b
ED		 4339 unsigned int start;
4340 u64 _bytes, _packets;
9d5c8243
AK		 4341
4342#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4343
4344 /*
4345 * Prevent stats update while adapter is being reset, or if the pci
4346 * connection is down.
4347 */
4348 if (adapter->link_speed == 0)
4349 return;
4350 if (pci_channel_offline(pdev))
4351 return;
4352
3f9c0164
AD		 4353 bytes = 0;
4354 packets = 0;
4355 for (i = 0; i < adapter->num_rx_queues; i++) {
4356 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4357 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4358
3025a446 4359 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4360 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4361
4362 do {
4363 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4364 _bytes = ring->rx_stats.bytes;
4365 _packets = ring->rx_stats.packets;
4366 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4367 bytes += _bytes;
4368 packets += _packets;
3f9c0164
AD		 4369 }
4370
128e45eb
AD		 4371 net_stats->rx_bytes = bytes;
4372 net_stats->rx_packets = packets;
3f9c0164
AD		 4373
4374 bytes = 0;
4375 packets = 0;
4376 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4377 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4378 do {
4379 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4380 _bytes = ring->tx_stats.bytes;
4381 _packets = ring->tx_stats.packets;
4382 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4383 bytes += _bytes;
4384 packets += _packets;
3f9c0164 4385 }
128e45eb
AD		 4386 net_stats->tx_bytes = bytes;
4387 net_stats->tx_packets = packets;
3f9c0164
AD		 4388
4389 /* read stats registers */
9d5c8243
AK		 4390 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4391 adapter->stats.gprc += rd32(E1000_GPRC);
4392 adapter->stats.gorc += rd32(E1000_GORCL);
4393 rd32(E1000_GORCH); /* clear GORCL */
4394 adapter->stats.bprc += rd32(E1000_BPRC);
4395 adapter->stats.mprc += rd32(E1000_MPRC);
4396 adapter->stats.roc += rd32(E1000_ROC);
4397
4398 adapter->stats.prc64 += rd32(E1000_PRC64);
4399 adapter->stats.prc127 += rd32(E1000_PRC127);
4400 adapter->stats.prc255 += rd32(E1000_PRC255);
4401 adapter->stats.prc511 += rd32(E1000_PRC511);
4402 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4403 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4404 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4405 adapter->stats.sec += rd32(E1000_SEC);
4406
fa3d9a6d
MW		 4407 mpc = rd32(E1000_MPC);
4408 adapter->stats.mpc += mpc;
4409 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4410 adapter->stats.scc += rd32(E1000_SCC);
4411 adapter->stats.ecol += rd32(E1000_ECOL);
4412 adapter->stats.mcc += rd32(E1000_MCC);
4413 adapter->stats.latecol += rd32(E1000_LATECOL);
4414 adapter->stats.dc += rd32(E1000_DC);
4415 adapter->stats.rlec += rd32(E1000_RLEC);
4416 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4417 adapter->stats.xontxc += rd32(E1000_XONTXC);
4418 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4419 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4420 adapter->stats.fcruc += rd32(E1000_FCRUC);
4421 adapter->stats.gptc += rd32(E1000_GPTC);
4422 adapter->stats.gotc += rd32(E1000_GOTCL);
4423 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4424 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4425 adapter->stats.ruc += rd32(E1000_RUC);
4426 adapter->stats.rfc += rd32(E1000_RFC);
4427 adapter->stats.rjc += rd32(E1000_RJC);
4428 adapter->stats.tor += rd32(E1000_TORH);
4429 adapter->stats.tot += rd32(E1000_TOTH);
4430 adapter->stats.tpr += rd32(E1000_TPR);
4431
4432 adapter->stats.ptc64 += rd32(E1000_PTC64);
4433 adapter->stats.ptc127 += rd32(E1000_PTC127);
4434 adapter->stats.ptc255 += rd32(E1000_PTC255);
4435 adapter->stats.ptc511 += rd32(E1000_PTC511);
4436 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4437 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4438
4439 adapter->stats.mptc += rd32(E1000_MPTC);
4440 adapter->stats.bptc += rd32(E1000_BPTC);
4441
2d0b0f69
NN		 4442 adapter->stats.tpt += rd32(E1000_TPT);
4443 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4444
4445 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4446 /* read internal phy specific stats */
4447 reg = rd32(E1000_CTRL_EXT);
4448 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4449 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4450 adapter->stats.tncrs += rd32(E1000_TNCRS);
4451 }
4452
9d5c8243
AK		 4453 adapter->stats.tsctc += rd32(E1000_TSCTC);
4454 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4455
4456 adapter->stats.iac += rd32(E1000_IAC);
4457 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4458 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4459 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4460 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4461 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4462 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4463 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4464 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4465
4466 /* Fill out the OS statistics structure */
128e45eb
AD		 4467 net_stats->multicast = adapter->stats.mprc;
4468 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4469
4470 /* Rx Errors */
4471
4472 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4473 * our own version based on RUC and ROC */
128e45eb 4474 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4475 adapter->stats.crcerrs + adapter->stats.algnerrc +
4476 adapter->stats.ruc + adapter->stats.roc +
4477 adapter->stats.cexterr;
128e45eb
AD		 4478 net_stats->rx_length_errors = adapter->stats.ruc +
4479 adapter->stats.roc;
4480 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4481 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4482 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4483
4484 /* Tx Errors */
128e45eb
AD		 4485 net_stats->tx_errors = adapter->stats.ecol +
4486 adapter->stats.latecol;
4487 net_stats->tx_aborted_errors = adapter->stats.ecol;
4488 net_stats->tx_window_errors = adapter->stats.latecol;
4489 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4490
4491 /* Tx Dropped needs to be maintained elsewhere */
4492
4493 /* Phy Stats */
4494 if (hw->phy.media_type == e1000_media_type_copper) {
4495 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4496 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4497 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4498 adapter->phy_stats.idle_errors += phy_tmp;
4499 }
4500 }
4501
4502 /* Management Stats */
4503 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4504 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4505 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4506}
4507
9d5c8243
AK		 4508static irqreturn_t igb_msix_other(int irq, void *data)
4509{
047e0030 4510 struct igb_adapter *adapter = data;
9d5c8243 4511 struct e1000_hw *hw = &adapter->hw;
844290e5 4512 u32 icr = rd32(E1000_ICR);
844290e5 4513 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4514
7f081d40
AD		 4515 if (icr & E1000_ICR_DRSTA)
4516 schedule_work(&adapter->reset_task);
4517
047e0030 4518 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4519 /* HW is reporting DMA is out of sync */
4520 adapter->stats.doosync++;
4521 }
eebbbdba 4522
4ae196df
AD		 4523 /* Check for a mailbox event */
4524 if (icr & E1000_ICR_VMMB)
4525 igb_msg_task(adapter);
4526
4527 if (icr & E1000_ICR_LSC) {
4528 hw->mac.get_link_status = 1;
4529 /* guard against interrupt when we're going down */
4530 if (!test_bit(__IGB_DOWN, &adapter->state))
4531 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4532 }
4533
25568a53
AD		 4534 if (adapter->vfs_allocated_count)
4535 wr32(E1000_IMS, E1000_IMS_LSC |
4536 E1000_IMS_VMMB |
4537 E1000_IMS_DOUTSYNC);
4538 else
4539 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4540 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4541
4542 return IRQ_HANDLED;
4543}
4544
047e0030 4545static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4546{
26b39276 4547 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4548 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4549
047e0030
AD		 4550 if (!q_vector->set_itr)
4551 return;
73cd78f1 4552
047e0030
AD		 4553 if (!itr_val)
4554 itr_val = 0x4;
661086df 4555
26b39276
AD		 4556 if (adapter->hw.mac.type == e1000_82575)
4557 itr_val |= itr_val << 16;
661086df 4558 else
047e0030 4559 itr_val |= 0x8000000;
661086df 4560
047e0030
AD		 4561 writel(itr_val, q_vector->itr_register);
4562 q_vector->set_itr = 0;
6eb5a7f1
AD		 4563}
4564
047e0030 4565static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4566{
047e0030 4567 struct igb_q_vector *q_vector = data;
9d5c8243 4568
047e0030
AD		 4569 /* Write the ITR value calculated from the previous interrupt. */
4570 igb_write_itr(q_vector);
9d5c8243 4571
047e0030 4572 napi_schedule(&q_vector->napi);
844290e5 4573
047e0030 4574 return IRQ_HANDLED;
fe4506b6
JC		 4575}
4576
421e02f0 4577#ifdef CONFIG_IGB_DCA
047e0030 4578static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4579{
047e0030 4580 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4581 struct e1000_hw *hw = &adapter->hw;
4582 int cpu = get_cpu();
fe4506b6 4583
047e0030
AD		 4584 if (q_vector->cpu == cpu)
4585 goto out_no_update;
4586
4587 if (q_vector->tx_ring) {
4588 int q = q_vector->tx_ring->reg_idx;
4589 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4590 if (hw->mac.type == e1000_82575) {
4591 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4592 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4593 } else {
047e0030
AD		 4594 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4595 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4596 E1000_DCA_TXCTRL_CPUID_SHIFT;
4597 }
4598 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4599 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4600 }
4601 if (q_vector->rx_ring) {
4602 int q = q_vector->rx_ring->reg_idx;
4603 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4604 if (hw->mac.type == e1000_82575) {
2d064c06 4605 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4606 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4607 } else {
4608 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4609 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4610 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4611 }
fe4506b6
JC		 4612 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4613 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4614 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4615 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4616 }
047e0030
AD		 4617 q_vector->cpu = cpu;
4618out_no_update:
fe4506b6
JC		 4619 put_cpu();
4620}
4621
4622static void igb_setup_dca(struct igb_adapter *adapter)
4623{
7e0e99ef 4624 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4625 int i;
4626
7dfc16fa 4627 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4628 return;
4629
7e0e99ef
AD		 4630 /* Always use CB2 mode, difference is masked in the CB driver. */
4631 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4632
047e0030 4633 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4634 adapter->q_vector[i]->cpu = -1;
4635 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4636 }
4637}
4638
4639static int __igb_notify_dca(struct device *dev, void *data)
4640{
4641 struct net_device *netdev = dev_get_drvdata(dev);
4642 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4643 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4644 struct e1000_hw *hw = &adapter->hw;
4645 unsigned long event = *(unsigned long *)data;
4646
4647 switch (event) {
4648 case DCA_PROVIDER_ADD:
4649 /* if already enabled, don't do it again */
7dfc16fa 4650 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4651 break;
fe4506b6 4652 if (dca_add_requester(dev) == 0) {
bbd98fe4 4653 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4654 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4655 igb_setup_dca(adapter);
4656 break;
4657 }
4658 /* Fall Through since DCA is disabled. */
4659 case DCA_PROVIDER_REMOVE:
7dfc16fa 4660 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4661 /* without this a class_device is left
047e0030 4662 * hanging around in the sysfs model */
fe4506b6 4663 dca_remove_requester(dev);
090b1795 4664 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4665 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4666 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4667 }
4668 break;
4669 }
bbd98fe4 4670
fe4506b6 4671 return 0;
9d5c8243
AK		 4672}
4673
fe4506b6
JC		 4674static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4675 void *p)
4676{
4677 int ret_val;
4678
4679 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4680 __igb_notify_dca);
4681
4682 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4683}
421e02f0 4684#endif /* CONFIG_IGB_DCA */
9d5c8243 4685
4ae196df
AD		 4686static void igb_ping_all_vfs(struct igb_adapter *adapter)
4687{
4688 struct e1000_hw *hw = &adapter->hw;
4689 u32 ping;
4690 int i;
4691
4692 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4693 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4694 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4695 ping |= E1000_VT_MSGTYPE_CTS;
4696 igb_write_mbx(hw, &ping, 1, i);
4697 }
4698}
4699
7d5753f0
AD		 4700static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4701{
4702 struct e1000_hw *hw = &adapter->hw;
4703 u32 vmolr = rd32(E1000_VMOLR(vf));
4704 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4705
d85b9004 4706 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4707 IGB_VF_FLAG_MULTI_PROMISC);
4708 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4709
4710 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4711 vmolr |= E1000_VMOLR_MPME;
d85b9004 4712 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4713 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4714 } else {
4715 /*
4716 * if we have hashes and we are clearing a multicast promisc
4717 * flag we need to write the hashes to the MTA as this step
4718 * was previously skipped
4719 */
4720 if (vf_data->num_vf_mc_hashes > 30) {
4721 vmolr |= E1000_VMOLR_MPME;
4722 } else if (vf_data->num_vf_mc_hashes) {
4723 int j;
4724 vmolr |= E1000_VMOLR_ROMPE;
4725 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4726 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4727 }
4728 }
4729
4730 wr32(E1000_VMOLR(vf), vmolr);
4731
4732 /* there are flags left unprocessed, likely not supported */
4733 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4734 return -EINVAL;
4735
4736 return 0;
4737
4738}
4739
4ae196df
AD		 4740static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4741 u32 *msgbuf, u32 vf)
4742{
4743 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4744 u16 *hash_list = (u16 *)&msgbuf[1];
4745 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4746 int i;
4747
7d5753f0 4748 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4749 * to this VF for later use to restore when the PF multi cast
4750 * list changes
4751 */
4752 vf_data->num_vf_mc_hashes = n;
4753
7d5753f0
AD		 4754 /* only up to 30 hash values supported */
4755 if (n > 30)
4756 n = 30;
4757
4758 /* store the hashes for later use */
4ae196df 4759 for (i = 0; i < n; i++)
a419aef8 4760 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4761
4762 /* Flush and reset the mta with the new values */
ff41f8dc 4763 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4764
4765 return 0;
4766}
4767
4768static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4769{
4770 struct e1000_hw *hw = &adapter->hw;
4771 struct vf_data_storage *vf_data;
4772 int i, j;
4773
4774 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4775 u32 vmolr = rd32(E1000_VMOLR(i));
4776 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4777
4ae196df 4778 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4779
4780 if ((vf_data->num_vf_mc_hashes > 30) ||
4781 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4782 vmolr |= E1000_VMOLR_MPME;
4783 } else if (vf_data->num_vf_mc_hashes) {
4784 vmolr |= E1000_VMOLR_ROMPE;
4785 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4786 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4787 }
4788 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4789 }
4790}
4791
4792static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4793{
4794 struct e1000_hw *hw = &adapter->hw;
4795 u32 pool_mask, reg, vid;
4796 int i;
4797
4798 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4799
4800 /* Find the vlan filter for this id */
4801 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4802 reg = rd32(E1000_VLVF(i));
4803
4804 /* remove the vf from the pool */
4805 reg &= ~pool_mask;
4806
4807 /* if pool is empty then remove entry from vfta */
4808 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4809 (reg & E1000_VLVF_VLANID_ENABLE)) {
4810 reg = 0;
4811 vid = reg & E1000_VLVF_VLANID_MASK;
4812 igb_vfta_set(hw, vid, false);
4813 }
4814
4815 wr32(E1000_VLVF(i), reg);
4816 }
ae641bdc
AD		 4817
4818 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4819}
4820
4821static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4822{
4823 struct e1000_hw *hw = &adapter->hw;
4824 u32 reg, i;
4825
51466239
AD		 4826 /* The vlvf table only exists on 82576 hardware and newer */
4827 if (hw->mac.type < e1000_82576)
4828 return -1;
4829
4830 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4831 if (!adapter->vfs_allocated_count)
4832 return -1;
4833
4834 /* Find the vlan filter for this id */
4835 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4836 reg = rd32(E1000_VLVF(i));
4837 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4838 vid == (reg & E1000_VLVF_VLANID_MASK))
4839 break;
4840 }
4841
4842 if (add) {
4843 if (i == E1000_VLVF_ARRAY_SIZE) {
4844 /* Did not find a matching VLAN ID entry that was
4845 * enabled. Search for a free filter entry, i.e.
4846 * one without the enable bit set
4847 */
4848 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4849 reg = rd32(E1000_VLVF(i));
4850 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4851 break;
4852 }
4853 }
4854 if (i < E1000_VLVF_ARRAY_SIZE) {
4855 /* Found an enabled/available entry */
4856 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4857
4858 /* if !enabled we need to set this up in vfta */
4859 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4860 /* add VID to filter table */
4861 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4862 reg |= E1000_VLVF_VLANID_ENABLE;
4863 }
cad6d05f
AD		 4864 reg &= ~E1000_VLVF_VLANID_MASK;
4865 reg |= vid;
4ae196df 4866 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4867
4868 /* do not modify RLPML for PF devices */
4869 if (vf >= adapter->vfs_allocated_count)
4870 return 0;
4871
4872 if (!adapter->vf_data[vf].vlans_enabled) {
4873 u32 size;
4874 reg = rd32(E1000_VMOLR(vf));
4875 size = reg & E1000_VMOLR_RLPML_MASK;
4876 size += 4;
4877 reg &= ~E1000_VMOLR_RLPML_MASK;
4878 reg |= size;
4879 wr32(E1000_VMOLR(vf), reg);
4880 }
ae641bdc 4881
51466239 4882 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4883 return 0;
4884 }
4885 } else {
4886 if (i < E1000_VLVF_ARRAY_SIZE) {
4887 /* remove vf from the pool */
4888 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4889 /* if pool is empty then remove entry from vfta */
4890 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4891 reg = 0;
4892 igb_vfta_set(hw, vid, false);
4893 }
4894 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4895
4896 /* do not modify RLPML for PF devices */
4897 if (vf >= adapter->vfs_allocated_count)
4898 return 0;
4899
4900 adapter->vf_data[vf].vlans_enabled--;
4901 if (!adapter->vf_data[vf].vlans_enabled) {
4902 u32 size;
4903 reg = rd32(E1000_VMOLR(vf));
4904 size = reg & E1000_VMOLR_RLPML_MASK;
4905 size -= 4;
4906 reg &= ~E1000_VMOLR_RLPML_MASK;
4907 reg |= size;
4908 wr32(E1000_VMOLR(vf), reg);
4909 }
4ae196df
AD		 4910 }
4911 }
8151d294
WM		 4912 return 0;
4913}
4914
4915static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4916{
4917 struct e1000_hw *hw = &adapter->hw;
4918
4919 if (vid)
4920 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4921 else
4922 wr32(E1000_VMVIR(vf), 0);
4923}
4924
4925static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4926 int vf, u16 vlan, u8 qos)
4927{
4928 int err = 0;
4929 struct igb_adapter *adapter = netdev_priv(netdev);
4930
4931 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4932 return -EINVAL;
4933 if (vlan || qos) {
4934 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4935 if (err)
4936 goto out;
4937 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4938 igb_set_vmolr(adapter, vf, !vlan);
4939 adapter->vf_data[vf].pf_vlan = vlan;
4940 adapter->vf_data[vf].pf_qos = qos;
4941 dev_info(&adapter->pdev->dev,
4942 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4943 if (test_bit(__IGB_DOWN, &adapter->state)) {
4944 dev_warn(&adapter->pdev->dev,
4945 "The VF VLAN has been set,"
4946 " but the PF device is not up.\n");
4947 dev_warn(&adapter->pdev->dev,
4948 "Bring the PF device up before"
4949 " attempting to use the VF device.\n");
4950 }
4951 } else {
4952 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4953 false, vf);
4954 igb_set_vmvir(adapter, vlan, vf);
4955 igb_set_vmolr(adapter, vf, true);
4956 adapter->vf_data[vf].pf_vlan = 0;
4957 adapter->vf_data[vf].pf_qos = 0;
4958 }
4959out:
4960 return err;
4ae196df
AD		 4961}
4962
4963static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4964{
4965 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4966 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4967
4968 return igb_vlvf_set(adapter, vid, add, vf);
4969}
4970
f2ca0dbe 4971static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4972{
8151d294
WM		 4973 /* clear flags */
4974 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4975 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4976
4977 /* reset offloads to defaults */
8151d294 4978 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4979
4980 /* reset vlans for device */
4981 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4982 if (adapter->vf_data[vf].pf_vlan)
4983 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4984 adapter->vf_data[vf].pf_vlan,
4985 adapter->vf_data[vf].pf_qos);
4986 else
4987 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4988
4989 /* reset multicast table array for vf */
4990 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4991
4992 /* Flush and reset the mta with the new values */
ff41f8dc 4993 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4994}
4995
f2ca0dbe
AD		 4996static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4997{
4998 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4999
5000 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5001 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5002 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5003
5004 /* process remaining reset events */
5005 igb_vf_reset(adapter, vf);
5006}
5007
5008static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5009{
5010 struct e1000_hw *hw = &adapter->hw;
5011 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5012 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5013 u32 reg, msgbuf[3];
5014 u8 *addr = (u8 *)(&msgbuf[1]);
5015
5016 /* process all the same items cleared in a function level reset */
f2ca0dbe 5017 igb_vf_reset(adapter, vf);
4ae196df
AD		 5018
5019 /* set vf mac address */
26ad9178 5020 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5021
5022 /* enable transmit and receive for vf */
5023 reg = rd32(E1000_VFTE);
5024 wr32(E1000_VFTE, reg | (1 << vf));
5025 reg = rd32(E1000_VFRE);
5026 wr32(E1000_VFRE, reg | (1 << vf));
5027
f2ca0dbe 5028 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 5029
5030 /* reply to reset with ack and vf mac address */
5031 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5032 memcpy(addr, vf_mac, 6);
5033 igb_write_mbx(hw, msgbuf, 3, vf);
5034}
5035
5036static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5037{
de42edde
GR		 5038 /*
5039 * The VF MAC Address is stored in a packed array of bytes
5040 * starting at the second 32 bit word of the msg array
5041 */
f2ca0dbe
AD		 5042 unsigned char *addr = (char *)&msg[1];
5043 int err = -1;
4ae196df 5044
f2ca0dbe
AD		 5045 if (is_valid_ether_addr(addr))
5046 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5047
f2ca0dbe 5048 return err;
4ae196df
AD		 5049}
5050
5051static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5052{
5053 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5054 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5055 u32 msg = E1000_VT_MSGTYPE_NACK;
5056
5057 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5058 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5059 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5060 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5061 vf_data->last_nack = jiffies;
4ae196df
AD		 5062 }
5063}
5064
f2ca0dbe 5065static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5066{
f2ca0dbe
AD		 5067 struct pci_dev *pdev = adapter->pdev;
5068 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5069 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5070 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5071 s32 retval;
5072
f2ca0dbe 5073 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5074
fef45f4c
AD		 5075 if (retval) {
5076 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5077 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5078 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5079 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5080 return;
5081 goto out;
5082 }
4ae196df
AD		 5083
5084 /* this is a message we already processed, do nothing */
5085 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5086 return;
4ae196df
AD		 5087
5088 /*
5089 * until the vf completes a reset it should not be
5090 * allowed to start any configuration.
5091 */
5092
5093 if (msgbuf[0] == E1000_VF_RESET) {
5094 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5095 return;
4ae196df
AD		 5096 }
5097
f2ca0dbe 5098 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5099 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5100 return;
5101 retval = -1;
5102 goto out;
4ae196df
AD		 5103 }
5104
5105 switch ((msgbuf[0] & 0xFFFF)) {
5106 case E1000_VF_SET_MAC_ADDR:
5107 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5108 break;
7d5753f0
AD		 5109 case E1000_VF_SET_PROMISC:
5110 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5111 break;
4ae196df
AD		 5112 case E1000_VF_SET_MULTICAST:
5113 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5114 break;
5115 case E1000_VF_SET_LPE:
5116 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5117 break;
5118 case E1000_VF_SET_VLAN:
8151d294
WM		 5119 if (adapter->vf_data[vf].pf_vlan)
5120 retval = -1;
5121 else
5122 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5123 break;
5124 default:
090b1795 5125 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5126 retval = -1;
5127 break;
5128 }
5129
fef45f4c
AD		 5130 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5131out:
4ae196df
AD		 5132 /* notify the VF of the results of what it sent us */
5133 if (retval)
5134 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5135 else
5136 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5137
4ae196df 5138 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5139}
4ae196df 5140
f2ca0dbe
AD		 5141static void igb_msg_task(struct igb_adapter *adapter)
5142{
5143 struct e1000_hw *hw = &adapter->hw;
5144 u32 vf;
5145
5146 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5147 /* process any reset requests */
5148 if (!igb_check_for_rst(hw, vf))
5149 igb_vf_reset_event(adapter, vf);
5150
5151 /* process any messages pending */
5152 if (!igb_check_for_msg(hw, vf))
5153 igb_rcv_msg_from_vf(adapter, vf);
5154
5155 /* process any acks */
5156 if (!igb_check_for_ack(hw, vf))
5157 igb_rcv_ack_from_vf(adapter, vf);
5158 }
4ae196df
AD		 5159}
5160
68d480c4
AD		 5161/**
5162 * igb_set_uta - Set unicast filter table address
5163 * @adapter: board private structure
5164 *
5165 * The unicast table address is a register array of 32-bit registers.
5166 * The table is meant to be used in a way similar to how the MTA is used
5167 * however due to certain limitations in the hardware it is necessary to
5168 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5169 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5170 **/
5171static void igb_set_uta(struct igb_adapter *adapter)
5172{
5173 struct e1000_hw *hw = &adapter->hw;
5174 int i;
5175
5176 /* The UTA table only exists on 82576 hardware and newer */
5177 if (hw->mac.type < e1000_82576)
5178 return;
5179
5180 /* we only need to do this if VMDq is enabled */
5181 if (!adapter->vfs_allocated_count)
5182 return;
5183
5184 for (i = 0; i < hw->mac.uta_reg_count; i++)
5185 array_wr32(E1000_UTA, i, ~0);
5186}
5187
9d5c8243
AK		 5188/**
5189 * igb_intr_msi - Interrupt Handler
5190 * @irq: interrupt number
5191 * @data: pointer to a network interface device structure
5192 **/
5193static irqreturn_t igb_intr_msi(int irq, void *data)
5194{
047e0030
AD		 5195 struct igb_adapter *adapter = data;
5196 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5197 struct e1000_hw *hw = &adapter->hw;
5198 /* read ICR disables interrupts using IAM */
5199 u32 icr = rd32(E1000_ICR);
5200
047e0030 5201 igb_write_itr(q_vector);
9d5c8243 5202
7f081d40
AD		 5203 if (icr & E1000_ICR_DRSTA)
5204 schedule_work(&adapter->reset_task);
5205
047e0030 5206 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5207 /* HW is reporting DMA is out of sync */
5208 adapter->stats.doosync++;
5209 }
5210
9d5c8243
AK		 5211 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5212 hw->mac.get_link_status = 1;
5213 if (!test_bit(__IGB_DOWN, &adapter->state))
5214 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5215 }
5216
047e0030 5217 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5218
5219 return IRQ_HANDLED;
5220}
5221
5222/**
4a3c6433 5223 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5224 * @irq: interrupt number
5225 * @data: pointer to a network interface device structure
5226 **/
5227static irqreturn_t igb_intr(int irq, void *data)
5228{
047e0030
AD		 5229 struct igb_adapter *adapter = data;
5230 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5231 struct e1000_hw *hw = &adapter->hw;
5232 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5233 * need for the IMC write */
5234 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5235 if (!icr)
5236 return IRQ_NONE; /* Not our interrupt */
5237
047e0030 5238 igb_write_itr(q_vector);
9d5c8243
AK		 5239
5240 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5241 * not set, then the adapter didn't send an interrupt */
5242 if (!(icr & E1000_ICR_INT_ASSERTED))
5243 return IRQ_NONE;
5244
7f081d40
AD		 5245 if (icr & E1000_ICR_DRSTA)
5246 schedule_work(&adapter->reset_task);
5247
047e0030 5248 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5249 /* HW is reporting DMA is out of sync */
5250 adapter->stats.doosync++;
5251 }
5252
9d5c8243
AK		 5253 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5254 hw->mac.get_link_status = 1;
5255 /* guard against interrupt when we're going down */
5256 if (!test_bit(__IGB_DOWN, &adapter->state))
5257 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5258 }
5259
047e0030 5260 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5261
5262 return IRQ_HANDLED;
5263}
5264
047e0030 5265static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5266{
047e0030 5267 struct igb_adapter *adapter = q_vector->adapter;
46544258 5268 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5269
4fc82adf
AD		 5270 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5271 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5272 if (!adapter->msix_entries)
6eb5a7f1 5273 igb_set_itr(adapter);
46544258 5274 else
047e0030 5275 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5276 }
5277
46544258
AD		 5278 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5279 if (adapter->msix_entries)
047e0030 5280 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5281 else
5282 igb_irq_enable(adapter);
5283 }
9d5c8243
AK		 5284}
5285
46544258
AD		 5286/**
5287 * igb_poll - NAPI Rx polling callback
5288 * @napi: napi polling structure
5289 * @budget: count of how many packets we should handle
5290 **/
5291static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5292{
047e0030
AD		 5293 struct igb_q_vector *q_vector = container_of(napi,
5294 struct igb_q_vector,
5295 napi);
5296 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5297
421e02f0 5298#ifdef CONFIG_IGB_DCA
047e0030
AD		 5299 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5300 igb_update_dca(q_vector);
fe4506b6 5301#endif
047e0030
AD		 5302 if (q_vector->tx_ring)
5303 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5304
047e0030
AD		 5305 if (q_vector->rx_ring)
5306 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5307
5308 if (!tx_clean_complete)
5309 work_done = budget;
46544258 5310
9d5c8243 5311 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5312 if (work_done < budget) {
288379f0 5313 napi_complete(napi);
047e0030 5314 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5315 }
5316
46544258 5317 return work_done;
9d5c8243 5318}
6d8126f9 5319
33af6bcc 5320/**
c5b9bd5e 5321 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5322 * @adapter: board private structure
c5b9bd5e
AD		 5323 * @shhwtstamps: timestamp structure to update
5324 * @regval: unsigned 64bit system time value.
5325 *
5326 * We need to convert the system time value stored in the RX/TXSTMP registers
5327 * into a hwtstamp which can be used by the upper level timestamping functions
5328 */
5329static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5330 struct skb_shared_hwtstamps *shhwtstamps,
5331 u64 regval)
5332{
5333 u64 ns;
5334
55cac248
AD		 5335 /*
5336 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5337 * 24 to match clock shift we setup earlier.
5338 */
5339 if (adapter->hw.mac.type == e1000_82580)
5340 regval <<= IGB_82580_TSYNC_SHIFT;
5341
c5b9bd5e
AD		 5342 ns = timecounter_cyc2time(&adapter->clock, regval);
5343 timecompare_update(&adapter->compare, ns);
5344 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5345 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5346 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5347}
5348
5349/**
5350 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5351 * @q_vector: pointer to q_vector containing needed info
2873957d 5352 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5353 *
5354 * If we were asked to do hardware stamping and such a time stamp is
5355 * available, then it must have been for this skb here because we only
5356 * allow only one such packet into the queue.
5357 */
2873957d 5358static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5359{
c5b9bd5e 5360 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5361 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5362 struct skb_shared_hwtstamps shhwtstamps;
5363 u64 regval;
33af6bcc 5364
c5b9bd5e 5365 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5366 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5367 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5368 return;
5369
5370 regval = rd32(E1000_TXSTMPL);
5371 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5372
5373 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5374 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5375}
5376
9d5c8243
AK		 5377/**
5378 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5379 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5380 * returns true if ring is completely cleaned
5381 **/
047e0030 5382static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5383{
047e0030
AD		 5384 struct igb_adapter *adapter = q_vector->adapter;
5385 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5386 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5387 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5388 struct igb_buffer *buffer_info;
0e014cb1 5389 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5390 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5391 unsigned int i, eop, count = 0;
5392 bool cleaned = false;
9d5c8243 5393
9d5c8243 5394 i = tx_ring->next_to_clean;
0e014cb1
AD		 5395 eop = tx_ring->buffer_info[i].next_to_watch;
5396 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5397
5398 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5399 (count < tx_ring->count)) {
2d0bb1c1 5400 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5401 for (cleaned = false; !cleaned; count++) {
5402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5403 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5404 cleaned = (i == eop);
9d5c8243 5405
2873957d
NN		 5406 if (buffer_info->skb) {
5407 total_bytes += buffer_info->bytecount;
9d5c8243 5408 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5409 total_packets += buffer_info->gso_segs;
5410 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5411 }
5412
80785298 5413 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5414 tx_desc->wb.status = 0;
9d5c8243
AK		 5415
5416 i++;
5417 if (i == tx_ring->count)
5418 i = 0;
9d5c8243 5419 }
0e014cb1
AD		 5420 eop = tx_ring->buffer_info[i].next_to_watch;
5421 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5422 }
5423
9d5c8243
AK		 5424 tx_ring->next_to_clean = i;
5425
fc7d345d 5426 if (unlikely(count &&
9d5c8243 5427 netif_carrier_ok(netdev) &&
c493ea45 5428 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5429 /* Make sure that anybody stopping the queue after this
5430 * sees the new next_to_clean.
5431 */
5432 smp_mb();
661086df
PWJ		 5433 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5434 !(test_bit(__IGB_DOWN, &adapter->state))) {
5435 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 5436
5437 u64_stats_update_begin(&tx_ring->tx_syncp);
04a5fcaa 5438 tx_ring->tx_stats.restart_queue++;
12dcd86b 5439 u64_stats_update_end(&tx_ring->tx_syncp);
661086df 5440 }
9d5c8243
AK		 5441 }
5442
5443 if (tx_ring->detect_tx_hung) {
5444 /* Detect a transmit hang in hardware, this serializes the
5445 * check with the clearing of time_stamp and movement of i */
5446 tx_ring->detect_tx_hung = false;
5447 if (tx_ring->buffer_info[i].time_stamp &&
5448 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5449 (adapter->tx_timeout_factor * HZ)) &&
5450 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5451
9d5c8243 5452 /* detected Tx unit hang */
59d71989 5453 dev_err(tx_ring->dev,
9d5c8243 5454 "Detected Tx Unit Hang\n"
2d064c06 5455 " Tx Queue <%d>\n"
9d5c8243
AK		 5456 " TDH <%x>\n"
5457 " TDT <%x>\n"
5458 " next_to_use <%x>\n"
5459 " next_to_clean <%x>\n"
9d5c8243
AK		 5460 "buffer_info[next_to_clean]\n"
5461 " time_stamp <%lx>\n"
0e014cb1 5462 " next_to_watch <%x>\n"
9d5c8243
AK		 5463 " jiffies <%lx>\n"
5464 " desc.status <%x>\n",
2d064c06 5465 tx_ring->queue_index,
fce99e34
AD		 5466 readl(tx_ring->head),
5467 readl(tx_ring->tail),
9d5c8243
AK		 5468 tx_ring->next_to_use,
5469 tx_ring->next_to_clean,
f7ba205e 5470 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5471 eop,
9d5c8243 5472 jiffies,
0e014cb1 5473 eop_desc->wb.status);
661086df 5474 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5475 }
5476 }
5477 tx_ring->total_bytes += total_bytes;
5478 tx_ring->total_packets += total_packets;
12dcd86b 5479 u64_stats_update_begin(&tx_ring->tx_syncp);
e21ed353
AD		 5480 tx_ring->tx_stats.bytes += total_bytes;
5481 tx_ring->tx_stats.packets += total_packets;
12dcd86b 5482 u64_stats_update_end(&tx_ring->tx_syncp);
807540ba 5483 return count < tx_ring->count;
9d5c8243
AK		 5484}
5485
9d5c8243
AK		 5486/**
5487 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5488 * @q_vector: structure containing interrupt and ring information
5489 * @skb: packet to send up
5490 * @vlan_tag: vlan tag for packet
9d5c8243 5491 **/
047e0030
AD		 5492static void igb_receive_skb(struct igb_q_vector *q_vector,
5493 struct sk_buff *skb,
5494 u16 vlan_tag)
5495{
5496 struct igb_adapter *adapter = q_vector->adapter;
5497
31b24b95 5498 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5499 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5500 vlan_tag, skb);
182ff8df 5501 else
047e0030 5502 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5503}
5504
04a5fcaa 5505static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5506 u32 status_err, struct sk_buff *skb)
5507{
bc8acf2c 5508 skb_checksum_none_assert(skb);
9d5c8243
AK		 5509
5510 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5511 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5512 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5513 return;
85ad76b2 5514
9d5c8243
AK		 5515 /* TCP/UDP checksum error bit is set */
5516 if (status_err &
5517 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5518 /*
5519 * work around errata with sctp packets where the TCPE aka
5520 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5521 * packets, (aka let the stack check the crc32c)
5522 */
85ad76b2 5523 if ((skb->len == 60) &&
12dcd86b
ED		 5524 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5525 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5526 ring->rx_stats.csum_err++;
12dcd86b
ED		 5527 u64_stats_update_end(&ring->rx_syncp);
5528 }
9d5c8243 5529 /* let the stack verify checksum errors */
9d5c8243
AK		 5530 return;
5531 }
5532 /* It must be a TCP or UDP packet with a valid checksum */
5533 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5534 skb->ip_summed = CHECKSUM_UNNECESSARY;
5535
59d71989 5536 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5537}
5538
757b77e2 5539static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5540 struct sk_buff *skb)
5541{
5542 struct igb_adapter *adapter = q_vector->adapter;
5543 struct e1000_hw *hw = &adapter->hw;
5544 u64 regval;
5545
5546 /*
5547 * If this bit is set, then the RX registers contain the time stamp. No
5548 * other packet will be time stamped until we read these registers, so
5549 * read the registers to make them available again. Because only one
5550 * packet can be time stamped at a time, we know that the register
5551 * values must belong to this one here and therefore we don't need to
5552 * compare any of the additional attributes stored for it.
5553 *
2244d07b 5554 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5555 * can turn into a skb_shared_hwtstamps.
5556 */
757b77e2
NN		 5557 if (staterr & E1000_RXDADV_STAT_TSIP) {
5558 u32 *stamp = (u32 *)skb->data;
5559 regval = le32_to_cpu(*(stamp + 2));
5560 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5561 skb_pull(skb, IGB_TS_HDR_LEN);
5562 } else {
5563 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5564 return;
c5b9bd5e 5565
757b77e2
NN		 5566 regval = rd32(E1000_RXSTMPL);
5567 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5568 }
c5b9bd5e
AD		 5569
5570 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5571}
4c844851 5572static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5573 union e1000_adv_rx_desc *rx_desc)
5574{
5575 /* HW will not DMA in data larger than the given buffer, even if it
5576 * parses the (NFS, of course) header to be larger. In that case, it
5577 * fills the header buffer and spills the rest into the page.
5578 */
5579 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5580 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5581 if (hlen > rx_ring->rx_buffer_len)
5582 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5583 return hlen;
5584}
5585
047e0030
AD		 5586static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5587 int *work_done, int budget)
9d5c8243 5588{
047e0030 5589 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5590 struct net_device *netdev = rx_ring->netdev;
59d71989 5591 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5592 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5593 struct igb_buffer *buffer_info , *next_buffer;
5594 struct sk_buff *skb;
9d5c8243
AK		 5595 bool cleaned = false;
5596 int cleaned_count = 0;
d1eff350 5597 int current_node = numa_node_id();
9d5c8243 5598 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5599 unsigned int i;
2d94d8ab
AD		 5600 u32 staterr;
5601 u16 length;
047e0030 5602 u16 vlan_tag;
9d5c8243
AK		 5603
5604 i = rx_ring->next_to_clean;
69d3ca53 5605 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5606 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5607 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5608
5609 while (staterr & E1000_RXD_STAT_DD) {
5610 if (*work_done >= budget)
5611 break;
5612 (*work_done)++;
2d0bb1c1 5613 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5614
69d3ca53
AD		 5615 skb = buffer_info->skb;
5616 prefetch(skb->data - NET_IP_ALIGN);
5617 buffer_info->skb = NULL;
5618
5619 i++;
5620 if (i == rx_ring->count)
5621 i = 0;
42d0781a 5622
69d3ca53
AD		 5623 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5624 prefetch(next_rxd);
5625 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5626
5627 length = le16_to_cpu(rx_desc->wb.upper.length);
5628 cleaned = true;
5629 cleaned_count++;
5630
2d94d8ab 5631 if (buffer_info->dma) {
59d71989 5632 dma_unmap_single(dev, buffer_info->dma,
4c844851 5633 rx_ring->rx_buffer_len,
59d71989 5634 DMA_FROM_DEVICE);
91615f76 5635 buffer_info->dma = 0;
4c844851 5636 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5637 skb_put(skb, length);
5638 goto send_up;
5639 }
4c844851 5640 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5641 }
5642
5643 if (length) {
59d71989
AD		 5644 dma_unmap_page(dev, buffer_info->page_dma,
5645 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5646 buffer_info->page_dma = 0;
bf36c1a0 5647
aa913403 5648 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5649 buffer_info->page,
5650 buffer_info->page_offset,
5651 length);
5652
d1eff350
AD		 5653 if ((page_count(buffer_info->page) != 1) ||
5654 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5655 buffer_info->page = NULL;
5656 else
5657 get_page(buffer_info->page);
9d5c8243
AK		 5658
5659 skb->len += length;
5660 skb->data_len += length;
bf36c1a0 5661 skb->truesize += length;
9d5c8243 5662 }
9d5c8243 5663
bf36c1a0 5664 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5665 buffer_info->skb = next_buffer->skb;
5666 buffer_info->dma = next_buffer->dma;
5667 next_buffer->skb = skb;
5668 next_buffer->dma = 0;
bf36c1a0
AD		 5669 goto next_desc;
5670 }
69d3ca53 5671send_up:
9d5c8243
AK		 5672 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5673 dev_kfree_skb_irq(skb);
5674 goto next_desc;
5675 }
9d5c8243 5676
757b77e2
NN		 5677 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5678 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5679 total_bytes += skb->len;
5680 total_packets++;
5681
85ad76b2 5682 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5683
5684 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5685 skb_record_rx_queue(skb, rx_ring->queue_index);
5686
5687 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5688 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5689
047e0030 5690 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5691
9d5c8243
AK		 5692next_desc:
5693 rx_desc->wb.upper.status_error = 0;
5694
5695 /* return some buffers to hardware, one at a time is too slow */
5696 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5697 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5698 cleaned_count = 0;
5699 }
5700
5701 /* use prefetched values */
5702 rx_desc = next_rxd;
5703 buffer_info = next_buffer;
9d5c8243
AK		 5704 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5705 }
bf36c1a0 5706
9d5c8243 5707 rx_ring->next_to_clean = i;
c493ea45 5708 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5709
5710 if (cleaned_count)
3b644cf6 5711 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5712
5713 rx_ring->total_packets += total_packets;
5714 rx_ring->total_bytes += total_bytes;
12dcd86b 5715 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5716 rx_ring->rx_stats.packets += total_packets;
5717 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5718 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5719 return cleaned;
5720}
5721
9d5c8243
AK		 5722/**
5723 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5724 * @adapter: address of board private structure
5725 **/
d7ee5b3a 5726void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5727{
e694e964 5728 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5729 union e1000_adv_rx_desc *rx_desc;
5730 struct igb_buffer *buffer_info;
5731 struct sk_buff *skb;
5732 unsigned int i;
db761762 5733 int bufsz;
9d5c8243
AK		 5734
5735 i = rx_ring->next_to_use;
5736 buffer_info = &rx_ring->buffer_info[i];
5737
4c844851 5738 bufsz = rx_ring->rx_buffer_len;
db761762 5739
9d5c8243
AK		 5740 while (cleaned_count--) {
5741 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5742
6ec43fe6 5743 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5744 if (!buffer_info->page) {
42d0781a 5745 buffer_info->page = netdev_alloc_page(netdev);
12dcd86b
ED		 5746 if (unlikely(!buffer_info->page)) {
5747 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5748 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5749 u64_stats_update_end(&rx_ring->rx_syncp);
bf36c1a0
AD		 5750 goto no_buffers;
5751 }
5752 buffer_info->page_offset = 0;
5753 } else {
5754 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5755 }
5756 buffer_info->page_dma =
59d71989 5757 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5758 buffer_info->page_offset,
5759 PAGE_SIZE / 2,
59d71989
AD		 5760 DMA_FROM_DEVICE);
5761 if (dma_mapping_error(rx_ring->dev,
5762 buffer_info->page_dma)) {
42d0781a 5763 buffer_info->page_dma = 0;
12dcd86b 5764 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5765 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5766 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5767 goto no_buffers;
5768 }
9d5c8243
AK		 5769 }
5770
42d0781a
AD		 5771 skb = buffer_info->skb;
5772 if (!skb) {
89d71a66 5773 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
12dcd86b
ED		 5774 if (unlikely(!skb)) {
5775 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5776 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5777 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5778 goto no_buffers;
5779 }
5780
9d5c8243 5781 buffer_info->skb = skb;
42d0781a
AD		 5782 }
5783 if (!buffer_info->dma) {
59d71989 5784 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5785 skb->data,
9d5c8243 5786 bufsz,
59d71989
AD		 5787 DMA_FROM_DEVICE);
5788 if (dma_mapping_error(rx_ring->dev,
5789 buffer_info->dma)) {
42d0781a 5790 buffer_info->dma = 0;
12dcd86b 5791 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5792 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5793 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5794 goto no_buffers;
5795 }
9d5c8243
AK		 5796 }
5797 /* Refresh the desc even if buffer_addrs didn't change because
5798 * each write-back erases this info. */
6ec43fe6 5799 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5800 rx_desc->read.pkt_addr =
5801 cpu_to_le64(buffer_info->page_dma);
5802 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5803 } else {
42d0781a 5804 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5805 rx_desc->read.hdr_addr = 0;
5806 }
5807
5808 i++;
5809 if (i == rx_ring->count)
5810 i = 0;
5811 buffer_info = &rx_ring->buffer_info[i];
5812 }
5813
5814no_buffers:
5815 if (rx_ring->next_to_use != i) {
5816 rx_ring->next_to_use = i;
5817 if (i == 0)
5818 i = (rx_ring->count - 1);
5819 else
5820 i--;
5821
5822 /* Force memory writes to complete before letting h/w
5823 * know there are new descriptors to fetch. (Only
5824 * applicable for weak-ordered memory model archs,
5825 * such as IA-64). */
5826 wmb();
fce99e34 5827 writel(i, rx_ring->tail);
9d5c8243
AK		 5828 }
5829}
5830
5831/**
5832 * igb_mii_ioctl -
5833 * @netdev:
5834 * @ifreq:
5835 * @cmd:
5836 **/
5837static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5838{
5839 struct igb_adapter *adapter = netdev_priv(netdev);
5840 struct mii_ioctl_data *data = if_mii(ifr);
5841
5842 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5843 return -EOPNOTSUPP;
5844
5845 switch (cmd) {
5846 case SIOCGMIIPHY:
5847 data->phy_id = adapter->hw.phy.addr;
5848 break;
5849 case SIOCGMIIREG:
f5f4cf08
AD		 5850 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5851 &data->val_out))
9d5c8243
AK		 5852 return -EIO;
5853 break;
5854 case SIOCSMIIREG:
5855 default:
5856 return -EOPNOTSUPP;
5857 }
5858 return 0;
5859}
5860
c6cb090b
PO		 5861/**
5862 * igb_hwtstamp_ioctl - control hardware time stamping
5863 * @netdev:
5864 * @ifreq:
5865 * @cmd:
5866 *
33af6bcc
PO		 5867 * Outgoing time stamping can be enabled and disabled. Play nice and
5868 * disable it when requested, although it shouldn't case any overhead
5869 * when no packet needs it. At most one packet in the queue may be
5870 * marked for time stamping, otherwise it would be impossible to tell
5871 * for sure to which packet the hardware time stamp belongs.
5872 *
5873 * Incoming time stamping has to be configured via the hardware
5874 * filters. Not all combinations are supported, in particular event
5875 * type has to be specified. Matching the kind of event packet is
5876 * not supported, with the exception of "all V2 events regardless of
5877 * level 2 or 4".
5878 *
c6cb090b
PO		 5879 **/
5880static int igb_hwtstamp_ioctl(struct net_device *netdev,
5881 struct ifreq *ifr, int cmd)
5882{
33af6bcc
PO		 5883 struct igb_adapter *adapter = netdev_priv(netdev);
5884 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5885 struct hwtstamp_config config;
c5b9bd5e
AD		 5886 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5887 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5888 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5889 bool is_l4 = false;
5890 bool is_l2 = false;
33af6bcc 5891 u32 regval;
c6cb090b
PO		 5892
5893 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5894 return -EFAULT;
5895
5896 /* reserved for future extensions */
5897 if (config.flags)
5898 return -EINVAL;
5899
33af6bcc
PO		 5900 switch (config.tx_type) {
5901 case HWTSTAMP_TX_OFF:
c5b9bd5e 5902 tsync_tx_ctl = 0;
33af6bcc 5903 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5904 break;
5905 default:
5906 return -ERANGE;
5907 }
5908
5909 switch (config.rx_filter) {
5910 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5911 tsync_rx_ctl = 0;
33af6bcc
PO		 5912 break;
5913 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5914 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5915 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5916 case HWTSTAMP_FILTER_ALL:
5917 /*
5918 * register TSYNCRXCFG must be set, therefore it is not
5919 * possible to time stamp both Sync and Delay_Req messages
5920 * => fall back to time stamping all packets
5921 */
c5b9bd5e 5922 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5923 config.rx_filter = HWTSTAMP_FILTER_ALL;
5924 break;
5925 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5926 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5927 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5928 is_l4 = true;
33af6bcc
PO		 5929 break;
5930 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5931 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5932 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5933 is_l4 = true;
33af6bcc
PO		 5934 break;
5935 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5936 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5937 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5938 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5939 is_l2 = true;
5940 is_l4 = true;
33af6bcc
PO		 5941 config.rx_filter = HWTSTAMP_FILTER_SOME;
5942 break;
5943 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5944 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5945 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5946 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5947 is_l2 = true;
5948 is_l4 = true;
33af6bcc
PO		 5949 config.rx_filter = HWTSTAMP_FILTER_SOME;
5950 break;
5951 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5952 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5953 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5954 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5955 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5956 is_l2 = true;
33af6bcc
PO		 5957 break;
5958 default:
5959 return -ERANGE;
5960 }
5961
c5b9bd5e
AD		 5962 if (hw->mac.type == e1000_82575) {
5963 if (tsync_rx_ctl | tsync_tx_ctl)
5964 return -EINVAL;
5965 return 0;
5966 }
5967
757b77e2
NN		 5968 /*
5969 * Per-packet timestamping only works if all packets are
5970 * timestamped, so enable timestamping in all packets as
5971 * long as one rx filter was configured.
5972 */
5973 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5974 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5975 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5976 }
5977
33af6bcc
PO		 5978 /* enable/disable TX */
5979 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5980 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5981 regval |= tsync_tx_ctl;
33af6bcc
PO		 5982 wr32(E1000_TSYNCTXCTL, regval);
5983
c5b9bd5e 5984 /* enable/disable RX */
33af6bcc 5985 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5986 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5987 regval |= tsync_rx_ctl;
33af6bcc 5988 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5989
c5b9bd5e
AD		 5990 /* define which PTP packets are time stamped */
5991 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5992
c5b9bd5e
AD		 5993 /* define ethertype filter for timestamped packets */
5994 if (is_l2)
5995 wr32(E1000_ETQF(3),
5996 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5997 E1000_ETQF_1588 | /* enable timestamping */
5998 ETH_P_1588)); /* 1588 eth protocol type */
5999 else
6000 wr32(E1000_ETQF(3), 0);
6001
6002#define PTP_PORT 319
6003 /* L4 Queue Filter[3]: filter by destination port and protocol */
6004 if (is_l4) {
6005 u32 ftqf = (IPPROTO_UDP /* UDP */
6006 | E1000_FTQF_VF_BP /* VF not compared */
6007 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6008 | E1000_FTQF_MASK); /* mask all inputs */
6009 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6010
6011 wr32(E1000_IMIR(3), htons(PTP_PORT));
6012 wr32(E1000_IMIREXT(3),
6013 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6014 if (hw->mac.type == e1000_82576) {
6015 /* enable source port check */
6016 wr32(E1000_SPQF(3), htons(PTP_PORT));
6017 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6018 }
6019 wr32(E1000_FTQF(3), ftqf);
6020 } else {
6021 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6022 }
33af6bcc
PO		 6023 wrfl();
6024
6025 adapter->hwtstamp_config = config;
6026
6027 /* clear TX/RX time stamp registers, just to be sure */
6028 regval = rd32(E1000_TXSTMPH);
6029 regval = rd32(E1000_RXSTMPH);
c6cb090b 6030
33af6bcc
PO		 6031 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6032 -EFAULT : 0;
c6cb090b
PO		 6033}
6034
9d5c8243
AK		 6035/**
6036 * igb_ioctl -
6037 * @netdev:
6038 * @ifreq:
6039 * @cmd:
6040 **/
6041static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6042{
6043 switch (cmd) {
6044 case SIOCGMIIPHY:
6045 case SIOCGMIIREG:
6046 case SIOCSMIIREG:
6047 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6048 case SIOCSHWTSTAMP:
6049 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6050 default:
6051 return -EOPNOTSUPP;
6052 }
6053}
6054
009bc06e
AD		 6055s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6056{
6057 struct igb_adapter *adapter = hw->back;
6058 u16 cap_offset;
6059
6060 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6061 if (!cap_offset)
6062 return -E1000_ERR_CONFIG;
6063
6064 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6065
6066 return 0;
6067}
6068
6069s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6070{
6071 struct igb_adapter *adapter = hw->back;
6072 u16 cap_offset;
6073
6074 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6075 if (!cap_offset)
6076 return -E1000_ERR_CONFIG;
6077
6078 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6079
6080 return 0;
6081}
6082
9d5c8243
AK		 6083static void igb_vlan_rx_register(struct net_device *netdev,
6084 struct vlan_group *grp)
6085{
6086 struct igb_adapter *adapter = netdev_priv(netdev);
6087 struct e1000_hw *hw = &adapter->hw;
6088 u32 ctrl, rctl;
6089
6090 igb_irq_disable(adapter);
6091 adapter->vlgrp = grp;
6092
6093 if (grp) {
6094 /* enable VLAN tag insert/strip */
6095 ctrl = rd32(E1000_CTRL);
6096 ctrl |= E1000_CTRL_VME;
6097 wr32(E1000_CTRL, ctrl);
6098
51466239 6099 /* Disable CFI check */
9d5c8243 6100 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6101 rctl &= ~E1000_RCTL_CFIEN;
6102 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6103 } else {
6104 /* disable VLAN tag insert/strip */
6105 ctrl = rd32(E1000_CTRL);
6106 ctrl &= ~E1000_CTRL_VME;
6107 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6108 }
6109
e1739522
AD		 6110 igb_rlpml_set(adapter);
6111
9d5c8243
AK		 6112 if (!test_bit(__IGB_DOWN, &adapter->state))
6113 igb_irq_enable(adapter);
6114}
6115
6116static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6117{
6118 struct igb_adapter *adapter = netdev_priv(netdev);
6119 struct e1000_hw *hw = &adapter->hw;
4ae196df 6120 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6121
51466239
AD		 6122 /* attempt to add filter to vlvf array */
6123 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6124
51466239
AD		 6125 /* add the filter since PF can receive vlans w/o entry in vlvf */
6126 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6127}
6128
6129static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6130{
6131 struct igb_adapter *adapter = netdev_priv(netdev);
6132 struct e1000_hw *hw = &adapter->hw;
4ae196df 6133 int pf_id = adapter->vfs_allocated_count;
51466239 6134 s32 err;
9d5c8243
AK		 6135
6136 igb_irq_disable(adapter);
6137 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6138
6139 if (!test_bit(__IGB_DOWN, &adapter->state))
6140 igb_irq_enable(adapter);
6141
51466239
AD		 6142 /* remove vlan from VLVF table array */
6143 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6144
51466239
AD		 6145 /* if vid was not present in VLVF just remove it from table */
6146 if (err)
4ae196df 6147 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6148}
6149
6150static void igb_restore_vlan(struct igb_adapter *adapter)
6151{
6152 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6153
6154 if (adapter->vlgrp) {
6155 u16 vid;
b738127d 6156 for (vid = 0; vid < VLAN_N_VID; vid++) {
9d5c8243
AK		 6157 if (!vlan_group_get_device(adapter->vlgrp, vid))
6158 continue;
6159 igb_vlan_rx_add_vid(adapter->netdev, vid);
6160 }
6161 }
6162}
6163
6164int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6165{
090b1795 6166 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6167 struct e1000_mac_info *mac = &adapter->hw.mac;
6168
6169 mac->autoneg = 0;
6170
cd2638a8
CW		 6171 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6172 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6173 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6174 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6175 return -EINVAL;
6176 }
6177
9d5c8243
AK		 6178 switch (spddplx) {
6179 case SPEED_10 + DUPLEX_HALF:
6180 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6181 break;
6182 case SPEED_10 + DUPLEX_FULL:
6183 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6184 break;
6185 case SPEED_100 + DUPLEX_HALF:
6186 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6187 break;
6188 case SPEED_100 + DUPLEX_FULL:
6189 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6190 break;
6191 case SPEED_1000 + DUPLEX_FULL:
6192 mac->autoneg = 1;
6193 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6194 break;
6195 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6196 default:
090b1795 6197 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6198 return -EINVAL;
6199 }
6200 return 0;
6201}
6202
3fe7c4c9 6203static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6204{
6205 struct net_device *netdev = pci_get_drvdata(pdev);
6206 struct igb_adapter *adapter = netdev_priv(netdev);
6207 struct e1000_hw *hw = &adapter->hw;
2d064c06 6208 u32 ctrl, rctl, status;
9d5c8243
AK		 6209 u32 wufc = adapter->wol;
6210#ifdef CONFIG_PM
6211 int retval = 0;
6212#endif
6213
6214 netif_device_detach(netdev);
6215
a88f10ec
AD		 6216 if (netif_running(netdev))
6217 igb_close(netdev);
6218
047e0030 6219 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6220
6221#ifdef CONFIG_PM
6222 retval = pci_save_state(pdev);
6223 if (retval)
6224 return retval;
6225#endif
6226
6227 status = rd32(E1000_STATUS);
6228 if (status & E1000_STATUS_LU)
6229 wufc &= ~E1000_WUFC_LNKC;
6230
6231 if (wufc) {
6232 igb_setup_rctl(adapter);
ff41f8dc 6233 igb_set_rx_mode(netdev);
9d5c8243
AK		 6234
6235 /* turn on all-multi mode if wake on multicast is enabled */
6236 if (wufc & E1000_WUFC_MC) {
6237 rctl = rd32(E1000_RCTL);
6238 rctl |= E1000_RCTL_MPE;
6239 wr32(E1000_RCTL, rctl);
6240 }
6241
6242 ctrl = rd32(E1000_CTRL);
6243 /* advertise wake from D3Cold */
6244 #define E1000_CTRL_ADVD3WUC 0x00100000
6245 /* phy power management enable */
6246 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6247 ctrl |= E1000_CTRL_ADVD3WUC;
6248 wr32(E1000_CTRL, ctrl);
6249
9d5c8243 6250 /* Allow time for pending master requests to run */
330a6d6a 6251 igb_disable_pcie_master(hw);
9d5c8243
AK		 6252
6253 wr32(E1000_WUC, E1000_WUC_PME_EN);
6254 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6255 } else {
6256 wr32(E1000_WUC, 0);
6257 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6258 }
6259
3fe7c4c9
RW		 6260 *enable_wake = wufc || adapter->en_mng_pt;
6261 if (!*enable_wake)
88a268c1
NN		 6262 igb_power_down_link(adapter);
6263 else
6264 igb_power_up_link(adapter);
9d5c8243
AK		 6265
6266 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6267 * would have already happened in close and is redundant. */
6268 igb_release_hw_control(adapter);
6269
6270 pci_disable_device(pdev);
6271
9d5c8243
AK		 6272 return 0;
6273}
6274
6275#ifdef CONFIG_PM
3fe7c4c9
RW		 6276static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6277{
6278 int retval;
6279 bool wake;
6280
6281 retval = __igb_shutdown(pdev, &wake);
6282 if (retval)
6283 return retval;
6284
6285 if (wake) {
6286 pci_prepare_to_sleep(pdev);
6287 } else {
6288 pci_wake_from_d3(pdev, false);
6289 pci_set_power_state(pdev, PCI_D3hot);
6290 }
6291
6292 return 0;
6293}
6294
9d5c8243
AK		 6295static int igb_resume(struct pci_dev *pdev)
6296{
6297 struct net_device *netdev = pci_get_drvdata(pdev);
6298 struct igb_adapter *adapter = netdev_priv(netdev);
6299 struct e1000_hw *hw = &adapter->hw;
6300 u32 err;
6301
6302 pci_set_power_state(pdev, PCI_D0);
6303 pci_restore_state(pdev);
b94f2d77 6304 pci_save_state(pdev);
42bfd33a 6305
aed5dec3 6306 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6307 if (err) {
6308 dev_err(&pdev->dev,
6309 "igb: Cannot enable PCI device from suspend\n");
6310 return err;
6311 }
6312 pci_set_master(pdev);
6313
6314 pci_enable_wake(pdev, PCI_D3hot, 0);
6315 pci_enable_wake(pdev, PCI_D3cold, 0);
6316
047e0030 6317 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6318 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6319 return -ENOMEM;
9d5c8243
AK		 6320 }
6321
9d5c8243 6322 igb_reset(adapter);
a8564f03
AD		 6323
6324 /* let the f/w know that the h/w is now under the control of the
6325 * driver. */
6326 igb_get_hw_control(adapter);
6327
9d5c8243
AK		 6328 wr32(E1000_WUS, ~0);
6329
a88f10ec
AD		 6330 if (netif_running(netdev)) {
6331 err = igb_open(netdev);
6332 if (err)
6333 return err;
6334 }
9d5c8243
AK		 6335
6336 netif_device_attach(netdev);
6337
9d5c8243
AK		 6338 return 0;
6339}
6340#endif
6341
6342static void igb_shutdown(struct pci_dev *pdev)
6343{
3fe7c4c9
RW		 6344 bool wake;
6345
6346 __igb_shutdown(pdev, &wake);
6347
6348 if (system_state == SYSTEM_POWER_OFF) {
6349 pci_wake_from_d3(pdev, wake);
6350 pci_set_power_state(pdev, PCI_D3hot);
6351 }
9d5c8243
AK		 6352}
6353
6354#ifdef CONFIG_NET_POLL_CONTROLLER
6355/*
6356 * Polling 'interrupt' - used by things like netconsole to send skbs
6357 * without having to re-enable interrupts. It's not called while
6358 * the interrupt routine is executing.
6359 */
6360static void igb_netpoll(struct net_device *netdev)
6361{
6362 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6363 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6364 int i;
9d5c8243 6365
eebbbdba 6366 if (!adapter->msix_entries) {
047e0030 6367 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6368 igb_irq_disable(adapter);
047e0030 6369 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6370 return;
6371 }
9d5c8243 6372
047e0030
AD		 6373 for (i = 0; i < adapter->num_q_vectors; i++) {
6374 struct igb_q_vector *q_vector = adapter->q_vector[i];
6375 wr32(E1000_EIMC, q_vector->eims_value);
6376 napi_schedule(&q_vector->napi);
eebbbdba 6377 }
9d5c8243
AK		 6378}
6379#endif /* CONFIG_NET_POLL_CONTROLLER */
6380
6381/**
6382 * igb_io_error_detected - called when PCI error is detected
6383 * @pdev: Pointer to PCI device
6384 * @state: The current pci connection state
6385 *
6386 * This function is called after a PCI bus error affecting
6387 * this device has been detected.
6388 */
6389static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6390 pci_channel_state_t state)
6391{
6392 struct net_device *netdev = pci_get_drvdata(pdev);
6393 struct igb_adapter *adapter = netdev_priv(netdev);
6394
6395 netif_device_detach(netdev);
6396
59ed6eec
AD		 6397 if (state == pci_channel_io_perm_failure)
6398 return PCI_ERS_RESULT_DISCONNECT;
6399
9d5c8243
AK		 6400 if (netif_running(netdev))
6401 igb_down(adapter);
6402 pci_disable_device(pdev);
6403
6404 /* Request a slot slot reset. */
6405 return PCI_ERS_RESULT_NEED_RESET;
6406}
6407
6408/**
6409 * igb_io_slot_reset - called after the pci bus has been reset.
6410 * @pdev: Pointer to PCI device
6411 *
6412 * Restart the card from scratch, as if from a cold-boot. Implementation
6413 * resembles the first-half of the igb_resume routine.
6414 */
6415static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6416{
6417 struct net_device *netdev = pci_get_drvdata(pdev);
6418 struct igb_adapter *adapter = netdev_priv(netdev);
6419 struct e1000_hw *hw = &adapter->hw;
40a914fa 6420 pci_ers_result_t result;
42bfd33a 6421 int err;
9d5c8243 6422
aed5dec3 6423 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6424 dev_err(&pdev->dev,
6425 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6426 result = PCI_ERS_RESULT_DISCONNECT;
6427 } else {
6428 pci_set_master(pdev);
6429 pci_restore_state(pdev);
b94f2d77 6430 pci_save_state(pdev);
9d5c8243 6431
40a914fa
AD		 6432 pci_enable_wake(pdev, PCI_D3hot, 0);
6433 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6434
40a914fa
AD		 6435 igb_reset(adapter);
6436 wr32(E1000_WUS, ~0);
6437 result = PCI_ERS_RESULT_RECOVERED;
6438 }
9d5c8243 6439
ea943d41
JK		 6440 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6441 if (err) {
6442 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6443 "failed 0x%0x\n", err);
6444 /* non-fatal, continue */
6445 }
40a914fa
AD		 6446
6447 return result;
9d5c8243
AK		 6448}
6449
6450/**
6451 * igb_io_resume - called when traffic can start flowing again.
6452 * @pdev: Pointer to PCI device
6453 *
6454 * This callback is called when the error recovery driver tells us that
6455 * its OK to resume normal operation. Implementation resembles the
6456 * second-half of the igb_resume routine.
6457 */
6458static void igb_io_resume(struct pci_dev *pdev)
6459{
6460 struct net_device *netdev = pci_get_drvdata(pdev);
6461 struct igb_adapter *adapter = netdev_priv(netdev);
6462
9d5c8243
AK		 6463 if (netif_running(netdev)) {
6464 if (igb_up(adapter)) {
6465 dev_err(&pdev->dev, "igb_up failed after reset\n");
6466 return;
6467 }
6468 }
6469
6470 netif_device_attach(netdev);
6471
6472 /* let the f/w know that the h/w is now under the control of the
6473 * driver. */
6474 igb_get_hw_control(adapter);
9d5c8243
AK		 6475}
6476
26ad9178
AD		 6477static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6478 u8 qsel)
6479{
6480 u32 rar_low, rar_high;
6481 struct e1000_hw *hw = &adapter->hw;
6482
6483 /* HW expects these in little endian so we reverse the byte order
6484 * from network order (big endian) to little endian
6485 */
6486 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6487 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6488 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6489
6490 /* Indicate to hardware the Address is Valid. */
6491 rar_high |= E1000_RAH_AV;
6492
6493 if (hw->mac.type == e1000_82575)
6494 rar_high |= E1000_RAH_POOL_1 * qsel;
6495 else
6496 rar_high |= E1000_RAH_POOL_1 << qsel;
6497
6498 wr32(E1000_RAL(index), rar_low);
6499 wrfl();
6500 wr32(E1000_RAH(index), rar_high);
6501 wrfl();
6502}
6503
4ae196df
AD		 6504static int igb_set_vf_mac(struct igb_adapter *adapter,
6505 int vf, unsigned char *mac_addr)
6506{
6507 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6508 /* VF MAC addresses start at end of receive addresses and moves
6509 * torwards the first, as a result a collision should not be possible */
6510 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6511
37680117 6512 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6513
26ad9178 6514 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6515
6516 return 0;
6517}
6518
8151d294
WM		 6519static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6520{
6521 struct igb_adapter *adapter = netdev_priv(netdev);
6522 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6523 return -EINVAL;
6524 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6525 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6526 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6527 " change effective.");
6528 if (test_bit(__IGB_DOWN, &adapter->state)) {
6529 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6530 " but the PF device is not up.\n");
6531 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6532 " attempting to use the VF device.\n");
6533 }
6534 return igb_set_vf_mac(adapter, vf, mac);
6535}
6536
6537static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6538{
6539 return -EOPNOTSUPP;
6540}
6541
6542static int igb_ndo_get_vf_config(struct net_device *netdev,
6543 int vf, struct ifla_vf_info *ivi)
6544{
6545 struct igb_adapter *adapter = netdev_priv(netdev);
6546 if (vf >= adapter->vfs_allocated_count)
6547 return -EINVAL;
6548 ivi->vf = vf;
6549 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6550 ivi->tx_rate = 0;
6551 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6552 ivi->qos = adapter->vf_data[vf].pf_qos;
6553 return 0;
6554}
6555
4ae196df
AD		 6556static void igb_vmm_control(struct igb_adapter *adapter)
6557{
6558 struct e1000_hw *hw = &adapter->hw;
10d8e907 6559 u32 reg;
4ae196df 6560
52a1dd4d
AD		 6561 switch (hw->mac.type) {
6562 case e1000_82575:
6563 default:
6564 /* replication is not supported for 82575 */
4ae196df 6565 return;
52a1dd4d
AD		 6566 case e1000_82576:
6567 /* notify HW that the MAC is adding vlan tags */
6568 reg = rd32(E1000_DTXCTL);
6569 reg |= E1000_DTXCTL_VLAN_ADDED;
6570 wr32(E1000_DTXCTL, reg);
6571 case e1000_82580:
6572 /* enable replication vlan tag stripping */
6573 reg = rd32(E1000_RPLOLR);
6574 reg |= E1000_RPLOLR_STRVLAN;
6575 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6576 case e1000_i350:
6577 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6578 break;
6579 }
10d8e907 6580
d4960307
AD		 6581 if (adapter->vfs_allocated_count) {
6582 igb_vmdq_set_loopback_pf(hw, true);
6583 igb_vmdq_set_replication_pf(hw, true);
6584 } else {
6585 igb_vmdq_set_loopback_pf(hw, false);
6586 igb_vmdq_set_replication_pf(hw, false);
6587 }
4ae196df
AD		 6588}
6589
9d5c8243 6590/* igb_main.c */
Clone of davem's net-next-2.6 tree