]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'bugzilla-16422' into release
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
86 {0, }
87};
88
89MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
90
91void igb_reset(struct igb_adapter *);
92static int igb_setup_all_tx_resources(struct igb_adapter *);
93static int igb_setup_all_rx_resources(struct igb_adapter *);
94static void igb_free_all_tx_resources(struct igb_adapter *);
95static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 96static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 97void igb_update_stats(struct igb_adapter *);
98static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99static void __devexit igb_remove(struct pci_dev *pdev);
100static int igb_sw_init(struct igb_adapter *);
101static int igb_open(struct net_device *);
102static int igb_close(struct net_device *);
103static void igb_configure_tx(struct igb_adapter *);
104static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 105static void igb_clean_all_tx_rings(struct igb_adapter *);
106static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 107static void igb_clean_tx_ring(struct igb_ring *);
108static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 109static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 110static void igb_update_phy_info(unsigned long);
111static void igb_watchdog(unsigned long);
112static void igb_watchdog_task(struct work_struct *);
b1a436c3 113static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 114static struct net_device_stats *igb_get_stats(struct net_device *);
115static int igb_change_mtu(struct net_device *, int);
116static int igb_set_mac(struct net_device *, void *);
68d480c4 117static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 118static irqreturn_t igb_intr(int irq, void *);
119static irqreturn_t igb_intr_msi(int irq, void *);
120static irqreturn_t igb_msix_other(int irq, void *);
047e0030 121static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 122#ifdef CONFIG_IGB_DCA
047e0030 123static void igb_update_dca(struct igb_q_vector *);
fe4506b6 124static void igb_setup_dca(struct igb_adapter *);
421e02f0 125#endif /* CONFIG_IGB_DCA */
047e0030 126static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 127static int igb_poll(struct napi_struct *, int);
047e0030 128static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 129static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130static void igb_tx_timeout(struct net_device *);
131static void igb_reset_task(struct work_struct *);
132static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133static void igb_vlan_rx_add_vid(struct net_device *, u16);
134static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135static void igb_restore_vlan(struct igb_adapter *);
26ad9178 136static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 137static void igb_ping_all_vfs(struct igb_adapter *);
138static void igb_msg_task(struct igb_adapter *);
4ae196df 139static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 140static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 141static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 142static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
9d5c8243 148
9d5c8243 149#ifdef CONFIG_PM
3fe7c4c9 150static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 151static int igb_resume(struct pci_dev *);
152#endif
153static void igb_shutdown(struct pci_dev *);
421e02f0 154#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 155static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
158 .next = NULL,
159 .priority = 0
160};
161#endif
9d5c8243
AK		 162#ifdef CONFIG_NET_POLL_CONTROLLER
163/* for netdump / net console */
164static void igb_netpoll(struct net_device *);
165#endif
37680117 166#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 167static unsigned int max_vfs = 0;
168module_param(max_vfs, uint, 0);
169MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171#endif /* CONFIG_PCI_IOV */
172
9d5c8243
AK		 173static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176static void igb_io_resume(struct pci_dev *);
177
178static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
182};
183
184
185static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
188 .probe = igb_probe,
189 .remove = __devexit_p(igb_remove),
190#ifdef CONFIG_PM
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
194#endif
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
197};
198
199MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201MODULE_LICENSE("GPL");
202MODULE_VERSION(DRV_VERSION);
203
c97ec42a
TI		 204struct igb_reg_info {
205 u32 ofs;
206 char *name;
207};
208
209static const struct igb_reg_info igb_reg_info_tbl[] = {
210
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
215
216 /* Interrupt Registers */
217 {E1000_ICR, "ICR"},
218
219 /* RX Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
227
228 /* TX Registers */
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
240
241 /* List Terminator */
242 {}
243};
244
245/*
246 * igb_regdump - register printout routine
247 */
248static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
249{
250 int n = 0;
251 char rname[16];
252 u32 regs[8];
253
254 switch (reginfo->ofs) {
255 case E1000_RDLEN(0):
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
258 break;
259 case E1000_RDH(0):
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
262 break;
263 case E1000_RDT(0):
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
266 break;
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
270 break;
271 case E1000_RDBAL(0):
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
274 break;
275 case E1000_RDBAH(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
278 break;
279 case E1000_TDBAL(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
282 break;
283 case E1000_TDBAH(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
286 break;
287 case E1000_TDLEN(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
290 break;
291 case E1000_TDH(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
294 break;
295 case E1000_TDT(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
298 break;
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
302 break;
303 default:
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
306 return;
307 }
308
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
314}
315
316/*
317 * igb_dump - Print registers, tx-rings and rx-rings
318 */
319static void igb_dump(struct igb_adapter *adapter)
320{
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
324 int n = 0;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
331 u32 staterr;
332 int i = 0;
333
334 if (!netif_msg_hw(adapter))
335 return;
336
337 /* Print netdevice Info */
338 if (netdev) {
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
343 netdev->name,
344 netdev->state,
345 netdev->trans_start,
346 netdev->last_rx);
347 }
348
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
355 }
356
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
359 goto exit;
360
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
370 buffer_info->length,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
373 }
374
375 /* Print TX Rings */
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
378
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
380
381 /* Transmit Descriptor Formats
382 *
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
390 */
391
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
400
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
407 le64_to_cpu(u0->a),
408 le64_to_cpu(u0->b),
409 (u64)buffer_info->dma,
410 buffer_info->length,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
413 buffer_info->skb);
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
421 else
422 printk(KERN_CONT "\n");
423
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
426 DUMP_PREFIX_ADDRESS,
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
429 }
430 }
431
432 /* Print RX Rings Summary */
433rx_ring_summary:
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
440 }
441
442 /* Print RX Rings */
443 if (!netif_msg_rx_status(adapter))
444 goto exit;
445
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
447
448 /* Advanced Receive Descriptor (Read) Format
449 * 63 1 0
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
455 *
456 *
457 * Advanced Receive Descriptor (Write-Back) Format
458 *
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
467 */
468
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
480
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
490 le64_to_cpu(u0->a),
491 le64_to_cpu(u0->b),
492 buffer_info->skb);
493 } else {
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
496 le64_to_cpu(u0->a),
497 le64_to_cpu(u0->b),
498 (u64)buffer_info->dma,
499 buffer_info->skb);
500
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
503 DUMP_PREFIX_ADDRESS,
504 16, 1,
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
508 < IGB_RXBUFFER_1024)
509 print_hex_dump(KERN_INFO, "",
510 DUMP_PREFIX_ADDRESS,
511 16, 1,
512 phys_to_virt(
513 buffer_info->page_dma +
514 buffer_info->page_offset),
515 PAGE_SIZE/2, true);
516 }
517 }
518
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
523 else
524 printk(KERN_CONT "\n");
525
526 }
527 }
528
529exit:
530 return;
531}
532
533
38c845c7
PO		 534/**
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
536 */
537static cycle_t igb_read_clock(const struct cyclecounter *tc)
538{
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 542 u64 stamp = 0;
543 int shift = 0;
38c845c7 544
55cac248
AD		 545 /*
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
549 */
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
553 }
554
c5b9bd5e
AD		 555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 557 return stamp;
558}
559
9d5c8243 560/**
c041076a 561 * igb_get_hw_dev - return device
9d5c8243
AK		 562 * used by hardware layer to print debugging information
563 **/
c041076a 564struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 565{
566 struct igb_adapter *adapter = hw->back;
c041076a 567 return adapter->netdev;
9d5c8243 568}
38c845c7 569
9d5c8243
AK		 570/**
571 * igb_init_module - Driver Registration Routine
572 *
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
575 **/
576static int __init igb_init_module(void)
577{
578 int ret;
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
581
582 printk(KERN_INFO "%s\n", igb_copyright);
583
421e02f0 584#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 585 dca_register_notify(&dca_notifier);
586#endif
bbd98fe4 587 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 588 return ret;
589}
590
591module_init(igb_init_module);
592
593/**
594 * igb_exit_module - Driver Exit Cleanup Routine
595 *
596 * igb_exit_module is called just before the driver is removed
597 * from memory.
598 **/
599static void __exit igb_exit_module(void)
600{
421e02f0 601#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 602 dca_unregister_notify(&dca_notifier);
603#endif
9d5c8243
AK		 604 pci_unregister_driver(&igb_driver);
605}
606
607module_exit(igb_exit_module);
608
26bc19ec
AD		 609#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
610/**
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
613 *
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
616 **/
617static void igb_cache_ring_register(struct igb_adapter *adapter)
618{
ee1b9f06 619 int i = 0, j = 0;
047e0030 620 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 621
622 switch (adapter->hw.mac.type) {
623 case e1000_82576:
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
628 */
ee1b9f06 629 if (adapter->vfs_allocated_count) {
a99955fc 630 for (; i < adapter->rss_queues; i++)
3025a446
AD		 631 adapter->rx_ring[i]->reg_idx = rbase_offset +
632 Q_IDX_82576(i);
a99955fc 633 for (; j < adapter->rss_queues; j++)
3025a446
AD		 634 adapter->tx_ring[j]->reg_idx = rbase_offset +
635 Q_IDX_82576(j);
ee1b9f06 636 }
26bc19ec 637 case e1000_82575:
55cac248 638 case e1000_82580:
d2ba2ed8 639 case e1000_i350:
26bc19ec 640 default:
ee1b9f06 641 for (; i < adapter->num_rx_queues; i++)
3025a446 642 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 643 for (; j < adapter->num_tx_queues; j++)
3025a446 644 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 645 break;
646 }
647}
648
047e0030
AD		 649static void igb_free_queues(struct igb_adapter *adapter)
650{
3025a446 651 int i;
047e0030 652
3025a446
AD		 653 for (i = 0; i < adapter->num_tx_queues; i++) {
654 kfree(adapter->tx_ring[i]);
655 adapter->tx_ring[i] = NULL;
656 }
657 for (i = 0; i < adapter->num_rx_queues; i++) {
658 kfree(adapter->rx_ring[i]);
659 adapter->rx_ring[i] = NULL;
660 }
047e0030
AD		 661 adapter->num_rx_queues = 0;
662 adapter->num_tx_queues = 0;
663}
664
9d5c8243
AK		 665/**
666 * igb_alloc_queues - Allocate memory for all rings
667 * @adapter: board private structure to initialize
668 *
669 * We allocate one ring per queue at run-time since we don't know the
670 * number of queues at compile-time.
671 **/
672static int igb_alloc_queues(struct igb_adapter *adapter)
673{
3025a446 674 struct igb_ring *ring;
9d5c8243
AK		 675 int i;
676
661086df 677 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 678 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
679 if (!ring)
680 goto err;
68fd9910 681 ring->count = adapter->tx_ring_count;
661086df 682 ring->queue_index = i;
59d71989 683 ring->dev = &adapter->pdev->dev;
e694e964 684 ring->netdev = adapter->netdev;
85ad76b2
AD		 685 /* For 82575, context index must be unique per ring. */
686 if (adapter->hw.mac.type == e1000_82575)
687 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 688 adapter->tx_ring[i] = ring;
661086df 689 }
85ad76b2 690
9d5c8243 691 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 692 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
693 if (!ring)
694 goto err;
68fd9910 695 ring->count = adapter->rx_ring_count;
844290e5 696 ring->queue_index = i;
59d71989 697 ring->dev = &adapter->pdev->dev;
e694e964 698 ring->netdev = adapter->netdev;
4c844851 699 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 700 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
701 /* set flag indicating ring supports SCTP checksum offload */
702 if (adapter->hw.mac.type >= e1000_82576)
703 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 704 adapter->rx_ring[i] = ring;
9d5c8243 705 }
26bc19ec
AD		 706
707 igb_cache_ring_register(adapter);
9d5c8243 708
047e0030 709 return 0;
a88f10ec 710
047e0030
AD		 711err:
712 igb_free_queues(adapter);
d1a8c9e1 713
047e0030 714 return -ENOMEM;
a88f10ec
AD		 715}
716
9d5c8243 717#define IGB_N0_QUEUE -1
047e0030 718static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 719{
720 u32 msixbm = 0;
047e0030 721 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 722 struct e1000_hw *hw = &adapter->hw;
2d064c06 723 u32 ivar, index;
047e0030
AD		 724 int rx_queue = IGB_N0_QUEUE;
725 int tx_queue = IGB_N0_QUEUE;
726
727 if (q_vector->rx_ring)
728 rx_queue = q_vector->rx_ring->reg_idx;
729 if (q_vector->tx_ring)
730 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 731
732 switch (hw->mac.type) {
733 case e1000_82575:
9d5c8243
AK		 734 /* The 82575 assigns vectors using a bitmask, which matches the
735 bitmask for the EICR/EIMS/EIMC registers. To assign one
736 or more queues to a vector, we write the appropriate bits
737 into the MSIXBM register for that vector. */
047e0030 738 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 739 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 740 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 741 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 742 if (!adapter->msix_entries && msix_vector == 0)
743 msixbm |= E1000_EIMS_OTHER;
9d5c8243 744 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 745 q_vector->eims_value = msixbm;
2d064c06
AD		 746 break;
747 case e1000_82576:
26bc19ec 748 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 749 Each queue has a single entry in the table to which we write
750 a vector number along with a "valid" bit. Sadly, the layout
751 of the table is somewhat counterintuitive. */
752 if (rx_queue > IGB_N0_QUEUE) {
047e0030 753 index = (rx_queue & 0x7);
2d064c06 754 ivar = array_rd32(E1000_IVAR0, index);
047e0030 755 if (rx_queue < 8) {
26bc19ec
AD		 756 /* vector goes into low byte of register */
757 ivar = ivar & 0xFFFFFF00;
758 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 759 } else {
760 /* vector goes into third byte of register */
761 ivar = ivar & 0xFF00FFFF;
762 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 763 }
2d064c06
AD		 764 array_wr32(E1000_IVAR0, index, ivar);
765 }
766 if (tx_queue > IGB_N0_QUEUE) {
047e0030 767 index = (tx_queue & 0x7);
2d064c06 768 ivar = array_rd32(E1000_IVAR0, index);
047e0030 769 if (tx_queue < 8) {
26bc19ec
AD		 770 /* vector goes into second byte of register */
771 ivar = ivar & 0xFFFF00FF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 773 } else {
774 /* vector goes into high byte of register */
775 ivar = ivar & 0x00FFFFFF;
776 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 777 }
2d064c06
AD		 778 array_wr32(E1000_IVAR0, index, ivar);
779 }
047e0030 780 q_vector->eims_value = 1 << msix_vector;
2d064c06 781 break;
55cac248 782 case e1000_82580:
d2ba2ed8 783 case e1000_i350:
55cac248
AD		 784 /* 82580 uses the same table-based approach as 82576 but has fewer
785 entries as a result we carry over for queues greater than 4. */
786 if (rx_queue > IGB_N0_QUEUE) {
787 index = (rx_queue >> 1);
788 ivar = array_rd32(E1000_IVAR0, index);
789 if (rx_queue & 0x1) {
790 /* vector goes into third byte of register */
791 ivar = ivar & 0xFF00FFFF;
792 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
793 } else {
794 /* vector goes into low byte of register */
795 ivar = ivar & 0xFFFFFF00;
796 ivar |= msix_vector | E1000_IVAR_VALID;
797 }
798 array_wr32(E1000_IVAR0, index, ivar);
799 }
800 if (tx_queue > IGB_N0_QUEUE) {
801 index = (tx_queue >> 1);
802 ivar = array_rd32(E1000_IVAR0, index);
803 if (tx_queue & 0x1) {
804 /* vector goes into high byte of register */
805 ivar = ivar & 0x00FFFFFF;
806 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
807 } else {
808 /* vector goes into second byte of register */
809 ivar = ivar & 0xFFFF00FF;
810 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
811 }
812 array_wr32(E1000_IVAR0, index, ivar);
813 }
814 q_vector->eims_value = 1 << msix_vector;
815 break;
2d064c06
AD		 816 default:
817 BUG();
818 break;
819 }
26b39276
AD		 820
821 /* add q_vector eims value to global eims_enable_mask */
822 adapter->eims_enable_mask |= q_vector->eims_value;
823
824 /* configure q_vector to set itr on first interrupt */
825 q_vector->set_itr = 1;
9d5c8243
AK		 826}
827
828/**
829 * igb_configure_msix - Configure MSI-X hardware
830 *
831 * igb_configure_msix sets up the hardware to properly
832 * generate MSI-X interrupts.
833 **/
834static void igb_configure_msix(struct igb_adapter *adapter)
835{
836 u32 tmp;
837 int i, vector = 0;
838 struct e1000_hw *hw = &adapter->hw;
839
840 adapter->eims_enable_mask = 0;
9d5c8243
AK		 841
842 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 843 switch (hw->mac.type) {
844 case e1000_82575:
9d5c8243
AK		 845 tmp = rd32(E1000_CTRL_EXT);
846 /* enable MSI-X PBA support*/
847 tmp |= E1000_CTRL_EXT_PBA_CLR;
848
849 /* Auto-Mask interrupts upon ICR read. */
850 tmp |= E1000_CTRL_EXT_EIAME;
851 tmp |= E1000_CTRL_EXT_IRCA;
852
853 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 854
855 /* enable msix_other interrupt */
856 array_wr32(E1000_MSIXBM(0), vector++,
857 E1000_EIMS_OTHER);
844290e5 858 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 859
2d064c06
AD		 860 break;
861
862 case e1000_82576:
55cac248 863 case e1000_82580:
d2ba2ed8 864 case e1000_i350:
047e0030
AD		 865 /* Turn on MSI-X capability first, or our settings
866 * won't stick. And it will take days to debug. */
867 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
868 E1000_GPIE_PBA | E1000_GPIE_EIAME |
869 E1000_GPIE_NSICR);
870
871 /* enable msix_other interrupt */
872 adapter->eims_other = 1 << vector;
2d064c06 873 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 874
047e0030 875 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 876 break;
877 default:
878 /* do nothing, since nothing else supports MSI-X */
879 break;
880 } /* switch (hw->mac.type) */
047e0030
AD		 881
882 adapter->eims_enable_mask |= adapter->eims_other;
883
26b39276
AD		 884 for (i = 0; i < adapter->num_q_vectors; i++)
885 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 886
9d5c8243
AK		 887 wrfl();
888}
889
890/**
891 * igb_request_msix - Initialize MSI-X interrupts
892 *
893 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
894 * kernel.
895 **/
896static int igb_request_msix(struct igb_adapter *adapter)
897{
898 struct net_device *netdev = adapter->netdev;
047e0030 899 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 900 int i, err = 0, vector = 0;
901
047e0030 902 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 903 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 904 if (err)
905 goto out;
906 vector++;
907
908 for (i = 0; i < adapter->num_q_vectors; i++) {
909 struct igb_q_vector *q_vector = adapter->q_vector[i];
910
911 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
912
913 if (q_vector->rx_ring && q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
915 q_vector->rx_ring->queue_index);
916 else if (q_vector->tx_ring)
917 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
918 q_vector->tx_ring->queue_index);
919 else if (q_vector->rx_ring)
920 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
921 q_vector->rx_ring->queue_index);
9d5c8243 922 else
047e0030
AD		 923 sprintf(q_vector->name, "%s-unused", netdev->name);
924
9d5c8243 925 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 926 igb_msix_ring, 0, q_vector->name,
047e0030 927 q_vector);
9d5c8243
AK		 928 if (err)
929 goto out;
9d5c8243
AK		 930 vector++;
931 }
932
9d5c8243
AK		 933 igb_configure_msix(adapter);
934 return 0;
935out:
936 return err;
937}
938
939static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
940{
941 if (adapter->msix_entries) {
942 pci_disable_msix(adapter->pdev);
943 kfree(adapter->msix_entries);
944 adapter->msix_entries = NULL;
047e0030 945 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 946 pci_disable_msi(adapter->pdev);
047e0030 947 }
9d5c8243
AK		 948}
949
047e0030
AD		 950/**
951 * igb_free_q_vectors - Free memory allocated for interrupt vectors
952 * @adapter: board private structure to initialize
953 *
954 * This function frees the memory allocated to the q_vectors. In addition if
955 * NAPI is enabled it will delete any references to the NAPI struct prior
956 * to freeing the q_vector.
957 **/
958static void igb_free_q_vectors(struct igb_adapter *adapter)
959{
960 int v_idx;
961
962 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
963 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
964 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 965 if (!q_vector)
966 continue;
047e0030
AD		 967 netif_napi_del(&q_vector->napi);
968 kfree(q_vector);
969 }
970 adapter->num_q_vectors = 0;
971}
972
973/**
974 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
975 *
976 * This function resets the device so that it has 0 rx queues, tx queues, and
977 * MSI-X interrupts allocated.
978 */
979static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
980{
981 igb_free_queues(adapter);
982 igb_free_q_vectors(adapter);
983 igb_reset_interrupt_capability(adapter);
984}
9d5c8243
AK		 985
986/**
987 * igb_set_interrupt_capability - set MSI or MSI-X if supported
988 *
989 * Attempt to configure interrupts using the best available
990 * capabilities of the hardware and kernel.
991 **/
992static void igb_set_interrupt_capability(struct igb_adapter *adapter)
993{
994 int err;
995 int numvecs, i;
996
83b7180d 997 /* Number of supported queues. */
a99955fc
AD		 998 adapter->num_rx_queues = adapter->rss_queues;
999 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1000
047e0030
AD		 1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1003
3ad2f3fb 1004 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
047e0030
AD		 1007
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1010
1011 /* add 1 vector for link status interrupts */
1012 numvecs++;
9d5c8243
AK		 1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1014 GFP_KERNEL);
1015 if (!adapter->msix_entries)
1016 goto msi_only;
1017
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1020
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1023 numvecs);
1024 if (err == 0)
34a20e89 1025 goto out;
9d5c8243
AK		 1026
1027 igb_reset_interrupt_capability(adapter);
1028
1029 /* If we can't do MSI-X, try MSI */
1030msi_only:
2a3abf6d
AD		 1031#ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1037 msleep(500);
1038
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1042 msleep(100);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1044 }
1045#endif
4fc82adf 1046 adapter->vfs_allocated_count = 0;
a99955fc 1047 adapter->rss_queues = 1;
4fc82adf 1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1049 adapter->num_rx_queues = 1;
661086df 1050 adapter->num_tx_queues = 1;
047e0030 1051 adapter->num_q_vectors = 1;
9d5c8243 1052 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1053 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1054out:
661086df 1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 1057}
1058
047e0030
AD		 1059/**
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1062 *
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1064 * return -ENOMEM.
1065 **/
1066static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1067{
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1070 int v_idx;
1071
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1074 if (!q_vector)
1075 goto err_out;
1076 q_vector->adapter = adapter;
047e0030
AD		 1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1081 }
1082 return 0;
1083
1084err_out:
fe0592b4 1085 igb_free_q_vectors(adapter);
047e0030
AD		 1086 return -ENOMEM;
1087}
1088
1089static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1091{
3025a446 1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1093
3025a446 1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1095 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1099}
1100
1101static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1103{
3025a446 1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1105
3025a446 1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1107 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1111}
1112
1113/**
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1115 *
1116 * This function maps the recently allocated queues to vectors.
1117 **/
1118static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1119{
1120 int i;
1121 int v_idx = 0;
1122
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1125 return -ENOMEM;
1126
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1133 } else {
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1138 }
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141 }
1142 return 0;
1143}
1144
1145/**
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1147 *
1148 * This function initializes the interrupts and allocates all of the queues.
1149 **/
1150static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1151{
1152 struct pci_dev *pdev = adapter->pdev;
1153 int err;
1154
1155 igb_set_interrupt_capability(adapter);
1156
1157 err = igb_alloc_q_vectors(adapter);
1158 if (err) {
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1161 }
1162
1163 err = igb_alloc_queues(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1167 }
1168
1169 err = igb_map_ring_to_vector(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1173 }
1174
1175
1176 return 0;
1177err_map_queues:
1178 igb_free_queues(adapter);
1179err_alloc_queues:
1180 igb_free_q_vectors(adapter);
1181err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1183 return err;
1184}
1185
9d5c8243
AK		 1186/**
1187 * igb_request_irq - initialize interrupts
1188 *
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1191 **/
1192static int igb_request_irq(struct igb_adapter *adapter)
1193{
1194 struct net_device *netdev = adapter->netdev;
047e0030 1195 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1196 int err = 0;
1197
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
844290e5 1200 if (!err)
9d5c8243 1201 goto request_done;
9d5c8243 1202 /* fall back to MSI */
047e0030 1203 igb_clear_interrupt_scheme(adapter);
9d5c8243 1204 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1205 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
047e0030 1208 adapter->num_tx_queues = 1;
9d5c8243 1209 adapter->num_rx_queues = 1;
047e0030
AD		 1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1212 if (err) {
1213 dev_err(&pdev->dev,
1214 "Unable to allocate memory for vectors\n");
1215 goto request_done;
1216 }
1217 err = igb_alloc_queues(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1222 goto request_done;
1223 }
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
844290e5 1226 } else {
feeb2721 1227 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1228 }
844290e5 1229
7dfc16fa 1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1232 netdev->name, adapter);
9d5c8243
AK		 1233 if (!err)
1234 goto request_done;
047e0030 1235
9d5c8243
AK		 1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
7dfc16fa 1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1239 }
1240
a0607fd3 1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1242 netdev->name, adapter);
9d5c8243 1243
6cb5e577 1244 if (err)
9d5c8243
AK		 1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1246 err);
9d5c8243
AK		 1247
1248request_done:
1249 return err;
1250}
1251
1252static void igb_free_irq(struct igb_adapter *adapter)
1253{
9d5c8243
AK		 1254 if (adapter->msix_entries) {
1255 int vector = 0, i;
1256
047e0030 1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1258
047e0030
AD		 1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1262 q_vector);
1263 }
1264 } else {
1265 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1266 }
9d5c8243
AK		 1267}
1268
1269/**
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1272 **/
1273static void igb_irq_disable(struct igb_adapter *adapter)
1274{
1275 struct e1000_hw *hw = &adapter->hw;
1276
25568a53
AD		 1277 /*
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1281 */
9d5c8243 1282 if (adapter->msix_entries) {
2dfd1212
AD		 1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1288 }
844290e5
PW		 1289
1290 wr32(E1000_IAM, 0);
9d5c8243
AK		 1291 wr32(E1000_IMC, ~0);
1292 wrfl();
1293 synchronize_irq(adapter->pdev->irq);
1294}
1295
1296/**
1297 * igb_irq_enable - Enable default interrupt generation settings
1298 * @adapter: board private structure
1299 **/
1300static void igb_irq_enable(struct igb_adapter *adapter)
1301{
1302 struct e1000_hw *hw = &adapter->hw;
1303
1304 if (adapter->msix_entries) {
25568a53 1305 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1306 u32 regval = rd32(E1000_EIAC);
1307 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308 regval = rd32(E1000_EIAM);
1309 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1310 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1311 if (adapter->vfs_allocated_count) {
4ae196df 1312 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1313 ims |= E1000_IMS_VMMB;
1314 }
55cac248
AD		 1315 if (adapter->hw.mac.type == e1000_82580)
1316 ims |= E1000_IMS_DRSTA;
1317
25568a53 1318 wr32(E1000_IMS, ims);
844290e5 1319 } else {
55cac248
AD		 1320 wr32(E1000_IMS, IMS_ENABLE_MASK |
1321 E1000_IMS_DRSTA);
1322 wr32(E1000_IAM, IMS_ENABLE_MASK |
1323 E1000_IMS_DRSTA);
844290e5 1324 }
9d5c8243
AK		 1325}
1326
1327static void igb_update_mng_vlan(struct igb_adapter *adapter)
1328{
51466239 1329 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1330 u16 vid = adapter->hw.mng_cookie.vlan_id;
1331 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1332
1333 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334 /* add VID to filter table */
1335 igb_vfta_set(hw, vid, true);
1336 adapter->mng_vlan_id = vid;
1337 } else {
1338 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1339 }
1340
1341 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1342 (vid != old_vid) &&
1343 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344 /* remove VID from filter table */
1345 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1346 }
1347}
1348
1349/**
1350 * igb_release_hw_control - release control of the h/w to f/w
1351 * @adapter: address of board private structure
1352 *
1353 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354 * For ASF and Pass Through versions of f/w this means that the
1355 * driver is no longer loaded.
1356 *
1357 **/
1358static void igb_release_hw_control(struct igb_adapter *adapter)
1359{
1360 struct e1000_hw *hw = &adapter->hw;
1361 u32 ctrl_ext;
1362
1363 /* Let firmware take over control of h/w */
1364 ctrl_ext = rd32(E1000_CTRL_EXT);
1365 wr32(E1000_CTRL_EXT,
1366 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1367}
1368
9d5c8243
AK		 1369/**
1370 * igb_get_hw_control - get control of the h/w from f/w
1371 * @adapter: address of board private structure
1372 *
1373 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374 * For ASF and Pass Through versions of f/w this means that
1375 * the driver is loaded.
1376 *
1377 **/
1378static void igb_get_hw_control(struct igb_adapter *adapter)
1379{
1380 struct e1000_hw *hw = &adapter->hw;
1381 u32 ctrl_ext;
1382
1383 /* Let firmware know the driver has taken over */
1384 ctrl_ext = rd32(E1000_CTRL_EXT);
1385 wr32(E1000_CTRL_EXT,
1386 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1387}
1388
9d5c8243
AK		 1389/**
1390 * igb_configure - configure the hardware for RX and TX
1391 * @adapter: private board structure
1392 **/
1393static void igb_configure(struct igb_adapter *adapter)
1394{
1395 struct net_device *netdev = adapter->netdev;
1396 int i;
1397
1398 igb_get_hw_control(adapter);
ff41f8dc 1399 igb_set_rx_mode(netdev);
9d5c8243
AK		 1400
1401 igb_restore_vlan(adapter);
9d5c8243 1402
85b430b4 1403 igb_setup_tctl(adapter);
06cf2666 1404 igb_setup_mrqc(adapter);
9d5c8243 1405 igb_setup_rctl(adapter);
85b430b4
AD		 1406
1407 igb_configure_tx(adapter);
9d5c8243 1408 igb_configure_rx(adapter);
662d7205
AD		 1409
1410 igb_rx_fifo_flush_82575(&adapter->hw);
1411
c493ea45 1412 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1413 * at least 1 descriptor unused to make sure
1414 * next_to_use != next_to_clean */
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1416 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1417 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1418 }
9d5c8243
AK		 1419}
1420
88a268c1
NN		 1421/**
1422 * igb_power_up_link - Power up the phy/serdes link
1423 * @adapter: address of board private structure
1424 **/
1425void igb_power_up_link(struct igb_adapter *adapter)
1426{
1427 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428 igb_power_up_phy_copper(&adapter->hw);
1429 else
1430 igb_power_up_serdes_link_82575(&adapter->hw);
1431}
1432
1433/**
1434 * igb_power_down_link - Power down the phy/serdes link
1435 * @adapter: address of board private structure
1436 */
1437static void igb_power_down_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_down_phy_copper_82575(&adapter->hw);
1441 else
1442 igb_shutdown_serdes_link_82575(&adapter->hw);
1443}
9d5c8243
AK		 1444
1445/**
1446 * igb_up - Open the interface and prepare it to handle traffic
1447 * @adapter: board private structure
1448 **/
9d5c8243
AK		 1449int igb_up(struct igb_adapter *adapter)
1450{
1451 struct e1000_hw *hw = &adapter->hw;
1452 int i;
1453
1454 /* hardware has been reset, we need to reload some things */
1455 igb_configure(adapter);
1456
1457 clear_bit(__IGB_DOWN, &adapter->state);
1458
047e0030
AD		 1459 for (i = 0; i < adapter->num_q_vectors; i++) {
1460 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461 napi_enable(&q_vector->napi);
1462 }
844290e5 1463 if (adapter->msix_entries)
9d5c8243 1464 igb_configure_msix(adapter);
feeb2721
AD		 1465 else
1466 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1467
1468 /* Clear any pending interrupts. */
1469 rd32(E1000_ICR);
1470 igb_irq_enable(adapter);
1471
d4960307
AD		 1472 /* notify VFs that reset has been completed */
1473 if (adapter->vfs_allocated_count) {
1474 u32 reg_data = rd32(E1000_CTRL_EXT);
1475 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476 wr32(E1000_CTRL_EXT, reg_data);
1477 }
1478
4cb9be7a
JB		 1479 netif_tx_start_all_queues(adapter->netdev);
1480
25568a53
AD		 1481 /* start the watchdog. */
1482 hw->mac.get_link_status = 1;
1483 schedule_work(&adapter->watchdog_task);
1484
9d5c8243
AK		 1485 return 0;
1486}
1487
1488void igb_down(struct igb_adapter *adapter)
1489{
9d5c8243 1490 struct net_device *netdev = adapter->netdev;
330a6d6a 1491 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1492 u32 tctl, rctl;
1493 int i;
1494
1495 /* signal that we're down so the interrupt handler does not
1496 * reschedule our watchdog timer */
1497 set_bit(__IGB_DOWN, &adapter->state);
1498
1499 /* disable receives in the hardware */
1500 rctl = rd32(E1000_RCTL);
1501 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502 /* flush and sleep below */
1503
fd2ea0a7 1504 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1505
1506 /* disable transmits in the hardware */
1507 tctl = rd32(E1000_TCTL);
1508 tctl &= ~E1000_TCTL_EN;
1509 wr32(E1000_TCTL, tctl);
1510 /* flush both disables and wait for them to finish */
1511 wrfl();
1512 msleep(10);
1513
047e0030
AD		 1514 for (i = 0; i < adapter->num_q_vectors; i++) {
1515 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516 napi_disable(&q_vector->napi);
1517 }
9d5c8243 1518
9d5c8243
AK		 1519 igb_irq_disable(adapter);
1520
1521 del_timer_sync(&adapter->watchdog_timer);
1522 del_timer_sync(&adapter->phy_info_timer);
1523
9d5c8243 1524 netif_carrier_off(netdev);
04fe6358
AD		 1525
1526 /* record the stats before reset*/
1527 igb_update_stats(adapter);
1528
9d5c8243
AK		 1529 adapter->link_speed = 0;
1530 adapter->link_duplex = 0;
1531
3023682e
JK		 1532 if (!pci_channel_offline(adapter->pdev))
1533 igb_reset(adapter);
9d5c8243
AK		 1534 igb_clean_all_tx_rings(adapter);
1535 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1536#ifdef CONFIG_IGB_DCA
1537
1538 /* since we reset the hardware DCA settings were cleared */
1539 igb_setup_dca(adapter);
1540#endif
9d5c8243
AK		 1541}
1542
1543void igb_reinit_locked(struct igb_adapter *adapter)
1544{
1545 WARN_ON(in_interrupt());
1546 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1547 msleep(1);
1548 igb_down(adapter);
1549 igb_up(adapter);
1550 clear_bit(__IGB_RESETTING, &adapter->state);
1551}
1552
1553void igb_reset(struct igb_adapter *adapter)
1554{
090b1795 1555 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1556 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1557 struct e1000_mac_info *mac = &hw->mac;
1558 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1559 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1560 u16 hwm;
1561
1562 /* Repartition Pba for greater than 9k mtu
1563 * To take effect CTRL.RST is required.
1564 */
fa4dfae0 1565 switch (mac->type) {
d2ba2ed8 1566 case e1000_i350:
55cac248
AD		 1567 case e1000_82580:
1568 pba = rd32(E1000_RXPBS);
1569 pba = igb_rxpbs_adjust_82580(pba);
1570 break;
fa4dfae0 1571 case e1000_82576:
d249be54
AD		 1572 pba = rd32(E1000_RXPBS);
1573 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1574 break;
1575 case e1000_82575:
1576 default:
1577 pba = E1000_PBA_34K;
1578 break;
2d064c06 1579 }
9d5c8243 1580
2d064c06
AD		 1581 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582 (mac->type < e1000_82576)) {
9d5c8243
AK		 1583 /* adjust PBA for jumbo frames */
1584 wr32(E1000_PBA, pba);
1585
1586 /* To maintain wire speed transmits, the Tx FIFO should be
1587 * large enough to accommodate two full transmit packets,
1588 * rounded up to the next 1KB and expressed in KB. Likewise,
1589 * the Rx FIFO should be large enough to accommodate at least
1590 * one full receive packet and is similarly rounded up and
1591 * expressed in KB. */
1592 pba = rd32(E1000_PBA);
1593 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594 tx_space = pba >> 16;
1595 /* lower 16 bits has Rx packet buffer allocation size in KB */
1596 pba &= 0xffff;
1597 /* the tx fifo also stores 16 bytes of information about the tx
1598 * but don't include ethernet FCS because hardware appends it */
1599 min_tx_space = (adapter->max_frame_size +
85e8d004 1600 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1601 ETH_FCS_LEN) * 2;
1602 min_tx_space = ALIGN(min_tx_space, 1024);
1603 min_tx_space >>= 10;
1604 /* software strips receive CRC, so leave room for it */
1605 min_rx_space = adapter->max_frame_size;
1606 min_rx_space = ALIGN(min_rx_space, 1024);
1607 min_rx_space >>= 10;
1608
1609 /* If current Tx allocation is less than the min Tx FIFO size,
1610 * and the min Tx FIFO size is less than the current Rx FIFO
1611 * allocation, take space away from current Rx allocation */
1612 if (tx_space < min_tx_space &&
1613 ((min_tx_space - tx_space) < pba)) {
1614 pba = pba - (min_tx_space - tx_space);
1615
1616 /* if short on rx space, rx wins and must trump tx
1617 * adjustment */
1618 if (pba < min_rx_space)
1619 pba = min_rx_space;
1620 }
2d064c06 1621 wr32(E1000_PBA, pba);
9d5c8243 1622 }
9d5c8243
AK		 1623
1624 /* flow control settings */
1625 /* The high water mark must be low enough to fit one full frame
1626 * (or the size used for early receive) above it in the Rx FIFO.
1627 * Set it to the lower of:
1628 * - 90% of the Rx FIFO size, or
1629 * - the full Rx FIFO size minus one full frame */
1630 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1631 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1632
d405ea3e
AD		 1633 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1634 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1635 fc->pause_time = 0xFFFF;
1636 fc->send_xon = 1;
0cce119a 1637 fc->current_mode = fc->requested_mode;
9d5c8243 1638
4ae196df
AD		 1639 /* disable receive for all VFs and wait one second */
1640 if (adapter->vfs_allocated_count) {
1641 int i;
1642 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1643 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1644
1645 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1646 igb_ping_all_vfs(adapter);
4ae196df
AD		 1647
1648 /* disable transmits and receives */
1649 wr32(E1000_VFRE, 0);
1650 wr32(E1000_VFTE, 0);
1651 }
1652
9d5c8243 1653 /* Allow time for pending master requests to run */
330a6d6a 1654 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1655 wr32(E1000_WUC, 0);
1656
330a6d6a 1657 if (hw->mac.ops.init_hw(hw))
090b1795 1658 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1659
55cac248
AD		 1660 if (hw->mac.type == e1000_82580) {
1661 u32 reg = rd32(E1000_PCIEMISC);
1662 wr32(E1000_PCIEMISC,
1663 reg & ~E1000_PCIEMISC_LX_DECISION);
1664 }
88a268c1
NN		 1665 if (!netif_running(adapter->netdev))
1666 igb_power_down_link(adapter);
1667
9d5c8243
AK		 1668 igb_update_mng_vlan(adapter);
1669
1670 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1672
330a6d6a 1673 igb_get_phy_info(hw);
9d5c8243
AK		 1674}
1675
2e5c6922 1676static const struct net_device_ops igb_netdev_ops = {
559e9c49 1677 .ndo_open = igb_open,
2e5c6922 1678 .ndo_stop = igb_close,
00829823 1679 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1680 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1681 .ndo_set_rx_mode = igb_set_rx_mode,
1682 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1683 .ndo_set_mac_address = igb_set_mac,
1684 .ndo_change_mtu = igb_change_mtu,
1685 .ndo_do_ioctl = igb_ioctl,
1686 .ndo_tx_timeout = igb_tx_timeout,
1687 .ndo_validate_addr = eth_validate_addr,
1688 .ndo_vlan_rx_register = igb_vlan_rx_register,
1689 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1690 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1691 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1692 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1693 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1694 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1695#ifdef CONFIG_NET_POLL_CONTROLLER
1696 .ndo_poll_controller = igb_netpoll,
1697#endif
1698};
1699
9d5c8243
AK		 1700/**
1701 * igb_probe - Device Initialization Routine
1702 * @pdev: PCI device information struct
1703 * @ent: entry in igb_pci_tbl
1704 *
1705 * Returns 0 on success, negative on failure
1706 *
1707 * igb_probe initializes an adapter identified by a pci_dev structure.
1708 * The OS initialization, configuring of the adapter private structure,
1709 * and a hardware reset occur.
1710 **/
1711static int __devinit igb_probe(struct pci_dev *pdev,
1712 const struct pci_device_id *ent)
1713{
1714 struct net_device *netdev;
1715 struct igb_adapter *adapter;
1716 struct e1000_hw *hw;
4337e993
AD		 1717 u16 eeprom_data = 0;
1718 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1719 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720 unsigned long mmio_start, mmio_len;
2d6a5e95 1721 int err, pci_using_dac;
9d5c8243
AK		 1722 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1723 u32 part_num;
1724
bded64a7
AG		 1725 /* Catch broken hardware that put the wrong VF device ID in
1726 * the PCIe SR-IOV capability.
1727 */
1728 if (pdev->is_virtfn) {
1729 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1730 pci_name(pdev), pdev->vendor, pdev->device);
1731 return -EINVAL;
1732 }
1733
aed5dec3 1734 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1735 if (err)
1736 return err;
1737
1738 pci_using_dac = 0;
59d71989 1739 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1740 if (!err) {
59d71989 1741 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1742 if (!err)
1743 pci_using_dac = 1;
1744 } else {
59d71989 1745 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1746 if (err) {
59d71989 1747 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1748 if (err) {
1749 dev_err(&pdev->dev, "No usable DMA "
1750 "configuration, aborting\n");
1751 goto err_dma;
1752 }
1753 }
1754 }
1755
aed5dec3
AD		 1756 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1757 IORESOURCE_MEM),
1758 igb_driver_name);
9d5c8243
AK		 1759 if (err)
1760 goto err_pci_reg;
1761
19d5afd4 1762 pci_enable_pcie_error_reporting(pdev);
40a914fa 1763
9d5c8243 1764 pci_set_master(pdev);
c682fc23 1765 pci_save_state(pdev);
9d5c8243
AK		 1766
1767 err = -ENOMEM;
1bfaf07b
AD		 1768 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1769 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1770 if (!netdev)
1771 goto err_alloc_etherdev;
1772
1773 SET_NETDEV_DEV(netdev, &pdev->dev);
1774
1775 pci_set_drvdata(pdev, netdev);
1776 adapter = netdev_priv(netdev);
1777 adapter->netdev = netdev;
1778 adapter->pdev = pdev;
1779 hw = &adapter->hw;
1780 hw->back = adapter;
1781 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1782
1783 mmio_start = pci_resource_start(pdev, 0);
1784 mmio_len = pci_resource_len(pdev, 0);
1785
1786 err = -EIO;
28b0759c
AD		 1787 hw->hw_addr = ioremap(mmio_start, mmio_len);
1788 if (!hw->hw_addr)
9d5c8243
AK		 1789 goto err_ioremap;
1790
2e5c6922 1791 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1792 igb_set_ethtool_ops(netdev);
9d5c8243 1793 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1794
1795 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1796
1797 netdev->mem_start = mmio_start;
1798 netdev->mem_end = mmio_start + mmio_len;
1799
9d5c8243
AK		 1800 /* PCI config space info */
1801 hw->vendor_id = pdev->vendor;
1802 hw->device_id = pdev->device;
1803 hw->revision_id = pdev->revision;
1804 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1805 hw->subsystem_device_id = pdev->subsystem_device;
1806
9d5c8243
AK		 1807 /* Copy the default MAC, PHY and NVM function pointers */
1808 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1809 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1810 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1811 /* Initialize skew-specific constants */
1812 err = ei->get_invariants(hw);
1813 if (err)
450c87c8 1814 goto err_sw_init;
9d5c8243 1815
450c87c8 1816 /* setup the private structure */
9d5c8243
AK		 1817 err = igb_sw_init(adapter);
1818 if (err)
1819 goto err_sw_init;
1820
1821 igb_get_bus_info_pcie(hw);
1822
1823 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1824
1825 /* Copper options */
1826 if (hw->phy.media_type == e1000_media_type_copper) {
1827 hw->phy.mdix = AUTO_ALL_MODES;
1828 hw->phy.disable_polarity_correction = false;
1829 hw->phy.ms_type = e1000_ms_hw_default;
1830 }
1831
1832 if (igb_check_reset_block(hw))
1833 dev_info(&pdev->dev,
1834 "PHY reset is blocked due to SOL/IDER session.\n");
1835
1836 netdev->features = NETIF_F_SG |
7d8eb29e 1837 NETIF_F_IP_CSUM |
9d5c8243
AK		 1838 NETIF_F_HW_VLAN_TX |
1839 NETIF_F_HW_VLAN_RX |
1840 NETIF_F_HW_VLAN_FILTER;
1841
7d8eb29e 1842 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1843 netdev->features |= NETIF_F_TSO;
9d5c8243 1844 netdev->features |= NETIF_F_TSO6;
5c0999b7 1845 netdev->features |= NETIF_F_GRO;
d3352520 1846
48f29ffc
JK		 1847 netdev->vlan_features |= NETIF_F_TSO;
1848 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1849 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1850 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1851 netdev->vlan_features |= NETIF_F_SG;
1852
9d5c8243
AK		 1853 if (pci_using_dac)
1854 netdev->features |= NETIF_F_HIGHDMA;
1855
5b043fb0 1856 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1857 netdev->features |= NETIF_F_SCTP_CSUM;
1858
330a6d6a 1859 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1860
1861 /* before reading the NVM, reset the controller to put the device in a
1862 * known good starting state */
1863 hw->mac.ops.reset_hw(hw);
1864
1865 /* make sure the NVM is good */
1866 if (igb_validate_nvm_checksum(hw) < 0) {
1867 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1868 err = -EIO;
1869 goto err_eeprom;
1870 }
1871
1872 /* copy the MAC address out of the NVM */
1873 if (hw->mac.ops.read_mac_addr(hw))
1874 dev_err(&pdev->dev, "NVM Read Error\n");
1875
1876 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1877 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1878
1879 if (!is_valid_ether_addr(netdev->perm_addr)) {
1880 dev_err(&pdev->dev, "Invalid MAC Address\n");
1881 err = -EIO;
1882 goto err_eeprom;
1883 }
1884
0e340485
AD		 1885 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1886 (unsigned long) adapter);
1887 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1888 (unsigned long) adapter);
9d5c8243
AK		 1889
1890 INIT_WORK(&adapter->reset_task, igb_reset_task);
1891 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1892
450c87c8 1893 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1894 adapter->fc_autoneg = true;
1895 hw->mac.autoneg = true;
1896 hw->phy.autoneg_advertised = 0x2f;
1897
0cce119a
AD		 1898 hw->fc.requested_mode = e1000_fc_default;
1899 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1900
9d5c8243
AK		 1901 igb_validate_mdi_setting(hw);
1902
9d5c8243
AK		 1903 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1904 * enable the ACPI Magic Packet filter
1905 */
1906
a2cf8b6c 1907 if (hw->bus.func == 0)
312c75ae 1908 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1909 else if (hw->mac.type == e1000_82580)
1910 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1911 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1912 &eeprom_data);
a2cf8b6c
AD		 1913 else if (hw->bus.func == 1)
1914 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1915
1916 if (eeprom_data & eeprom_apme_mask)
1917 adapter->eeprom_wol |= E1000_WUFC_MAG;
1918
1919 /* now that we have the eeprom settings, apply the special cases where
1920 * the eeprom may be wrong or the board simply won't support wake on
1921 * lan on a particular port */
1922 switch (pdev->device) {
1923 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1924 adapter->eeprom_wol = 0;
1925 break;
1926 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1927 case E1000_DEV_ID_82576_FIBER:
1928 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1929 /* Wake events only supported on port A for dual fiber
1930 * regardless of eeprom setting */
1931 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1932 adapter->eeprom_wol = 0;
1933 break;
c8ea5ea9 1934 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1935 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1936 /* if quad port adapter, disable WoL on all but port A */
1937 if (global_quad_port_a != 0)
1938 adapter->eeprom_wol = 0;
1939 else
1940 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1941 /* Reset for multiple quad port adapters */
1942 if (++global_quad_port_a == 4)
1943 global_quad_port_a = 0;
1944 break;
9d5c8243
AK		 1945 }
1946
1947 /* initialize the wol settings based on the eeprom settings */
1948 adapter->wol = adapter->eeprom_wol;
e1b86d84 1949 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1950
1951 /* reset the hardware with the new settings */
1952 igb_reset(adapter);
1953
1954 /* let the f/w know that the h/w is now under the control of the
1955 * driver. */
1956 igb_get_hw_control(adapter);
1957
9d5c8243
AK		 1958 strcpy(netdev->name, "eth%d");
1959 err = register_netdev(netdev);
1960 if (err)
1961 goto err_register;
1962
b168dfc5
JB		 1963 /* carrier off reporting is important to ethtool even BEFORE open */
1964 netif_carrier_off(netdev);
1965
421e02f0 1966#ifdef CONFIG_IGB_DCA
bbd98fe4 1967 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1968 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1969 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1970 igb_setup_dca(adapter);
1971 }
fe4506b6 1972
38c845c7 1973#endif
9d5c8243
AK		 1974 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1975 /* print bus type/speed/width info */
7c510e4b 1976 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1977 netdev->name,
559e9c49 1978 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1979 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1980 "unknown"),
59c3de89
AD		 1981 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1982 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1983 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1984 "unknown"),
7c510e4b 1985 netdev->dev_addr);
9d5c8243
AK		 1986
1987 igb_read_part_num(hw, &part_num);
1988 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1989 (part_num >> 8), (part_num & 0xff));
1990
1991 dev_info(&pdev->dev,
1992 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1993 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1994 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1995 adapter->num_rx_queues, adapter->num_tx_queues);
1996
9d5c8243
AK		 1997 return 0;
1998
1999err_register:
2000 igb_release_hw_control(adapter);
2001err_eeprom:
2002 if (!igb_check_reset_block(hw))
f5f4cf08 2003 igb_reset_phy(hw);
9d5c8243
AK		 2004
2005 if (hw->flash_address)
2006 iounmap(hw->flash_address);
9d5c8243 2007err_sw_init:
047e0030 2008 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2009 iounmap(hw->hw_addr);
2010err_ioremap:
2011 free_netdev(netdev);
2012err_alloc_etherdev:
559e9c49
AD		 2013 pci_release_selected_regions(pdev,
2014 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2015err_pci_reg:
2016err_dma:
2017 pci_disable_device(pdev);
2018 return err;
2019}
2020
2021/**
2022 * igb_remove - Device Removal Routine
2023 * @pdev: PCI device information struct
2024 *
2025 * igb_remove is called by the PCI subsystem to alert the driver
2026 * that it should release a PCI device. The could be caused by a
2027 * Hot-Plug event, or because the driver is going to be removed from
2028 * memory.
2029 **/
2030static void __devexit igb_remove(struct pci_dev *pdev)
2031{
2032 struct net_device *netdev = pci_get_drvdata(pdev);
2033 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2034 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2035
2036 /* flush_scheduled work may reschedule our watchdog task, so
2037 * explicitly disable watchdog tasks from being rescheduled */
2038 set_bit(__IGB_DOWN, &adapter->state);
2039 del_timer_sync(&adapter->watchdog_timer);
2040 del_timer_sync(&adapter->phy_info_timer);
2041
2042 flush_scheduled_work();
2043
421e02f0 2044#ifdef CONFIG_IGB_DCA
7dfc16fa 2045 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2046 dev_info(&pdev->dev, "DCA disabled\n");
2047 dca_remove_requester(&pdev->dev);
7dfc16fa 2048 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2049 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2050 }
2051#endif
2052
9d5c8243
AK		 2053 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2054 * would have already happened in close and is redundant. */
2055 igb_release_hw_control(adapter);
2056
2057 unregister_netdev(netdev);
2058
047e0030 2059 igb_clear_interrupt_scheme(adapter);
9d5c8243 2060
37680117
AD		 2061#ifdef CONFIG_PCI_IOV
2062 /* reclaim resources allocated to VFs */
2063 if (adapter->vf_data) {
2064 /* disable iov and allow time for transactions to clear */
2065 pci_disable_sriov(pdev);
2066 msleep(500);
2067
2068 kfree(adapter->vf_data);
2069 adapter->vf_data = NULL;
2070 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2071 msleep(100);
2072 dev_info(&pdev->dev, "IOV Disabled\n");
2073 }
2074#endif
559e9c49 2075
28b0759c
AD		 2076 iounmap(hw->hw_addr);
2077 if (hw->flash_address)
2078 iounmap(hw->flash_address);
559e9c49
AD		 2079 pci_release_selected_regions(pdev,
2080 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2081
2082 free_netdev(netdev);
2083
19d5afd4 2084 pci_disable_pcie_error_reporting(pdev);
40a914fa 2085
9d5c8243
AK		 2086 pci_disable_device(pdev);
2087}
2088
a6b623e0
AD		 2089/**
2090 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2091 * @adapter: board private structure to initialize
2092 *
2093 * This function initializes the vf specific data storage and then attempts to
2094 * allocate the VFs. The reason for ordering it this way is because it is much
2095 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2096 * the memory for the VFs.
2097 **/
2098static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2099{
2100#ifdef CONFIG_PCI_IOV
2101 struct pci_dev *pdev = adapter->pdev;
2102
2103 if (adapter->vfs_allocated_count > 7)
2104 adapter->vfs_allocated_count = 7;
2105
2106 if (adapter->vfs_allocated_count) {
2107 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2108 sizeof(struct vf_data_storage),
2109 GFP_KERNEL);
2110 /* if allocation failed then we do not support SR-IOV */
2111 if (!adapter->vf_data) {
2112 adapter->vfs_allocated_count = 0;
2113 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2114 "Data Storage\n");
2115 }
2116 }
2117
2118 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2119 kfree(adapter->vf_data);
2120 adapter->vf_data = NULL;
2121#endif /* CONFIG_PCI_IOV */
2122 adapter->vfs_allocated_count = 0;
2123#ifdef CONFIG_PCI_IOV
2124 } else {
2125 unsigned char mac_addr[ETH_ALEN];
2126 int i;
2127 dev_info(&pdev->dev, "%d vfs allocated\n",
2128 adapter->vfs_allocated_count);
2129 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2130 random_ether_addr(mac_addr);
2131 igb_set_vf_mac(adapter, i, mac_addr);
2132 }
2133 }
2134#endif /* CONFIG_PCI_IOV */
2135}
2136
115f459a
AD		 2137
2138/**
2139 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2140 * @adapter: board private structure to initialize
2141 *
2142 * igb_init_hw_timer initializes the function pointer and values for the hw
2143 * timer found in hardware.
2144 **/
2145static void igb_init_hw_timer(struct igb_adapter *adapter)
2146{
2147 struct e1000_hw *hw = &adapter->hw;
2148
2149 switch (hw->mac.type) {
d2ba2ed8 2150 case e1000_i350:
55cac248
AD		 2151 case e1000_82580:
2152 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2153 adapter->cycles.read = igb_read_clock;
2154 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2155 adapter->cycles.mult = 1;
2156 /*
2157 * The 82580 timesync updates the system timer every 8ns by 8ns
2158 * and the value cannot be shifted. Instead we need to shift
2159 * the registers to generate a 64bit timer value. As a result
2160 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2161 * 24 in order to generate a larger value for synchronization.
2162 */
2163 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2164 /* disable system timer temporarily by setting bit 31 */
2165 wr32(E1000_TSAUXC, 0x80000000);
2166 wrfl();
2167
2168 /* Set registers so that rollover occurs soon to test this. */
2169 wr32(E1000_SYSTIMR, 0x00000000);
2170 wr32(E1000_SYSTIML, 0x80000000);
2171 wr32(E1000_SYSTIMH, 0x000000FF);
2172 wrfl();
2173
2174 /* enable system timer by clearing bit 31 */
2175 wr32(E1000_TSAUXC, 0x0);
2176 wrfl();
2177
2178 timecounter_init(&adapter->clock,
2179 &adapter->cycles,
2180 ktime_to_ns(ktime_get_real()));
2181 /*
2182 * Synchronize our NIC clock against system wall clock. NIC
2183 * time stamp reading requires ~3us per sample, each sample
2184 * was pretty stable even under load => only require 10
2185 * samples for each offset comparison.
2186 */
2187 memset(&adapter->compare, 0, sizeof(adapter->compare));
2188 adapter->compare.source = &adapter->clock;
2189 adapter->compare.target = ktime_get_real;
2190 adapter->compare.num_samples = 10;
2191 timecompare_update(&adapter->compare, 0);
2192 break;
115f459a
AD		 2193 case e1000_82576:
2194 /*
2195 * Initialize hardware timer: we keep it running just in case
2196 * that some program needs it later on.
2197 */
2198 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2199 adapter->cycles.read = igb_read_clock;
2200 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2201 adapter->cycles.mult = 1;
2202 /**
2203 * Scale the NIC clock cycle by a large factor so that
2204 * relatively small clock corrections can be added or
2205 * substracted at each clock tick. The drawbacks of a large
2206 * factor are a) that the clock register overflows more quickly
2207 * (not such a big deal) and b) that the increment per tick has
2208 * to fit into 24 bits. As a result we need to use a shift of
2209 * 19 so we can fit a value of 16 into the TIMINCA register.
2210 */
2211 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2212 wr32(E1000_TIMINCA,
2213 (1 << E1000_TIMINCA_16NS_SHIFT) |
2214 (16 << IGB_82576_TSYNC_SHIFT));
2215
2216 /* Set registers so that rollover occurs soon to test this. */
2217 wr32(E1000_SYSTIML, 0x00000000);
2218 wr32(E1000_SYSTIMH, 0xFF800000);
2219 wrfl();
2220
2221 timecounter_init(&adapter->clock,
2222 &adapter->cycles,
2223 ktime_to_ns(ktime_get_real()));
2224 /*
2225 * Synchronize our NIC clock against system wall clock. NIC
2226 * time stamp reading requires ~3us per sample, each sample
2227 * was pretty stable even under load => only require 10
2228 * samples for each offset comparison.
2229 */
2230 memset(&adapter->compare, 0, sizeof(adapter->compare));
2231 adapter->compare.source = &adapter->clock;
2232 adapter->compare.target = ktime_get_real;
2233 adapter->compare.num_samples = 10;
2234 timecompare_update(&adapter->compare, 0);
2235 break;
2236 case e1000_82575:
2237 /* 82575 does not support timesync */
2238 default:
2239 break;
2240 }
2241
2242}
2243
9d5c8243
AK		 2244/**
2245 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2246 * @adapter: board private structure to initialize
2247 *
2248 * igb_sw_init initializes the Adapter private data structure.
2249 * Fields are initialized based on PCI device information and
2250 * OS network device settings (MTU size).
2251 **/
2252static int __devinit igb_sw_init(struct igb_adapter *adapter)
2253{
2254 struct e1000_hw *hw = &adapter->hw;
2255 struct net_device *netdev = adapter->netdev;
2256 struct pci_dev *pdev = adapter->pdev;
2257
2258 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2259
68fd9910
AD		 2260 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2261 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2262 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2263 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2264
9d5c8243
AK		 2265 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2266 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2267
a6b623e0
AD		 2268#ifdef CONFIG_PCI_IOV
2269 if (hw->mac.type == e1000_82576)
2270 adapter->vfs_allocated_count = max_vfs;
2271
2272#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2273 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2274
2275 /*
2276 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2277 * then we should combine the queues into a queue pair in order to
2278 * conserve interrupts due to limited supply
2279 */
2280 if ((adapter->rss_queues > 4) ||
2281 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2282 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2283
a6b623e0 2284 /* This call may decrease the number of queues */
047e0030 2285 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2286 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2287 return -ENOMEM;
2288 }
2289
115f459a 2290 igb_init_hw_timer(adapter);
a6b623e0
AD		 2291 igb_probe_vfs(adapter);
2292
9d5c8243
AK		 2293 /* Explicitly disable IRQ since the NIC can be in any state. */
2294 igb_irq_disable(adapter);
2295
2296 set_bit(__IGB_DOWN, &adapter->state);
2297 return 0;
2298}
2299
2300/**
2301 * igb_open - Called when a network interface is made active
2302 * @netdev: network interface device structure
2303 *
2304 * Returns 0 on success, negative value on failure
2305 *
2306 * The open entry point is called when a network interface is made
2307 * active by the system (IFF_UP). At this point all resources needed
2308 * for transmit and receive operations are allocated, the interrupt
2309 * handler is registered with the OS, the watchdog timer is started,
2310 * and the stack is notified that the interface is ready.
2311 **/
2312static int igb_open(struct net_device *netdev)
2313{
2314 struct igb_adapter *adapter = netdev_priv(netdev);
2315 struct e1000_hw *hw = &adapter->hw;
2316 int err;
2317 int i;
2318
2319 /* disallow open during test */
2320 if (test_bit(__IGB_TESTING, &adapter->state))
2321 return -EBUSY;
2322
b168dfc5
JB		 2323 netif_carrier_off(netdev);
2324
9d5c8243
AK		 2325 /* allocate transmit descriptors */
2326 err = igb_setup_all_tx_resources(adapter);
2327 if (err)
2328 goto err_setup_tx;
2329
2330 /* allocate receive descriptors */
2331 err = igb_setup_all_rx_resources(adapter);
2332 if (err)
2333 goto err_setup_rx;
2334
88a268c1 2335 igb_power_up_link(adapter);
9d5c8243 2336
9d5c8243
AK		 2337 /* before we allocate an interrupt, we must be ready to handle it.
2338 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2339 * as soon as we call pci_request_irq, so we have to setup our
2340 * clean_rx handler before we do so. */
2341 igb_configure(adapter);
2342
2343 err = igb_request_irq(adapter);
2344 if (err)
2345 goto err_req_irq;
2346
2347 /* From here on the code is the same as igb_up() */
2348 clear_bit(__IGB_DOWN, &adapter->state);
2349
047e0030
AD		 2350 for (i = 0; i < adapter->num_q_vectors; i++) {
2351 struct igb_q_vector *q_vector = adapter->q_vector[i];
2352 napi_enable(&q_vector->napi);
2353 }
9d5c8243
AK		 2354
2355 /* Clear any pending interrupts. */
2356 rd32(E1000_ICR);
844290e5
PW		 2357
2358 igb_irq_enable(adapter);
2359
d4960307
AD		 2360 /* notify VFs that reset has been completed */
2361 if (adapter->vfs_allocated_count) {
2362 u32 reg_data = rd32(E1000_CTRL_EXT);
2363 reg_data |= E1000_CTRL_EXT_PFRSTD;
2364 wr32(E1000_CTRL_EXT, reg_data);
2365 }
2366
d55b53ff
JK		 2367 netif_tx_start_all_queues(netdev);
2368
25568a53
AD		 2369 /* start the watchdog. */
2370 hw->mac.get_link_status = 1;
2371 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2372
2373 return 0;
2374
2375err_req_irq:
2376 igb_release_hw_control(adapter);
88a268c1 2377 igb_power_down_link(adapter);
9d5c8243
AK		 2378 igb_free_all_rx_resources(adapter);
2379err_setup_rx:
2380 igb_free_all_tx_resources(adapter);
2381err_setup_tx:
2382 igb_reset(adapter);
2383
2384 return err;
2385}
2386
2387/**
2388 * igb_close - Disables a network interface
2389 * @netdev: network interface device structure
2390 *
2391 * Returns 0, this is not allowed to fail
2392 *
2393 * The close entry point is called when an interface is de-activated
2394 * by the OS. The hardware is still under the driver's control, but
2395 * needs to be disabled. A global MAC reset is issued to stop the
2396 * hardware, and all transmit and receive resources are freed.
2397 **/
2398static int igb_close(struct net_device *netdev)
2399{
2400 struct igb_adapter *adapter = netdev_priv(netdev);
2401
2402 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2403 igb_down(adapter);
2404
2405 igb_free_irq(adapter);
2406
2407 igb_free_all_tx_resources(adapter);
2408 igb_free_all_rx_resources(adapter);
2409
9d5c8243
AK		 2410 return 0;
2411}
2412
2413/**
2414 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2415 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2416 *
2417 * Return 0 on success, negative on failure
2418 **/
80785298 2419int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2420{
59d71989 2421 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2422 int size;
2423
2424 size = sizeof(struct igb_buffer) * tx_ring->count;
2425 tx_ring->buffer_info = vmalloc(size);
2426 if (!tx_ring->buffer_info)
2427 goto err;
2428 memset(tx_ring->buffer_info, 0, size);
2429
2430 /* round up to nearest 4K */
85e8d004 2431 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2432 tx_ring->size = ALIGN(tx_ring->size, 4096);
2433
59d71989
AD		 2434 tx_ring->desc = dma_alloc_coherent(dev,
2435 tx_ring->size,
2436 &tx_ring->dma,
2437 GFP_KERNEL);
9d5c8243
AK		 2438
2439 if (!tx_ring->desc)
2440 goto err;
2441
9d5c8243
AK		 2442 tx_ring->next_to_use = 0;
2443 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2444 return 0;
2445
2446err:
2447 vfree(tx_ring->buffer_info);
59d71989 2448 dev_err(dev,
9d5c8243
AK		 2449 "Unable to allocate memory for the transmit descriptor ring\n");
2450 return -ENOMEM;
2451}
2452
2453/**
2454 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2455 * (Descriptors) for all queues
2456 * @adapter: board private structure
2457 *
2458 * Return 0 on success, negative on failure
2459 **/
2460static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2461{
439705e1 2462 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2463 int i, err = 0;
2464
2465 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2466 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2467 if (err) {
439705e1 2468 dev_err(&pdev->dev,
9d5c8243
AK		 2469 "Allocation for Tx Queue %u failed\n", i);
2470 for (i--; i >= 0; i--)
3025a446 2471 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2472 break;
2473 }
2474 }
2475
a99955fc 2476 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2477 int r_idx = i % adapter->num_tx_queues;
3025a446 2478 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2479 }
9d5c8243
AK		 2480 return err;
2481}
2482
2483/**
85b430b4
AD		 2484 * igb_setup_tctl - configure the transmit control registers
2485 * @adapter: Board private structure
9d5c8243 2486 **/
d7ee5b3a 2487void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2488{
9d5c8243
AK		 2489 struct e1000_hw *hw = &adapter->hw;
2490 u32 tctl;
9d5c8243 2491
85b430b4
AD		 2492 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2493 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2494
2495 /* Program the Transmit Control Register */
9d5c8243
AK		 2496 tctl = rd32(E1000_TCTL);
2497 tctl &= ~E1000_TCTL_CT;
2498 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2499 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2500
2501 igb_config_collision_dist(hw);
2502
9d5c8243
AK		 2503 /* Enable transmits */
2504 tctl |= E1000_TCTL_EN;
2505
2506 wr32(E1000_TCTL, tctl);
2507}
2508
85b430b4
AD		 2509/**
2510 * igb_configure_tx_ring - Configure transmit ring after Reset
2511 * @adapter: board private structure
2512 * @ring: tx ring to configure
2513 *
2514 * Configure a transmit ring after a reset.
2515 **/
d7ee5b3a
AD		 2516void igb_configure_tx_ring(struct igb_adapter *adapter,
2517 struct igb_ring *ring)
85b430b4
AD		 2518{
2519 struct e1000_hw *hw = &adapter->hw;
2520 u32 txdctl;
2521 u64 tdba = ring->dma;
2522 int reg_idx = ring->reg_idx;
2523
2524 /* disable the queue */
2525 txdctl = rd32(E1000_TXDCTL(reg_idx));
2526 wr32(E1000_TXDCTL(reg_idx),
2527 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2528 wrfl();
2529 mdelay(10);
2530
2531 wr32(E1000_TDLEN(reg_idx),
2532 ring->count * sizeof(union e1000_adv_tx_desc));
2533 wr32(E1000_TDBAL(reg_idx),
2534 tdba & 0x00000000ffffffffULL);
2535 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2536
fce99e34
AD		 2537 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2538 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2539 writel(0, ring->head);
2540 writel(0, ring->tail);
85b430b4
AD		 2541
2542 txdctl |= IGB_TX_PTHRESH;
2543 txdctl |= IGB_TX_HTHRESH << 8;
2544 txdctl |= IGB_TX_WTHRESH << 16;
2545
2546 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2547 wr32(E1000_TXDCTL(reg_idx), txdctl);
2548}
2549
2550/**
2551 * igb_configure_tx - Configure transmit Unit after Reset
2552 * @adapter: board private structure
2553 *
2554 * Configure the Tx unit of the MAC after a reset.
2555 **/
2556static void igb_configure_tx(struct igb_adapter *adapter)
2557{
2558 int i;
2559
2560 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2561 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2562}
2563
9d5c8243
AK		 2564/**
2565 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2566 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2567 *
2568 * Returns 0 on success, negative on failure
2569 **/
80785298 2570int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2571{
59d71989 2572 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2573 int size, desc_len;
2574
2575 size = sizeof(struct igb_buffer) * rx_ring->count;
2576 rx_ring->buffer_info = vmalloc(size);
2577 if (!rx_ring->buffer_info)
2578 goto err;
2579 memset(rx_ring->buffer_info, 0, size);
2580
2581 desc_len = sizeof(union e1000_adv_rx_desc);
2582
2583 /* Round up to nearest 4K */
2584 rx_ring->size = rx_ring->count * desc_len;
2585 rx_ring->size = ALIGN(rx_ring->size, 4096);
2586
59d71989
AD		 2587 rx_ring->desc = dma_alloc_coherent(dev,
2588 rx_ring->size,
2589 &rx_ring->dma,
2590 GFP_KERNEL);
9d5c8243
AK		 2591
2592 if (!rx_ring->desc)
2593 goto err;
2594
2595 rx_ring->next_to_clean = 0;
2596 rx_ring->next_to_use = 0;
9d5c8243 2597
9d5c8243
AK		 2598 return 0;
2599
2600err:
2601 vfree(rx_ring->buffer_info);
439705e1 2602 rx_ring->buffer_info = NULL;
59d71989
AD		 2603 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2604 " ring\n");
9d5c8243
AK		 2605 return -ENOMEM;
2606}
2607
2608/**
2609 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2610 * (Descriptors) for all queues
2611 * @adapter: board private structure
2612 *
2613 * Return 0 on success, negative on failure
2614 **/
2615static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2616{
439705e1 2617 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2618 int i, err = 0;
2619
2620 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2621 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2622 if (err) {
439705e1 2623 dev_err(&pdev->dev,
9d5c8243
AK		 2624 "Allocation for Rx Queue %u failed\n", i);
2625 for (i--; i >= 0; i--)
3025a446 2626 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2627 break;
2628 }
2629 }
2630
2631 return err;
2632}
2633
06cf2666
AD		 2634/**
2635 * igb_setup_mrqc - configure the multiple receive queue control registers
2636 * @adapter: Board private structure
2637 **/
2638static void igb_setup_mrqc(struct igb_adapter *adapter)
2639{
2640 struct e1000_hw *hw = &adapter->hw;
2641 u32 mrqc, rxcsum;
2642 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2643 union e1000_reta {
2644 u32 dword;
2645 u8 bytes[4];
2646 } reta;
2647 static const u8 rsshash[40] = {
2648 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2649 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2650 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2651 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2652
2653 /* Fill out hash function seeds */
2654 for (j = 0; j < 10; j++) {
2655 u32 rsskey = rsshash[(j * 4)];
2656 rsskey |= rsshash[(j * 4) + 1] << 8;
2657 rsskey |= rsshash[(j * 4) + 2] << 16;
2658 rsskey |= rsshash[(j * 4) + 3] << 24;
2659 array_wr32(E1000_RSSRK(0), j, rsskey);
2660 }
2661
a99955fc 2662 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2663
2664 if (adapter->vfs_allocated_count) {
2665 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2666 switch (hw->mac.type) {
d2ba2ed8 2667 case e1000_i350:
55cac248
AD		 2668 case e1000_82580:
2669 num_rx_queues = 1;
2670 shift = 0;
2671 break;
06cf2666
AD		 2672 case e1000_82576:
2673 shift = 3;
2674 num_rx_queues = 2;
2675 break;
2676 case e1000_82575:
2677 shift = 2;
2678 shift2 = 6;
2679 default:
2680 break;
2681 }
2682 } else {
2683 if (hw->mac.type == e1000_82575)
2684 shift = 6;
2685 }
2686
2687 for (j = 0; j < (32 * 4); j++) {
2688 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2689 if (shift2)
2690 reta.bytes[j & 3] |= num_rx_queues << shift2;
2691 if ((j & 3) == 3)
2692 wr32(E1000_RETA(j >> 2), reta.dword);
2693 }
2694
2695 /*
2696 * Disable raw packet checksumming so that RSS hash is placed in
2697 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2698 * offloads as they are enabled by default
2699 */
2700 rxcsum = rd32(E1000_RXCSUM);
2701 rxcsum |= E1000_RXCSUM_PCSD;
2702
2703 if (adapter->hw.mac.type >= e1000_82576)
2704 /* Enable Receive Checksum Offload for SCTP */
2705 rxcsum |= E1000_RXCSUM_CRCOFL;
2706
2707 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2708 wr32(E1000_RXCSUM, rxcsum);
2709
2710 /* If VMDq is enabled then we set the appropriate mode for that, else
2711 * we default to RSS so that an RSS hash is calculated per packet even
2712 * if we are only using one queue */
2713 if (adapter->vfs_allocated_count) {
2714 if (hw->mac.type > e1000_82575) {
2715 /* Set the default pool for the PF's first queue */
2716 u32 vtctl = rd32(E1000_VT_CTL);
2717 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2718 E1000_VT_CTL_DISABLE_DEF_POOL);
2719 vtctl |= adapter->vfs_allocated_count <<
2720 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2721 wr32(E1000_VT_CTL, vtctl);
2722 }
a99955fc 2723 if (adapter->rss_queues > 1)
06cf2666
AD		 2724 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2725 else
2726 mrqc = E1000_MRQC_ENABLE_VMDQ;
2727 } else {
2728 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2729 }
2730 igb_vmm_control(adapter);
2731
2732 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2733 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2734 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2735 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2736 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2737 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2738 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2739 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2740
2741 wr32(E1000_MRQC, mrqc);
2742}
2743
9d5c8243
AK		 2744/**
2745 * igb_setup_rctl - configure the receive control registers
2746 * @adapter: Board private structure
2747 **/
d7ee5b3a 2748void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2749{
2750 struct e1000_hw *hw = &adapter->hw;
2751 u32 rctl;
9d5c8243
AK		 2752
2753 rctl = rd32(E1000_RCTL);
2754
2755 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2756 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2757
69d728ba 2758 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2759 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2760
87cb7e8c
AK		 2761 /*
2762 * enable stripping of CRC. It's unlikely this will break BMC
2763 * redirection as it did with e1000. Newer features require
2764 * that the HW strips the CRC.
73cd78f1 2765 */
87cb7e8c 2766 rctl |= E1000_RCTL_SECRC;
9d5c8243 2767
559e9c49 2768 /* disable store bad packets and clear size bits. */
ec54d7d6 2769 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2770
6ec43fe6
AD		 2771 /* enable LPE to prevent packets larger than max_frame_size */
2772 rctl |= E1000_RCTL_LPE;
9d5c8243 2773
952f72a8
AD		 2774 /* disable queue 0 to prevent tail write w/o re-config */
2775 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2776
e1739522
AD		 2777 /* Attention!!! For SR-IOV PF driver operations you must enable
2778 * queue drop for all VF and PF queues to prevent head of line blocking
2779 * if an un-trusted VF does not provide descriptors to hardware.
2780 */
2781 if (adapter->vfs_allocated_count) {
e1739522
AD		 2782 /* set all queue drop enable bits */
2783 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2784 }
2785
9d5c8243
AK		 2786 wr32(E1000_RCTL, rctl);
2787}
2788
7d5753f0
AD		 2789static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2790 int vfn)
2791{
2792 struct e1000_hw *hw = &adapter->hw;
2793 u32 vmolr;
2794
2795 /* if it isn't the PF check to see if VFs are enabled and
2796 * increase the size to support vlan tags */
2797 if (vfn < adapter->vfs_allocated_count &&
2798 adapter->vf_data[vfn].vlans_enabled)
2799 size += VLAN_TAG_SIZE;
2800
2801 vmolr = rd32(E1000_VMOLR(vfn));
2802 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2803 vmolr |= size | E1000_VMOLR_LPE;
2804 wr32(E1000_VMOLR(vfn), vmolr);
2805
2806 return 0;
2807}
2808
e1739522
AD		 2809/**
2810 * igb_rlpml_set - set maximum receive packet size
2811 * @adapter: board private structure
2812 *
2813 * Configure maximum receivable packet size.
2814 **/
2815static void igb_rlpml_set(struct igb_adapter *adapter)
2816{
2817 u32 max_frame_size = adapter->max_frame_size;
2818 struct e1000_hw *hw = &adapter->hw;
2819 u16 pf_id = adapter->vfs_allocated_count;
2820
2821 if (adapter->vlgrp)
2822 max_frame_size += VLAN_TAG_SIZE;
2823
2824 /* if vfs are enabled we set RLPML to the largest possible request
2825 * size and set the VMOLR RLPML to the size we need */
2826 if (pf_id) {
2827 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2828 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2829 }
2830
2831 wr32(E1000_RLPML, max_frame_size);
2832}
2833
8151d294
WM		 2834static inline void igb_set_vmolr(struct igb_adapter *adapter,
2835 int vfn, bool aupe)
7d5753f0
AD		 2836{
2837 struct e1000_hw *hw = &adapter->hw;
2838 u32 vmolr;
2839
2840 /*
2841 * This register exists only on 82576 and newer so if we are older then
2842 * we should exit and do nothing
2843 */
2844 if (hw->mac.type < e1000_82576)
2845 return;
2846
2847 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2848 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2849 if (aupe)
2850 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2851 else
2852 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2853
2854 /* clear all bits that might not be set */
2855 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2856
a99955fc 2857 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2858 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2859 /*
2860 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2861 * multicast packets
2862 */
2863 if (vfn <= adapter->vfs_allocated_count)
2864 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2865
2866 wr32(E1000_VMOLR(vfn), vmolr);
2867}
2868
85b430b4
AD		 2869/**
2870 * igb_configure_rx_ring - Configure a receive ring after Reset
2871 * @adapter: board private structure
2872 * @ring: receive ring to be configured
2873 *
2874 * Configure the Rx unit of the MAC after a reset.
2875 **/
d7ee5b3a
AD		 2876void igb_configure_rx_ring(struct igb_adapter *adapter,
2877 struct igb_ring *ring)
85b430b4
AD		 2878{
2879 struct e1000_hw *hw = &adapter->hw;
2880 u64 rdba = ring->dma;
2881 int reg_idx = ring->reg_idx;
952f72a8 2882 u32 srrctl, rxdctl;
85b430b4
AD		 2883
2884 /* disable the queue */
2885 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2886 wr32(E1000_RXDCTL(reg_idx),
2887 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2888
2889 /* Set DMA base address registers */
2890 wr32(E1000_RDBAL(reg_idx),
2891 rdba & 0x00000000ffffffffULL);
2892 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2893 wr32(E1000_RDLEN(reg_idx),
2894 ring->count * sizeof(union e1000_adv_rx_desc));
2895
2896 /* initialize head and tail */
fce99e34
AD		 2897 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2898 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2899 writel(0, ring->head);
2900 writel(0, ring->tail);
85b430b4 2901
952f72a8 2902 /* set descriptor configuration */
4c844851
AD		 2903 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2904 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2905 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2906#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2907 srrctl |= IGB_RXBUFFER_16384 >>
2908 E1000_SRRCTL_BSIZEPKT_SHIFT;
2909#else
2910 srrctl |= (PAGE_SIZE / 2) >>
2911 E1000_SRRCTL_BSIZEPKT_SHIFT;
2912#endif
2913 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2914 } else {
4c844851 2915 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2916 E1000_SRRCTL_BSIZEPKT_SHIFT;
2917 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2918 }
757b77e2
NN		 2919 if (hw->mac.type == e1000_82580)
2920 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2921 /* Only set Drop Enable if we are supporting multiple queues */
2922 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2923 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2924
2925 wr32(E1000_SRRCTL(reg_idx), srrctl);
2926
7d5753f0 2927 /* set filtering for VMDQ pools */
8151d294 2928 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2929
85b430b4
AD		 2930 /* enable receive descriptor fetching */
2931 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2932 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2933 rxdctl &= 0xFFF00000;
2934 rxdctl |= IGB_RX_PTHRESH;
2935 rxdctl |= IGB_RX_HTHRESH << 8;
2936 rxdctl |= IGB_RX_WTHRESH << 16;
2937 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2938}
2939
9d5c8243
AK		 2940/**
2941 * igb_configure_rx - Configure receive Unit after Reset
2942 * @adapter: board private structure
2943 *
2944 * Configure the Rx unit of the MAC after a reset.
2945 **/
2946static void igb_configure_rx(struct igb_adapter *adapter)
2947{
9107584e 2948 int i;
9d5c8243 2949
68d480c4
AD		 2950 /* set UTA to appropriate mode */
2951 igb_set_uta(adapter);
2952
26ad9178
AD		 2953 /* set the correct pool for the PF default MAC address in entry 0 */
2954 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2955 adapter->vfs_allocated_count);
2956
06cf2666
AD		 2957 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2958 * the Base and Length of the Rx Descriptor Ring */
2959 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2960 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2961}
2962
2963/**
2964 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2965 * @tx_ring: Tx descriptor ring for a specific queue
2966 *
2967 * Free all transmit software resources
2968 **/
68fd9910 2969void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2970{
3b644cf6 2971 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2972
2973 vfree(tx_ring->buffer_info);
2974 tx_ring->buffer_info = NULL;
2975
439705e1
AD		 2976 /* if not set, then don't free */
2977 if (!tx_ring->desc)
2978 return;
2979
59d71989
AD		 2980 dma_free_coherent(tx_ring->dev, tx_ring->size,
2981 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2982
2983 tx_ring->desc = NULL;
2984}
2985
2986/**
2987 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2988 * @adapter: board private structure
2989 *
2990 * Free all transmit software resources
2991 **/
2992static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2993{
2994 int i;
2995
2996 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2997 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2998}
2999
b1a436c3
AD		 3000void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3001 struct igb_buffer *buffer_info)
9d5c8243 3002{
6366ad33
AD		 3003 if (buffer_info->dma) {
3004 if (buffer_info->mapped_as_page)
59d71989 3005 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3006 buffer_info->dma,
3007 buffer_info->length,
59d71989 3008 DMA_TO_DEVICE);
6366ad33 3009 else
59d71989 3010 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3011 buffer_info->dma,
3012 buffer_info->length,
59d71989 3013 DMA_TO_DEVICE);
6366ad33
AD		 3014 buffer_info->dma = 0;
3015 }
9d5c8243
AK		 3016 if (buffer_info->skb) {
3017 dev_kfree_skb_any(buffer_info->skb);
3018 buffer_info->skb = NULL;
3019 }
3020 buffer_info->time_stamp = 0;
6366ad33
AD		 3021 buffer_info->length = 0;
3022 buffer_info->next_to_watch = 0;
3023 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3024}
3025
3026/**
3027 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3028 * @tx_ring: ring to be cleaned
3029 **/
3b644cf6 3030static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3031{
3032 struct igb_buffer *buffer_info;
3033 unsigned long size;
3034 unsigned int i;
3035
3036 if (!tx_ring->buffer_info)
3037 return;
3038 /* Free all the Tx ring sk_buffs */
3039
3040 for (i = 0; i < tx_ring->count; i++) {
3041 buffer_info = &tx_ring->buffer_info[i];
80785298 3042 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3043 }
3044
3045 size = sizeof(struct igb_buffer) * tx_ring->count;
3046 memset(tx_ring->buffer_info, 0, size);
3047
3048 /* Zero out the descriptor ring */
9d5c8243
AK		 3049 memset(tx_ring->desc, 0, tx_ring->size);
3050
3051 tx_ring->next_to_use = 0;
3052 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3053}
3054
3055/**
3056 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3057 * @adapter: board private structure
3058 **/
3059static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3060{
3061 int i;
3062
3063 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3064 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3065}
3066
3067/**
3068 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3069 * @rx_ring: ring to clean the resources from
3070 *
3071 * Free all receive software resources
3072 **/
68fd9910 3073void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3074{
3b644cf6 3075 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3076
3077 vfree(rx_ring->buffer_info);
3078 rx_ring->buffer_info = NULL;
3079
439705e1
AD		 3080 /* if not set, then don't free */
3081 if (!rx_ring->desc)
3082 return;
3083
59d71989
AD		 3084 dma_free_coherent(rx_ring->dev, rx_ring->size,
3085 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3086
3087 rx_ring->desc = NULL;
3088}
3089
3090/**
3091 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3092 * @adapter: board private structure
3093 *
3094 * Free all receive software resources
3095 **/
3096static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3097{
3098 int i;
3099
3100 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3101 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3102}
3103
3104/**
3105 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3106 * @rx_ring: ring to free buffers from
3107 **/
3b644cf6 3108static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3109{
3110 struct igb_buffer *buffer_info;
9d5c8243
AK		 3111 unsigned long size;
3112 unsigned int i;
3113
3114 if (!rx_ring->buffer_info)
3115 return;
439705e1 3116
9d5c8243
AK		 3117 /* Free all the Rx ring sk_buffs */
3118 for (i = 0; i < rx_ring->count; i++) {
3119 buffer_info = &rx_ring->buffer_info[i];
3120 if (buffer_info->dma) {
59d71989 3121 dma_unmap_single(rx_ring->dev,
80785298 3122 buffer_info->dma,
4c844851 3123 rx_ring->rx_buffer_len,
59d71989 3124 DMA_FROM_DEVICE);
9d5c8243
AK		 3125 buffer_info->dma = 0;
3126 }
3127
3128 if (buffer_info->skb) {
3129 dev_kfree_skb(buffer_info->skb);
3130 buffer_info->skb = NULL;
3131 }
6ec43fe6 3132 if (buffer_info->page_dma) {
59d71989 3133 dma_unmap_page(rx_ring->dev,
80785298 3134 buffer_info->page_dma,
6ec43fe6 3135 PAGE_SIZE / 2,
59d71989 3136 DMA_FROM_DEVICE);
6ec43fe6
AD		 3137 buffer_info->page_dma = 0;
3138 }
9d5c8243 3139 if (buffer_info->page) {
9d5c8243
AK		 3140 put_page(buffer_info->page);
3141 buffer_info->page = NULL;
bf36c1a0 3142 buffer_info->page_offset = 0;
9d5c8243
AK		 3143 }
3144 }
3145
9d5c8243
AK		 3146 size = sizeof(struct igb_buffer) * rx_ring->count;
3147 memset(rx_ring->buffer_info, 0, size);
3148
3149 /* Zero out the descriptor ring */
3150 memset(rx_ring->desc, 0, rx_ring->size);
3151
3152 rx_ring->next_to_clean = 0;
3153 rx_ring->next_to_use = 0;
9d5c8243
AK		 3154}
3155
3156/**
3157 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3158 * @adapter: board private structure
3159 **/
3160static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3161{
3162 int i;
3163
3164 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3165 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3166}
3167
3168/**
3169 * igb_set_mac - Change the Ethernet Address of the NIC
3170 * @netdev: network interface device structure
3171 * @p: pointer to an address structure
3172 *
3173 * Returns 0 on success, negative on failure
3174 **/
3175static int igb_set_mac(struct net_device *netdev, void *p)
3176{
3177 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3178 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3179 struct sockaddr *addr = p;
3180
3181 if (!is_valid_ether_addr(addr->sa_data))
3182 return -EADDRNOTAVAIL;
3183
3184 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3185 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3186
26ad9178
AD		 3187 /* set the correct pool for the new PF MAC address in entry 0 */
3188 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3189 adapter->vfs_allocated_count);
e1739522 3190
9d5c8243
AK		 3191 return 0;
3192}
3193
3194/**
68d480c4 3195 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3196 * @netdev: network interface device structure
3197 *
68d480c4
AD		 3198 * Writes multicast address list to the MTA hash table.
3199 * Returns: -ENOMEM on failure
3200 * 0 on no addresses written
3201 * X on writing X addresses to MTA
9d5c8243 3202 **/
68d480c4 3203static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3204{
3205 struct igb_adapter *adapter = netdev_priv(netdev);
3206 struct e1000_hw *hw = &adapter->hw;
22bedad3 3207 struct netdev_hw_addr *ha;
68d480c4 3208 u8 *mta_list;
9d5c8243
AK		 3209 int i;
3210
4cd24eaf 3211 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3212 /* nothing to program, so clear mc list */
3213 igb_update_mc_addr_list(hw, NULL, 0);
3214 igb_restore_vf_multicasts(adapter);
3215 return 0;
3216 }
9d5c8243 3217
4cd24eaf 3218 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3219 if (!mta_list)
3220 return -ENOMEM;
ff41f8dc 3221
68d480c4 3222 /* The shared function expects a packed array of only addresses. */
48e2f183 3223 i = 0;
22bedad3
JP		 3224 netdev_for_each_mc_addr(ha, netdev)
3225 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3226
68d480c4
AD		 3227 igb_update_mc_addr_list(hw, mta_list, i);
3228 kfree(mta_list);
3229
4cd24eaf 3230 return netdev_mc_count(netdev);
68d480c4
AD		 3231}
3232
3233/**
3234 * igb_write_uc_addr_list - write unicast addresses to RAR table
3235 * @netdev: network interface device structure
3236 *
3237 * Writes unicast address list to the RAR table.
3238 * Returns: -ENOMEM on failure/insufficient address space
3239 * 0 on no addresses written
3240 * X on writing X addresses to the RAR table
3241 **/
3242static int igb_write_uc_addr_list(struct net_device *netdev)
3243{
3244 struct igb_adapter *adapter = netdev_priv(netdev);
3245 struct e1000_hw *hw = &adapter->hw;
3246 unsigned int vfn = adapter->vfs_allocated_count;
3247 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3248 int count = 0;
3249
3250 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3251 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3252 return -ENOMEM;
9d5c8243 3253
32e7bfc4 3254 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3255 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3256
3257 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3258 if (!rar_entries)
3259 break;
26ad9178
AD		 3260 igb_rar_set_qsel(adapter, ha->addr,
3261 rar_entries--,
68d480c4
AD		 3262 vfn);
3263 count++;
ff41f8dc
AD		 3264 }
3265 }
3266 /* write the addresses in reverse order to avoid write combining */
3267 for (; rar_entries > 0 ; rar_entries--) {
3268 wr32(E1000_RAH(rar_entries), 0);
3269 wr32(E1000_RAL(rar_entries), 0);
3270 }
3271 wrfl();
3272
68d480c4
AD		 3273 return count;
3274}
3275
3276/**
3277 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3278 * @netdev: network interface device structure
3279 *
3280 * The set_rx_mode entry point is called whenever the unicast or multicast
3281 * address lists or the network interface flags are updated. This routine is
3282 * responsible for configuring the hardware for proper unicast, multicast,
3283 * promiscuous mode, and all-multi behavior.
3284 **/
3285static void igb_set_rx_mode(struct net_device *netdev)
3286{
3287 struct igb_adapter *adapter = netdev_priv(netdev);
3288 struct e1000_hw *hw = &adapter->hw;
3289 unsigned int vfn = adapter->vfs_allocated_count;
3290 u32 rctl, vmolr = 0;
3291 int count;
3292
3293 /* Check for Promiscuous and All Multicast modes */
3294 rctl = rd32(E1000_RCTL);
3295
3296 /* clear the effected bits */
3297 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3298
3299 if (netdev->flags & IFF_PROMISC) {
3300 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3301 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3302 } else {
3303 if (netdev->flags & IFF_ALLMULTI) {
3304 rctl |= E1000_RCTL_MPE;
3305 vmolr |= E1000_VMOLR_MPME;
3306 } else {
3307 /*
3308 * Write addresses to the MTA, if the attempt fails
3309 * then we should just turn on promiscous mode so
3310 * that we can at least receive multicast traffic
3311 */
3312 count = igb_write_mc_addr_list(netdev);
3313 if (count < 0) {
3314 rctl |= E1000_RCTL_MPE;
3315 vmolr |= E1000_VMOLR_MPME;
3316 } else if (count) {
3317 vmolr |= E1000_VMOLR_ROMPE;
3318 }
3319 }
3320 /*
3321 * Write addresses to available RAR registers, if there is not
3322 * sufficient space to store all the addresses then enable
3323 * unicast promiscous mode
3324 */
3325 count = igb_write_uc_addr_list(netdev);
3326 if (count < 0) {
3327 rctl |= E1000_RCTL_UPE;
3328 vmolr |= E1000_VMOLR_ROPE;
3329 }
3330 rctl |= E1000_RCTL_VFE;
28fc06f5 3331 }
68d480c4 3332 wr32(E1000_RCTL, rctl);
28fc06f5 3333
68d480c4
AD		 3334 /*
3335 * In order to support SR-IOV and eventually VMDq it is necessary to set
3336 * the VMOLR to enable the appropriate modes. Without this workaround
3337 * we will have issues with VLAN tag stripping not being done for frames
3338 * that are only arriving because we are the default pool
3339 */
3340 if (hw->mac.type < e1000_82576)
28fc06f5 3341 return;
9d5c8243 3342
68d480c4
AD		 3343 vmolr |= rd32(E1000_VMOLR(vfn)) &
3344 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3345 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3346 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3347}
3348
3349/* Need to wait a few seconds after link up to get diagnostic information from
3350 * the phy */
3351static void igb_update_phy_info(unsigned long data)
3352{
3353 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3354 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3355}
3356
4d6b725e
AD		 3357/**
3358 * igb_has_link - check shared code for link and determine up/down
3359 * @adapter: pointer to driver private info
3360 **/
3145535a 3361bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3362{
3363 struct e1000_hw *hw = &adapter->hw;
3364 bool link_active = false;
3365 s32 ret_val = 0;
3366
3367 /* get_link_status is set on LSC (link status) interrupt or
3368 * rx sequence error interrupt. get_link_status will stay
3369 * false until the e1000_check_for_link establishes link
3370 * for copper adapters ONLY
3371 */
3372 switch (hw->phy.media_type) {
3373 case e1000_media_type_copper:
3374 if (hw->mac.get_link_status) {
3375 ret_val = hw->mac.ops.check_for_link(hw);
3376 link_active = !hw->mac.get_link_status;
3377 } else {
3378 link_active = true;
3379 }
3380 break;
4d6b725e
AD		 3381 case e1000_media_type_internal_serdes:
3382 ret_val = hw->mac.ops.check_for_link(hw);
3383 link_active = hw->mac.serdes_has_link;
3384 break;
3385 default:
3386 case e1000_media_type_unknown:
3387 break;
3388 }
3389
3390 return link_active;
3391}
3392
9d5c8243
AK		 3393/**
3394 * igb_watchdog - Timer Call-back
3395 * @data: pointer to adapter cast into an unsigned long
3396 **/
3397static void igb_watchdog(unsigned long data)
3398{
3399 struct igb_adapter *adapter = (struct igb_adapter *)data;
3400 /* Do the rest outside of interrupt context */
3401 schedule_work(&adapter->watchdog_task);
3402}
3403
3404static void igb_watchdog_task(struct work_struct *work)
3405{
3406 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3407 struct igb_adapter,
3408 watchdog_task);
9d5c8243 3409 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3410 struct net_device *netdev = adapter->netdev;
9d5c8243 3411 u32 link;
7a6ea550 3412 int i;
9d5c8243 3413
4d6b725e 3414 link = igb_has_link(adapter);
9d5c8243
AK		 3415 if (link) {
3416 if (!netif_carrier_ok(netdev)) {
3417 u32 ctrl;
330a6d6a
AD		 3418 hw->mac.ops.get_speed_and_duplex(hw,
3419 &adapter->link_speed,
3420 &adapter->link_duplex);
9d5c8243
AK		 3421
3422 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3423 /* Links status message must follow this format */
3424 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3425 "Flow Control: %s\n",
559e9c49
AD		 3426 netdev->name,
3427 adapter->link_speed,
3428 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3429 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3430 ((ctrl & E1000_CTRL_TFCE) &&
3431 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3432 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3433 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3434
d07f3e37 3435 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3436 adapter->tx_timeout_factor = 1;
3437 switch (adapter->link_speed) {
3438 case SPEED_10:
9d5c8243
AK		 3439 adapter->tx_timeout_factor = 14;
3440 break;
3441 case SPEED_100:
9d5c8243
AK		 3442 /* maybe add some timeout factor ? */
3443 break;
3444 }
3445
3446 netif_carrier_on(netdev);
9d5c8243 3447
4ae196df
AD		 3448 igb_ping_all_vfs(adapter);
3449
4b1a9877 3450 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3451 if (!test_bit(__IGB_DOWN, &adapter->state))
3452 mod_timer(&adapter->phy_info_timer,
3453 round_jiffies(jiffies + 2 * HZ));
3454 }
3455 } else {
3456 if (netif_carrier_ok(netdev)) {
3457 adapter->link_speed = 0;
3458 adapter->link_duplex = 0;
527d47c1
AD		 3459 /* Links status message must follow this format */
3460 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3461 netdev->name);
9d5c8243 3462 netif_carrier_off(netdev);
4b1a9877 3463
4ae196df
AD		 3464 igb_ping_all_vfs(adapter);
3465
4b1a9877 3466 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3470 }
3471 }
3472
9d5c8243 3473 igb_update_stats(adapter);
9d5c8243 3474
dbabb065 3475 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3476 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3477 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3478 /* We've lost link, so the controller stops DMA,
3479 * but we've got queued Tx work that's never going
3480 * to get done, so reset controller to flush Tx.
3481 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3482 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3483 adapter->tx_timeout_count++;
3484 schedule_work(&adapter->reset_task);
3485 /* return immediately since reset is imminent */
3486 return;
3487 }
9d5c8243 3488 }
9d5c8243 3489
dbabb065
AD		 3490 /* Force detection of hung controller every watchdog period */
3491 tx_ring->detect_tx_hung = true;
3492 }
f7ba205e 3493
9d5c8243 3494 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3495 if (adapter->msix_entries) {
047e0030
AD		 3496 u32 eics = 0;
3497 for (i = 0; i < adapter->num_q_vectors; i++) {
3498 struct igb_q_vector *q_vector = adapter->q_vector[i];
3499 eics |= q_vector->eims_value;
3500 }
7a6ea550
AD		 3501 wr32(E1000_EICS, eics);
3502 } else {
3503 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3504 }
9d5c8243 3505
9d5c8243
AK		 3506 /* Reset the timer */
3507 if (!test_bit(__IGB_DOWN, &adapter->state))
3508 mod_timer(&adapter->watchdog_timer,
3509 round_jiffies(jiffies + 2 * HZ));
3510}
3511
3512enum latency_range {
3513 lowest_latency = 0,
3514 low_latency = 1,
3515 bulk_latency = 2,
3516 latency_invalid = 255
3517};
3518
6eb5a7f1
AD		 3519/**
3520 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3521 *
3522 * Stores a new ITR value based on strictly on packet size. This
3523 * algorithm is less sophisticated than that used in igb_update_itr,
3524 * due to the difficulty of synchronizing statistics across multiple
3525 * receive rings. The divisors and thresholds used by this fuction
3526 * were determined based on theoretical maximum wire speed and testing
3527 * data, in order to minimize response time while increasing bulk
3528 * throughput.
3529 * This functionality is controlled by the InterruptThrottleRate module
3530 * parameter (see igb_param.c)
3531 * NOTE: This function is called only when operating in a multiqueue
3532 * receive environment.
047e0030 3533 * @q_vector: pointer to q_vector
6eb5a7f1 3534 **/
047e0030 3535static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3536{
047e0030 3537 int new_val = q_vector->itr_val;
6eb5a7f1 3538 int avg_wire_size = 0;
047e0030 3539 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3540
6eb5a7f1
AD		 3541 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3542 * ints/sec - ITR timer value of 120 ticks.
3543 */
3544 if (adapter->link_speed != SPEED_1000) {
047e0030 3545 new_val = 976;
6eb5a7f1 3546 goto set_itr_val;
9d5c8243 3547 }
047e0030
AD		 3548
3549 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3550 struct igb_ring *ring = q_vector->rx_ring;
3551 avg_wire_size = ring->total_bytes / ring->total_packets;
3552 }
3553
3554 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3555 struct igb_ring *ring = q_vector->tx_ring;
3556 avg_wire_size = max_t(u32, avg_wire_size,
3557 (ring->total_bytes /
3558 ring->total_packets));
3559 }
3560
3561 /* if avg_wire_size isn't set no work was done */
3562 if (!avg_wire_size)
3563 goto clear_counts;
9d5c8243 3564
6eb5a7f1
AD		 3565 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3566 avg_wire_size += 24;
3567
3568 /* Don't starve jumbo frames */
3569 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3570
6eb5a7f1
AD		 3571 /* Give a little boost to mid-size frames */
3572 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3573 new_val = avg_wire_size / 3;
3574 else
3575 new_val = avg_wire_size / 2;
9d5c8243 3576
abe1c363
NN		 3577 /* when in itr mode 3 do not exceed 20K ints/sec */
3578 if (adapter->rx_itr_setting == 3 && new_val < 196)
3579 new_val = 196;
3580
6eb5a7f1 3581set_itr_val:
047e0030
AD		 3582 if (new_val != q_vector->itr_val) {
3583 q_vector->itr_val = new_val;
3584 q_vector->set_itr = 1;
9d5c8243 3585 }
6eb5a7f1 3586clear_counts:
047e0030
AD		 3587 if (q_vector->rx_ring) {
3588 q_vector->rx_ring->total_bytes = 0;
3589 q_vector->rx_ring->total_packets = 0;
3590 }
3591 if (q_vector->tx_ring) {
3592 q_vector->tx_ring->total_bytes = 0;
3593 q_vector->tx_ring->total_packets = 0;
3594 }
9d5c8243
AK		 3595}
3596
3597/**
3598 * igb_update_itr - update the dynamic ITR value based on statistics
3599 * Stores a new ITR value based on packets and byte
3600 * counts during the last interrupt. The advantage of per interrupt
3601 * computation is faster updates and more accurate ITR for the current
3602 * traffic pattern. Constants in this function were computed
3603 * based on theoretical maximum wire speed and thresholds were set based
3604 * on testing data as well as attempting to minimize response time
3605 * while increasing bulk throughput.
3606 * this functionality is controlled by the InterruptThrottleRate module
3607 * parameter (see igb_param.c)
3608 * NOTE: These calculations are only valid when operating in a single-
3609 * queue environment.
3610 * @adapter: pointer to adapter
047e0030 3611 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3612 * @packets: the number of packets during this measurement interval
3613 * @bytes: the number of bytes during this measurement interval
3614 **/
3615static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3616 int packets, int bytes)
3617{
3618 unsigned int retval = itr_setting;
3619
3620 if (packets == 0)
3621 goto update_itr_done;
3622
3623 switch (itr_setting) {
3624 case lowest_latency:
3625 /* handle TSO and jumbo frames */
3626 if (bytes/packets > 8000)
3627 retval = bulk_latency;
3628 else if ((packets < 5) && (bytes > 512))
3629 retval = low_latency;
3630 break;
3631 case low_latency: /* 50 usec aka 20000 ints/s */
3632 if (bytes > 10000) {
3633 /* this if handles the TSO accounting */
3634 if (bytes/packets > 8000) {
3635 retval = bulk_latency;
3636 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3637 retval = bulk_latency;
3638 } else if ((packets > 35)) {
3639 retval = lowest_latency;
3640 }
3641 } else if (bytes/packets > 2000) {
3642 retval = bulk_latency;
3643 } else if (packets <= 2 && bytes < 512) {
3644 retval = lowest_latency;
3645 }
3646 break;
3647 case bulk_latency: /* 250 usec aka 4000 ints/s */
3648 if (bytes > 25000) {
3649 if (packets > 35)
3650 retval = low_latency;
1e5c3d21 3651 } else if (bytes < 1500) {
9d5c8243
AK		 3652 retval = low_latency;
3653 }
3654 break;
3655 }
3656
3657update_itr_done:
3658 return retval;
3659}
3660
6eb5a7f1 3661static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3662{
047e0030 3663 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3664 u16 current_itr;
047e0030 3665 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3666
3667 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3668 if (adapter->link_speed != SPEED_1000) {
3669 current_itr = 0;
3670 new_itr = 4000;
3671 goto set_itr_now;
3672 }
3673
3674 adapter->rx_itr = igb_update_itr(adapter,
3675 adapter->rx_itr,
3025a446
AD		 3676 q_vector->rx_ring->total_packets,
3677 q_vector->rx_ring->total_bytes);
9d5c8243 3678
047e0030
AD		 3679 adapter->tx_itr = igb_update_itr(adapter,
3680 adapter->tx_itr,
3025a446
AD		 3681 q_vector->tx_ring->total_packets,
3682 q_vector->tx_ring->total_bytes);
047e0030 3683 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3684
6eb5a7f1 3685 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3686 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3687 current_itr = low_latency;
3688
9d5c8243
AK		 3689 switch (current_itr) {
3690 /* counts and packets in update_itr are dependent on these numbers */
3691 case lowest_latency:
78b1f607 3692 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3693 break;
3694 case low_latency:
78b1f607 3695 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3696 break;
3697 case bulk_latency:
78b1f607 3698 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3699 break;
3700 default:
3701 break;
3702 }
3703
3704set_itr_now:
3025a446
AD		 3705 q_vector->rx_ring->total_bytes = 0;
3706 q_vector->rx_ring->total_packets = 0;
3707 q_vector->tx_ring->total_bytes = 0;
3708 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3709
047e0030 3710 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3711 /* this attempts to bias the interrupt rate towards Bulk
3712 * by adding intermediate steps when interrupt rate is
3713 * increasing */
047e0030
AD		 3714 new_itr = new_itr > q_vector->itr_val ?
3715 max((new_itr * q_vector->itr_val) /
3716 (new_itr + (q_vector->itr_val >> 2)),
3717 new_itr) :
9d5c8243
AK		 3718 new_itr;
3719 /* Don't write the value here; it resets the adapter's
3720 * internal timer, and causes us to delay far longer than
3721 * we should between interrupts. Instead, we write the ITR
3722 * value at the beginning of the next interrupt so the timing
3723 * ends up being correct.
3724 */
047e0030
AD		 3725 q_vector->itr_val = new_itr;
3726 q_vector->set_itr = 1;
9d5c8243 3727 }
9d5c8243
AK		 3728}
3729
9d5c8243
AK		 3730#define IGB_TX_FLAGS_CSUM 0x00000001
3731#define IGB_TX_FLAGS_VLAN 0x00000002
3732#define IGB_TX_FLAGS_TSO 0x00000004
3733#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3734#define IGB_TX_FLAGS_TSTAMP 0x00000010
3735#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3736#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3737
85ad76b2 3738static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3739 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3740{
3741 struct e1000_adv_tx_context_desc *context_desc;
3742 unsigned int i;
3743 int err;
3744 struct igb_buffer *buffer_info;
3745 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3746 u32 mss_l4len_idx;
3747 u8 l4len;
9d5c8243
AK		 3748
3749 if (skb_header_cloned(skb)) {
3750 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3751 if (err)
3752 return err;
3753 }
3754
3755 l4len = tcp_hdrlen(skb);
3756 *hdr_len += l4len;
3757
3758 if (skb->protocol == htons(ETH_P_IP)) {
3759 struct iphdr *iph = ip_hdr(skb);
3760 iph->tot_len = 0;
3761 iph->check = 0;
3762 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3763 iph->daddr, 0,
3764 IPPROTO_TCP,
3765 0);
8e1e8a47 3766 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3767 ipv6_hdr(skb)->payload_len = 0;
3768 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3769 &ipv6_hdr(skb)->daddr,
3770 0, IPPROTO_TCP, 0);
3771 }
3772
3773 i = tx_ring->next_to_use;
3774
3775 buffer_info = &tx_ring->buffer_info[i];
3776 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3777 /* VLAN MACLEN IPLEN */
3778 if (tx_flags & IGB_TX_FLAGS_VLAN)
3779 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3780 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3781 *hdr_len += skb_network_offset(skb);
3782 info |= skb_network_header_len(skb);
3783 *hdr_len += skb_network_header_len(skb);
3784 context_desc->vlan_macip_lens = cpu_to_le32(info);
3785
3786 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3787 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3788
3789 if (skb->protocol == htons(ETH_P_IP))
3790 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3791 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3792
3793 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3794
3795 /* MSS L4LEN IDX */
3796 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3797 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3798
73cd78f1 3799 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3800 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3801 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3802
3803 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3804 context_desc->seqnum_seed = 0;
3805
3806 buffer_info->time_stamp = jiffies;
0e014cb1 3807 buffer_info->next_to_watch = i;
9d5c8243
AK		 3808 buffer_info->dma = 0;
3809 i++;
3810 if (i == tx_ring->count)
3811 i = 0;
3812
3813 tx_ring->next_to_use = i;
3814
3815 return true;
3816}
3817
85ad76b2
AD		 3818static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3819 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3820{
3821 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3822 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3823 struct igb_buffer *buffer_info;
3824 u32 info = 0, tu_cmd = 0;
80785298 3825 unsigned int i;
9d5c8243
AK		 3826
3827 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3828 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3829 i = tx_ring->next_to_use;
3830 buffer_info = &tx_ring->buffer_info[i];
3831 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3832
3833 if (tx_flags & IGB_TX_FLAGS_VLAN)
3834 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3835
9d5c8243
AK		 3836 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3837 if (skb->ip_summed == CHECKSUM_PARTIAL)
3838 info |= skb_network_header_len(skb);
3839
3840 context_desc->vlan_macip_lens = cpu_to_le32(info);
3841
3842 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3843
3844 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3845 __be16 protocol;
3846
3847 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3848 const struct vlan_ethhdr *vhdr =
3849 (const struct vlan_ethhdr*)skb->data;
3850
3851 protocol = vhdr->h_vlan_encapsulated_proto;
3852 } else {
3853 protocol = skb->protocol;
3854 }
3855
3856 switch (protocol) {
09640e63 3857 case cpu_to_be16(ETH_P_IP):
9d5c8243 3858 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3859 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3860 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3861 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3862 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3863 break;
09640e63 3864 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3865 /* XXX what about other V6 headers?? */
3866 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3867 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3868 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3869 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3870 break;
3871 default:
3872 if (unlikely(net_ratelimit()))
59d71989 3873 dev_warn(dev,
44b0cda3
MW		 3874 "partial checksum but proto=%x!\n",
3875 skb->protocol);
3876 break;
3877 }
9d5c8243
AK		 3878 }
3879
3880 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3881 context_desc->seqnum_seed = 0;
85ad76b2 3882 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3883 context_desc->mss_l4len_idx =
85ad76b2 3884 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3885
3886 buffer_info->time_stamp = jiffies;
0e014cb1 3887 buffer_info->next_to_watch = i;
9d5c8243
AK		 3888 buffer_info->dma = 0;
3889
3890 i++;
3891 if (i == tx_ring->count)
3892 i = 0;
3893 tx_ring->next_to_use = i;
3894
3895 return true;
3896 }
9d5c8243
AK		 3897 return false;
3898}
3899
3900#define IGB_MAX_TXD_PWR 16
3901#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3902
80785298 3903static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3904 unsigned int first)
9d5c8243
AK		 3905{
3906 struct igb_buffer *buffer_info;
59d71989 3907 struct device *dev = tx_ring->dev;
2873957d 3908 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3909 unsigned int count = 0, i;
3910 unsigned int f;
2873957d 3911 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3912
3913 i = tx_ring->next_to_use;
3914
3915 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3916 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3917 buffer_info->length = hlen;
9d5c8243
AK		 3918 /* set time_stamp *before* dma to help avoid a possible race */
3919 buffer_info->time_stamp = jiffies;
0e014cb1 3920 buffer_info->next_to_watch = i;
2873957d 3921 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3922 DMA_TO_DEVICE);
3923 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3924 goto dma_error;
9d5c8243
AK		 3925
3926 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3927 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3928 unsigned int len = frag->size;
9d5c8243 3929
8581145f 3930 count++;
65689fef
AD		 3931 i++;
3932 if (i == tx_ring->count)
3933 i = 0;
3934
9d5c8243
AK		 3935 buffer_info = &tx_ring->buffer_info[i];
3936 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3937 buffer_info->length = len;
3938 buffer_info->time_stamp = jiffies;
0e014cb1 3939 buffer_info->next_to_watch = i;
6366ad33 3940 buffer_info->mapped_as_page = true;
59d71989 3941 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3942 frag->page,
3943 frag->page_offset,
3944 len,
59d71989
AD		 3945 DMA_TO_DEVICE);
3946 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3947 goto dma_error;
3948
9d5c8243
AK		 3949 }
3950
9d5c8243 3951 tx_ring->buffer_info[i].skb = skb;
2873957d
NN		 3952 tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3953 /* multiply data chunks by size of headers */
3954 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3955 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3956 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3957
cdfd01fc 3958 return ++count;
6366ad33
AD		 3959
3960dma_error:
59d71989 3961 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3962
3963 /* clear timestamp and dma mappings for failed buffer_info mapping */
3964 buffer_info->dma = 0;
3965 buffer_info->time_stamp = 0;
3966 buffer_info->length = 0;
3967 buffer_info->next_to_watch = 0;
3968 buffer_info->mapped_as_page = false;
6366ad33
AD		 3969
3970 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3971 while (count--) {
3972 if (i == 0)
3973 i = tx_ring->count;
6366ad33 3974 i--;
6366ad33
AD		 3975 buffer_info = &tx_ring->buffer_info[i];
3976 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3977 }
3978
3979 return 0;
9d5c8243
AK		 3980}
3981
85ad76b2 3982static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3983 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3984 u8 hdr_len)
3985{
cdfd01fc 3986 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3987 struct igb_buffer *buffer_info;
3988 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3989 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3990
3991 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3992 E1000_ADVTXD_DCMD_DEXT);
3993
3994 if (tx_flags & IGB_TX_FLAGS_VLAN)
3995 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3996
33af6bcc
PO		 3997 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3998 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3999
9d5c8243
AK		 4000 if (tx_flags & IGB_TX_FLAGS_TSO) {
4001 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4002
4003 /* insert tcp checksum */
4004 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4005
4006 /* insert ip checksum */
4007 if (tx_flags & IGB_TX_FLAGS_IPV4)
4008 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4009
4010 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4011 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4012 }
4013
85ad76b2
AD		 4014 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4015 (tx_flags & (IGB_TX_FLAGS_CSUM |
4016 IGB_TX_FLAGS_TSO |
7dfc16fa 4017 IGB_TX_FLAGS_VLAN)))
85ad76b2 4018 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4019
4020 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4021
cdfd01fc 4022 do {
9d5c8243
AK		 4023 buffer_info = &tx_ring->buffer_info[i];
4024 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4025 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4026 tx_desc->read.cmd_type_len =
4027 cpu_to_le32(cmd_type_len | buffer_info->length);
4028 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4029 count--;
9d5c8243
AK		 4030 i++;
4031 if (i == tx_ring->count)
4032 i = 0;
cdfd01fc 4033 } while (count > 0);
9d5c8243 4034
85ad76b2 4035 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4036 /* Force memory writes to complete before letting h/w
4037 * know there are new descriptors to fetch. (Only
4038 * applicable for weak-ordered memory model archs,
4039 * such as IA-64). */
4040 wmb();
4041
4042 tx_ring->next_to_use = i;
fce99e34 4043 writel(i, tx_ring->tail);
9d5c8243
AK		 4044 /* we need this if more than one processor can write to our tail
4045 * at a time, it syncronizes IO on IA64/Altix systems */
4046 mmiowb();
4047}
4048
e694e964 4049static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4050{
e694e964
AD		 4051 struct net_device *netdev = tx_ring->netdev;
4052
661086df 4053 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4054
9d5c8243
AK		 4055 /* Herbert's original patch had:
4056 * smp_mb__after_netif_stop_queue();
4057 * but since that doesn't exist yet, just open code it. */
4058 smp_mb();
4059
4060 /* We need to check again in a case another CPU has just
4061 * made room available. */
c493ea45 4062 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4063 return -EBUSY;
4064
4065 /* A reprieve! */
661086df 4066 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4067 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 4068 return 0;
4069}
4070
717ba089 4071static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4072{
c493ea45 4073 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4074 return 0;
e694e964 4075 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4076}
4077
b1a436c3
AD		 4078netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4079 struct igb_ring *tx_ring)
9d5c8243 4080{
e694e964 4081 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 4082 int tso = 0, count;
91d4ee33
NN		 4083 u32 tx_flags = 0;
4084 u16 first;
4085 u8 hdr_len = 0;
c5b9bd5e 4086 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 4087
9d5c8243
AK		 4088 /* need: 1 descriptor per page,
4089 * + 2 desc gap to keep tail from touching head,
4090 * + 1 desc for skb->data,
4091 * + 1 desc for context descriptor,
4092 * otherwise try next time */
e694e964 4093 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4094 /* this is a hard error */
9d5c8243
AK		 4095 return NETDEV_TX_BUSY;
4096 }
33af6bcc 4097
33af6bcc
PO		 4098 if (unlikely(shtx->hardware)) {
4099 shtx->in_progress = 1;
4100 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4101 }
9d5c8243 4102
cdfd01fc 4103 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 4104 tx_flags |= IGB_TX_FLAGS_VLAN;
4105 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4106 }
4107
661086df
PWJ		 4108 if (skb->protocol == htons(ETH_P_IP))
4109 tx_flags |= IGB_TX_FLAGS_IPV4;
4110
0e014cb1 4111 first = tx_ring->next_to_use;
85ad76b2
AD		 4112 if (skb_is_gso(skb)) {
4113 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4114
85ad76b2
AD		 4115 if (tso < 0) {
4116 dev_kfree_skb_any(skb);
4117 return NETDEV_TX_OK;
4118 }
9d5c8243
AK		 4119 }
4120
4121 if (tso)
4122 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4123 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4124 (skb->ip_summed == CHECKSUM_PARTIAL))
4125 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4126
65689fef 4127 /*
cdfd01fc 4128 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4129 * has occured and we need to rewind the descriptor queue
4130 */
80785298 4131 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4132 if (!count) {
65689fef
AD		 4133 dev_kfree_skb_any(skb);
4134 tx_ring->buffer_info[first].time_stamp = 0;
4135 tx_ring->next_to_use = first;
85ad76b2 4136 return NETDEV_TX_OK;
65689fef 4137 }
9d5c8243 4138
85ad76b2
AD		 4139 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4140
4141 /* Make sure there is space in the ring for the next send. */
e694e964 4142 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4143
9d5c8243
AK		 4144 return NETDEV_TX_OK;
4145}
4146
3b29a56d
SH		 4147static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4148 struct net_device *netdev)
9d5c8243
AK		 4149{
4150 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4151 struct igb_ring *tx_ring;
661086df 4152 int r_idx = 0;
b1a436c3
AD		 4153
4154 if (test_bit(__IGB_DOWN, &adapter->state)) {
4155 dev_kfree_skb_any(skb);
4156 return NETDEV_TX_OK;
4157 }
4158
4159 if (skb->len <= 0) {
4160 dev_kfree_skb_any(skb);
4161 return NETDEV_TX_OK;
4162 }
4163
1bfaf07b 4164 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4165 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4166
4167 /* This goes back to the question of how to logically map a tx queue
4168 * to a flow. Right now, performance is impacted slightly negatively
4169 * if using multiple tx queues. If the stack breaks away from a
4170 * single qdisc implementation, we can look at this again. */
e694e964 4171 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4172}
4173
4174/**
4175 * igb_tx_timeout - Respond to a Tx Hang
4176 * @netdev: network interface device structure
4177 **/
4178static void igb_tx_timeout(struct net_device *netdev)
4179{
4180 struct igb_adapter *adapter = netdev_priv(netdev);
4181 struct e1000_hw *hw = &adapter->hw;
4182
4183 /* Do the reset outside of interrupt context */
4184 adapter->tx_timeout_count++;
f7ba205e 4185
55cac248
AD		 4186 if (hw->mac.type == e1000_82580)
4187 hw->dev_spec._82575.global_device_reset = true;
4188
9d5c8243 4189 schedule_work(&adapter->reset_task);
265de409
AD		 4190 wr32(E1000_EICS,
4191 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4192}
4193
4194static void igb_reset_task(struct work_struct *work)
4195{
4196 struct igb_adapter *adapter;
4197 adapter = container_of(work, struct igb_adapter, reset_task);
4198
c97ec42a
TI		 4199 igb_dump(adapter);
4200 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4201 igb_reinit_locked(adapter);
4202}
4203
4204/**
4205 * igb_get_stats - Get System Network Statistics
4206 * @netdev: network interface device structure
4207 *
4208 * Returns the address of the device statistics structure.
4209 * The statistics are actually updated from the timer callback.
4210 **/
73cd78f1 4211static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 4212{
9d5c8243 4213 /* only return the current stats */
8d24e933 4214 return &netdev->stats;
9d5c8243
AK		 4215}
4216
4217/**
4218 * igb_change_mtu - Change the Maximum Transfer Unit
4219 * @netdev: network interface device structure
4220 * @new_mtu: new value for maximum frame size
4221 *
4222 * Returns 0 on success, negative on failure
4223 **/
4224static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4225{
4226 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4227 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4228 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4229 u32 rx_buffer_len, i;
9d5c8243 4230
c809d227 4231 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4232 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4233 return -EINVAL;
4234 }
4235
9d5c8243 4236 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4237 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4238 return -EINVAL;
4239 }
4240
4241 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4242 msleep(1);
73cd78f1 4243
9d5c8243
AK		 4244 /* igb_down has a dependency on max_frame_size */
4245 adapter->max_frame_size = max_frame;
559e9c49 4246
9d5c8243
AK		 4247 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4248 * means we reserve 2 more, this pushes us to allocate from the next
4249 * larger slab size.
4250 * i.e. RXBUFFER_2048 --> size-4096 slab
4251 */
4252
757b77e2
NN		 4253 if (adapter->hw.mac.type == e1000_82580)
4254 max_frame += IGB_TS_HDR_LEN;
4255
7d95b717 4256 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4257 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4258 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4259 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4260 else
4c844851
AD		 4261 rx_buffer_len = IGB_RXBUFFER_128;
4262
757b77e2
NN		 4263 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4264 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4265 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4266
4267 if ((adapter->hw.mac.type == e1000_82580) &&
4268 (rx_buffer_len == IGB_RXBUFFER_128))
4269 rx_buffer_len += IGB_RXBUFFER_64;
4270
4c844851
AD		 4271 if (netif_running(netdev))
4272 igb_down(adapter);
9d5c8243 4273
090b1795 4274 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4275 netdev->mtu, new_mtu);
4276 netdev->mtu = new_mtu;
4277
4c844851 4278 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4279 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4280
9d5c8243
AK		 4281 if (netif_running(netdev))
4282 igb_up(adapter);
4283 else
4284 igb_reset(adapter);
4285
4286 clear_bit(__IGB_RESETTING, &adapter->state);
4287
4288 return 0;
4289}
4290
4291/**
4292 * igb_update_stats - Update the board statistics counters
4293 * @adapter: board private structure
4294 **/
4295
4296void igb_update_stats(struct igb_adapter *adapter)
4297{
128e45eb 4298 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 4299 struct e1000_hw *hw = &adapter->hw;
4300 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4301 u32 reg, mpc;
9d5c8243 4302 u16 phy_tmp;
3f9c0164
AD		 4303 int i;
4304 u64 bytes, packets;
9d5c8243
AK		 4305
4306#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4307
4308 /*
4309 * Prevent stats update while adapter is being reset, or if the pci
4310 * connection is down.
4311 */
4312 if (adapter->link_speed == 0)
4313 return;
4314 if (pci_channel_offline(pdev))
4315 return;
4316
3f9c0164
AD		 4317 bytes = 0;
4318 packets = 0;
4319 for (i = 0; i < adapter->num_rx_queues; i++) {
4320 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 4321 struct igb_ring *ring = adapter->rx_ring[i];
4322 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4323 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 4324 bytes += ring->rx_stats.bytes;
4325 packets += ring->rx_stats.packets;
3f9c0164
AD		 4326 }
4327
128e45eb
AD		 4328 net_stats->rx_bytes = bytes;
4329 net_stats->rx_packets = packets;
3f9c0164
AD		 4330
4331 bytes = 0;
4332 packets = 0;
4333 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4334 struct igb_ring *ring = adapter->tx_ring[i];
4335 bytes += ring->tx_stats.bytes;
4336 packets += ring->tx_stats.packets;
3f9c0164 4337 }
128e45eb
AD		 4338 net_stats->tx_bytes = bytes;
4339 net_stats->tx_packets = packets;
3f9c0164
AD		 4340
4341 /* read stats registers */
9d5c8243
AK		 4342 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4343 adapter->stats.gprc += rd32(E1000_GPRC);
4344 adapter->stats.gorc += rd32(E1000_GORCL);
4345 rd32(E1000_GORCH); /* clear GORCL */
4346 adapter->stats.bprc += rd32(E1000_BPRC);
4347 adapter->stats.mprc += rd32(E1000_MPRC);
4348 adapter->stats.roc += rd32(E1000_ROC);
4349
4350 adapter->stats.prc64 += rd32(E1000_PRC64);
4351 adapter->stats.prc127 += rd32(E1000_PRC127);
4352 adapter->stats.prc255 += rd32(E1000_PRC255);
4353 adapter->stats.prc511 += rd32(E1000_PRC511);
4354 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4355 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4356 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4357 adapter->stats.sec += rd32(E1000_SEC);
4358
fa3d9a6d
MW		 4359 mpc = rd32(E1000_MPC);
4360 adapter->stats.mpc += mpc;
4361 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4362 adapter->stats.scc += rd32(E1000_SCC);
4363 adapter->stats.ecol += rd32(E1000_ECOL);
4364 adapter->stats.mcc += rd32(E1000_MCC);
4365 adapter->stats.latecol += rd32(E1000_LATECOL);
4366 adapter->stats.dc += rd32(E1000_DC);
4367 adapter->stats.rlec += rd32(E1000_RLEC);
4368 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4369 adapter->stats.xontxc += rd32(E1000_XONTXC);
4370 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4371 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4372 adapter->stats.fcruc += rd32(E1000_FCRUC);
4373 adapter->stats.gptc += rd32(E1000_GPTC);
4374 adapter->stats.gotc += rd32(E1000_GOTCL);
4375 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4376 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4377 adapter->stats.ruc += rd32(E1000_RUC);
4378 adapter->stats.rfc += rd32(E1000_RFC);
4379 adapter->stats.rjc += rd32(E1000_RJC);
4380 adapter->stats.tor += rd32(E1000_TORH);
4381 adapter->stats.tot += rd32(E1000_TOTH);
4382 adapter->stats.tpr += rd32(E1000_TPR);
4383
4384 adapter->stats.ptc64 += rd32(E1000_PTC64);
4385 adapter->stats.ptc127 += rd32(E1000_PTC127);
4386 adapter->stats.ptc255 += rd32(E1000_PTC255);
4387 adapter->stats.ptc511 += rd32(E1000_PTC511);
4388 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4389 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4390
4391 adapter->stats.mptc += rd32(E1000_MPTC);
4392 adapter->stats.bptc += rd32(E1000_BPTC);
4393
2d0b0f69
NN		 4394 adapter->stats.tpt += rd32(E1000_TPT);
4395 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4396
4397 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4398 /* read internal phy specific stats */
4399 reg = rd32(E1000_CTRL_EXT);
4400 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4401 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4402 adapter->stats.tncrs += rd32(E1000_TNCRS);
4403 }
4404
9d5c8243
AK		 4405 adapter->stats.tsctc += rd32(E1000_TSCTC);
4406 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4407
4408 adapter->stats.iac += rd32(E1000_IAC);
4409 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4410 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4411 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4412 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4413 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4414 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4415 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4416 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4417
4418 /* Fill out the OS statistics structure */
128e45eb
AD		 4419 net_stats->multicast = adapter->stats.mprc;
4420 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4421
4422 /* Rx Errors */
4423
4424 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4425 * our own version based on RUC and ROC */
128e45eb 4426 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4427 adapter->stats.crcerrs + adapter->stats.algnerrc +
4428 adapter->stats.ruc + adapter->stats.roc +
4429 adapter->stats.cexterr;
128e45eb
AD		 4430 net_stats->rx_length_errors = adapter->stats.ruc +
4431 adapter->stats.roc;
4432 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4433 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4434 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4435
4436 /* Tx Errors */
128e45eb
AD		 4437 net_stats->tx_errors = adapter->stats.ecol +
4438 adapter->stats.latecol;
4439 net_stats->tx_aborted_errors = adapter->stats.ecol;
4440 net_stats->tx_window_errors = adapter->stats.latecol;
4441 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4442
4443 /* Tx Dropped needs to be maintained elsewhere */
4444
4445 /* Phy Stats */
4446 if (hw->phy.media_type == e1000_media_type_copper) {
4447 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4448 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4449 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4450 adapter->phy_stats.idle_errors += phy_tmp;
4451 }
4452 }
4453
4454 /* Management Stats */
4455 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4456 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4457 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4458}
4459
9d5c8243
AK		 4460static irqreturn_t igb_msix_other(int irq, void *data)
4461{
047e0030 4462 struct igb_adapter *adapter = data;
9d5c8243 4463 struct e1000_hw *hw = &adapter->hw;
844290e5 4464 u32 icr = rd32(E1000_ICR);
844290e5 4465 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4466
7f081d40
AD		 4467 if (icr & E1000_ICR_DRSTA)
4468 schedule_work(&adapter->reset_task);
4469
047e0030 4470 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4471 /* HW is reporting DMA is out of sync */
4472 adapter->stats.doosync++;
4473 }
eebbbdba 4474
4ae196df
AD		 4475 /* Check for a mailbox event */
4476 if (icr & E1000_ICR_VMMB)
4477 igb_msg_task(adapter);
4478
4479 if (icr & E1000_ICR_LSC) {
4480 hw->mac.get_link_status = 1;
4481 /* guard against interrupt when we're going down */
4482 if (!test_bit(__IGB_DOWN, &adapter->state))
4483 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4484 }
4485
25568a53
AD		 4486 if (adapter->vfs_allocated_count)
4487 wr32(E1000_IMS, E1000_IMS_LSC |
4488 E1000_IMS_VMMB |
4489 E1000_IMS_DOUTSYNC);
4490 else
4491 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4492 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4493
4494 return IRQ_HANDLED;
4495}
4496
047e0030 4497static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4498{
26b39276 4499 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4500 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4501
047e0030
AD		 4502 if (!q_vector->set_itr)
4503 return;
73cd78f1 4504
047e0030
AD		 4505 if (!itr_val)
4506 itr_val = 0x4;
661086df 4507
26b39276
AD		 4508 if (adapter->hw.mac.type == e1000_82575)
4509 itr_val |= itr_val << 16;
661086df 4510 else
047e0030 4511 itr_val |= 0x8000000;
661086df 4512
047e0030
AD		 4513 writel(itr_val, q_vector->itr_register);
4514 q_vector->set_itr = 0;
6eb5a7f1
AD		 4515}
4516
047e0030 4517static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4518{
047e0030 4519 struct igb_q_vector *q_vector = data;
9d5c8243 4520
047e0030
AD		 4521 /* Write the ITR value calculated from the previous interrupt. */
4522 igb_write_itr(q_vector);
9d5c8243 4523
047e0030 4524 napi_schedule(&q_vector->napi);
844290e5 4525
047e0030 4526 return IRQ_HANDLED;
fe4506b6
JC		 4527}
4528
421e02f0 4529#ifdef CONFIG_IGB_DCA
047e0030 4530static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4531{
047e0030 4532 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4533 struct e1000_hw *hw = &adapter->hw;
4534 int cpu = get_cpu();
fe4506b6 4535
047e0030
AD		 4536 if (q_vector->cpu == cpu)
4537 goto out_no_update;
4538
4539 if (q_vector->tx_ring) {
4540 int q = q_vector->tx_ring->reg_idx;
4541 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4542 if (hw->mac.type == e1000_82575) {
4543 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4544 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4545 } else {
047e0030
AD		 4546 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4547 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4548 E1000_DCA_TXCTRL_CPUID_SHIFT;
4549 }
4550 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4551 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4552 }
4553 if (q_vector->rx_ring) {
4554 int q = q_vector->rx_ring->reg_idx;
4555 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4556 if (hw->mac.type == e1000_82575) {
2d064c06 4557 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4558 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4559 } else {
4560 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4561 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4562 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4563 }
fe4506b6
JC		 4564 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4565 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4566 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4567 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4568 }
047e0030
AD		 4569 q_vector->cpu = cpu;
4570out_no_update:
fe4506b6
JC		 4571 put_cpu();
4572}
4573
4574static void igb_setup_dca(struct igb_adapter *adapter)
4575{
7e0e99ef 4576 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4577 int i;
4578
7dfc16fa 4579 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4580 return;
4581
7e0e99ef
AD		 4582 /* Always use CB2 mode, difference is masked in the CB driver. */
4583 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4584
047e0030 4585 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4586 adapter->q_vector[i]->cpu = -1;
4587 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4588 }
4589}
4590
4591static int __igb_notify_dca(struct device *dev, void *data)
4592{
4593 struct net_device *netdev = dev_get_drvdata(dev);
4594 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4595 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4596 struct e1000_hw *hw = &adapter->hw;
4597 unsigned long event = *(unsigned long *)data;
4598
4599 switch (event) {
4600 case DCA_PROVIDER_ADD:
4601 /* if already enabled, don't do it again */
7dfc16fa 4602 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4603 break;
fe4506b6 4604 if (dca_add_requester(dev) == 0) {
bbd98fe4 4605 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4606 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4607 igb_setup_dca(adapter);
4608 break;
4609 }
4610 /* Fall Through since DCA is disabled. */
4611 case DCA_PROVIDER_REMOVE:
7dfc16fa 4612 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4613 /* without this a class_device is left
047e0030 4614 * hanging around in the sysfs model */
fe4506b6 4615 dca_remove_requester(dev);
090b1795 4616 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4617 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4618 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4619 }
4620 break;
4621 }
bbd98fe4 4622
fe4506b6 4623 return 0;
9d5c8243
AK		 4624}
4625
fe4506b6
JC		 4626static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4627 void *p)
4628{
4629 int ret_val;
4630
4631 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4632 __igb_notify_dca);
4633
4634 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4635}
421e02f0 4636#endif /* CONFIG_IGB_DCA */
9d5c8243 4637
4ae196df
AD		 4638static void igb_ping_all_vfs(struct igb_adapter *adapter)
4639{
4640 struct e1000_hw *hw = &adapter->hw;
4641 u32 ping;
4642 int i;
4643
4644 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4645 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4646 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4647 ping |= E1000_VT_MSGTYPE_CTS;
4648 igb_write_mbx(hw, &ping, 1, i);
4649 }
4650}
4651
7d5753f0
AD		 4652static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4653{
4654 struct e1000_hw *hw = &adapter->hw;
4655 u32 vmolr = rd32(E1000_VMOLR(vf));
4656 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4657
4658 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4659 IGB_VF_FLAG_MULTI_PROMISC);
4660 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4661
4662 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4663 vmolr |= E1000_VMOLR_MPME;
4664 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4665 } else {
4666 /*
4667 * if we have hashes and we are clearing a multicast promisc
4668 * flag we need to write the hashes to the MTA as this step
4669 * was previously skipped
4670 */
4671 if (vf_data->num_vf_mc_hashes > 30) {
4672 vmolr |= E1000_VMOLR_MPME;
4673 } else if (vf_data->num_vf_mc_hashes) {
4674 int j;
4675 vmolr |= E1000_VMOLR_ROMPE;
4676 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4677 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4678 }
4679 }
4680
4681 wr32(E1000_VMOLR(vf), vmolr);
4682
4683 /* there are flags left unprocessed, likely not supported */
4684 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4685 return -EINVAL;
4686
4687 return 0;
4688
4689}
4690
4ae196df
AD		 4691static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4692 u32 *msgbuf, u32 vf)
4693{
4694 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4695 u16 *hash_list = (u16 *)&msgbuf[1];
4696 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4697 int i;
4698
7d5753f0 4699 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4700 * to this VF for later use to restore when the PF multi cast
4701 * list changes
4702 */
4703 vf_data->num_vf_mc_hashes = n;
4704
7d5753f0
AD		 4705 /* only up to 30 hash values supported */
4706 if (n > 30)
4707 n = 30;
4708
4709 /* store the hashes for later use */
4ae196df 4710 for (i = 0; i < n; i++)
a419aef8 4711 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4712
4713 /* Flush and reset the mta with the new values */
ff41f8dc 4714 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4715
4716 return 0;
4717}
4718
4719static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4720{
4721 struct e1000_hw *hw = &adapter->hw;
4722 struct vf_data_storage *vf_data;
4723 int i, j;
4724
4725 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4726 u32 vmolr = rd32(E1000_VMOLR(i));
4727 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4728
4ae196df 4729 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4730
4731 if ((vf_data->num_vf_mc_hashes > 30) ||
4732 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4733 vmolr |= E1000_VMOLR_MPME;
4734 } else if (vf_data->num_vf_mc_hashes) {
4735 vmolr |= E1000_VMOLR_ROMPE;
4736 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4737 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4738 }
4739 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4740 }
4741}
4742
4743static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4744{
4745 struct e1000_hw *hw = &adapter->hw;
4746 u32 pool_mask, reg, vid;
4747 int i;
4748
4749 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4750
4751 /* Find the vlan filter for this id */
4752 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4753 reg = rd32(E1000_VLVF(i));
4754
4755 /* remove the vf from the pool */
4756 reg &= ~pool_mask;
4757
4758 /* if pool is empty then remove entry from vfta */
4759 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4760 (reg & E1000_VLVF_VLANID_ENABLE)) {
4761 reg = 0;
4762 vid = reg & E1000_VLVF_VLANID_MASK;
4763 igb_vfta_set(hw, vid, false);
4764 }
4765
4766 wr32(E1000_VLVF(i), reg);
4767 }
ae641bdc
AD		 4768
4769 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4770}
4771
4772static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4773{
4774 struct e1000_hw *hw = &adapter->hw;
4775 u32 reg, i;
4776
51466239
AD		 4777 /* The vlvf table only exists on 82576 hardware and newer */
4778 if (hw->mac.type < e1000_82576)
4779 return -1;
4780
4781 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4782 if (!adapter->vfs_allocated_count)
4783 return -1;
4784
4785 /* Find the vlan filter for this id */
4786 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4787 reg = rd32(E1000_VLVF(i));
4788 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4789 vid == (reg & E1000_VLVF_VLANID_MASK))
4790 break;
4791 }
4792
4793 if (add) {
4794 if (i == E1000_VLVF_ARRAY_SIZE) {
4795 /* Did not find a matching VLAN ID entry that was
4796 * enabled. Search for a free filter entry, i.e.
4797 * one without the enable bit set
4798 */
4799 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4800 reg = rd32(E1000_VLVF(i));
4801 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4802 break;
4803 }
4804 }
4805 if (i < E1000_VLVF_ARRAY_SIZE) {
4806 /* Found an enabled/available entry */
4807 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4808
4809 /* if !enabled we need to set this up in vfta */
4810 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4811 /* add VID to filter table */
4812 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4813 reg |= E1000_VLVF_VLANID_ENABLE;
4814 }
cad6d05f
AD		 4815 reg &= ~E1000_VLVF_VLANID_MASK;
4816 reg |= vid;
4ae196df 4817 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4818
4819 /* do not modify RLPML for PF devices */
4820 if (vf >= adapter->vfs_allocated_count)
4821 return 0;
4822
4823 if (!adapter->vf_data[vf].vlans_enabled) {
4824 u32 size;
4825 reg = rd32(E1000_VMOLR(vf));
4826 size = reg & E1000_VMOLR_RLPML_MASK;
4827 size += 4;
4828 reg &= ~E1000_VMOLR_RLPML_MASK;
4829 reg |= size;
4830 wr32(E1000_VMOLR(vf), reg);
4831 }
ae641bdc 4832
51466239 4833 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4834 return 0;
4835 }
4836 } else {
4837 if (i < E1000_VLVF_ARRAY_SIZE) {
4838 /* remove vf from the pool */
4839 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4840 /* if pool is empty then remove entry from vfta */
4841 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4842 reg = 0;
4843 igb_vfta_set(hw, vid, false);
4844 }
4845 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4846
4847 /* do not modify RLPML for PF devices */
4848 if (vf >= adapter->vfs_allocated_count)
4849 return 0;
4850
4851 adapter->vf_data[vf].vlans_enabled--;
4852 if (!adapter->vf_data[vf].vlans_enabled) {
4853 u32 size;
4854 reg = rd32(E1000_VMOLR(vf));
4855 size = reg & E1000_VMOLR_RLPML_MASK;
4856 size -= 4;
4857 reg &= ~E1000_VMOLR_RLPML_MASK;
4858 reg |= size;
4859 wr32(E1000_VMOLR(vf), reg);
4860 }
4ae196df
AD		 4861 }
4862 }
8151d294
WM		 4863 return 0;
4864}
4865
4866static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4867{
4868 struct e1000_hw *hw = &adapter->hw;
4869
4870 if (vid)
4871 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4872 else
4873 wr32(E1000_VMVIR(vf), 0);
4874}
4875
4876static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4877 int vf, u16 vlan, u8 qos)
4878{
4879 int err = 0;
4880 struct igb_adapter *adapter = netdev_priv(netdev);
4881
4882 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4883 return -EINVAL;
4884 if (vlan || qos) {
4885 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4886 if (err)
4887 goto out;
4888 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4889 igb_set_vmolr(adapter, vf, !vlan);
4890 adapter->vf_data[vf].pf_vlan = vlan;
4891 adapter->vf_data[vf].pf_qos = qos;
4892 dev_info(&adapter->pdev->dev,
4893 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4894 if (test_bit(__IGB_DOWN, &adapter->state)) {
4895 dev_warn(&adapter->pdev->dev,
4896 "The VF VLAN has been set,"
4897 " but the PF device is not up.\n");
4898 dev_warn(&adapter->pdev->dev,
4899 "Bring the PF device up before"
4900 " attempting to use the VF device.\n");
4901 }
4902 } else {
4903 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4904 false, vf);
4905 igb_set_vmvir(adapter, vlan, vf);
4906 igb_set_vmolr(adapter, vf, true);
4907 adapter->vf_data[vf].pf_vlan = 0;
4908 adapter->vf_data[vf].pf_qos = 0;
4909 }
4910out:
4911 return err;
4ae196df
AD		 4912}
4913
4914static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4915{
4916 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4917 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4918
4919 return igb_vlvf_set(adapter, vid, add, vf);
4920}
4921
f2ca0dbe 4922static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4923{
8151d294
WM		 4924 /* clear flags */
4925 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4926 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4927
4928 /* reset offloads to defaults */
8151d294 4929 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4930
4931 /* reset vlans for device */
4932 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4933 if (adapter->vf_data[vf].pf_vlan)
4934 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4935 adapter->vf_data[vf].pf_vlan,
4936 adapter->vf_data[vf].pf_qos);
4937 else
4938 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4939
4940 /* reset multicast table array for vf */
4941 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4942
4943 /* Flush and reset the mta with the new values */
ff41f8dc 4944 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4945}
4946
f2ca0dbe
AD		 4947static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4948{
4949 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4950
4951 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4952 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4953 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4954
4955 /* process remaining reset events */
4956 igb_vf_reset(adapter, vf);
4957}
4958
4959static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4960{
4961 struct e1000_hw *hw = &adapter->hw;
4962 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4963 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4964 u32 reg, msgbuf[3];
4965 u8 *addr = (u8 *)(&msgbuf[1]);
4966
4967 /* process all the same items cleared in a function level reset */
f2ca0dbe 4968 igb_vf_reset(adapter, vf);
4ae196df
AD		 4969
4970 /* set vf mac address */
26ad9178 4971 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4972
4973 /* enable transmit and receive for vf */
4974 reg = rd32(E1000_VFTE);
4975 wr32(E1000_VFTE, reg | (1 << vf));
4976 reg = rd32(E1000_VFRE);
4977 wr32(E1000_VFRE, reg | (1 << vf));
4978
f2ca0dbe 4979 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4980
4981 /* reply to reset with ack and vf mac address */
4982 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4983 memcpy(addr, vf_mac, 6);
4984 igb_write_mbx(hw, msgbuf, 3, vf);
4985}
4986
4987static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4988{
f2ca0dbe
AD		 4989 unsigned char *addr = (char *)&msg[1];
4990 int err = -1;
4ae196df 4991
f2ca0dbe
AD		 4992 if (is_valid_ether_addr(addr))
4993 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4994
f2ca0dbe 4995 return err;
4ae196df
AD		 4996}
4997
4998static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4999{
5000 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5001 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5002 u32 msg = E1000_VT_MSGTYPE_NACK;
5003
5004 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5005 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5006 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5007 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5008 vf_data->last_nack = jiffies;
4ae196df
AD		 5009 }
5010}
5011
f2ca0dbe 5012static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5013{
f2ca0dbe
AD		 5014 struct pci_dev *pdev = adapter->pdev;
5015 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5016 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5017 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5018 s32 retval;
5019
f2ca0dbe 5020 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5021
fef45f4c
AD		 5022 if (retval) {
5023 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5024 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5025 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5026 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5027 return;
5028 goto out;
5029 }
4ae196df
AD		 5030
5031 /* this is a message we already processed, do nothing */
5032 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5033 return;
4ae196df
AD		 5034
5035 /*
5036 * until the vf completes a reset it should not be
5037 * allowed to start any configuration.
5038 */
5039
5040 if (msgbuf[0] == E1000_VF_RESET) {
5041 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5042 return;
4ae196df
AD		 5043 }
5044
f2ca0dbe 5045 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5046 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5047 return;
5048 retval = -1;
5049 goto out;
4ae196df
AD		 5050 }
5051
5052 switch ((msgbuf[0] & 0xFFFF)) {
5053 case E1000_VF_SET_MAC_ADDR:
5054 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5055 break;
7d5753f0
AD		 5056 case E1000_VF_SET_PROMISC:
5057 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5058 break;
4ae196df
AD		 5059 case E1000_VF_SET_MULTICAST:
5060 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5061 break;
5062 case E1000_VF_SET_LPE:
5063 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5064 break;
5065 case E1000_VF_SET_VLAN:
8151d294
WM		 5066 if (adapter->vf_data[vf].pf_vlan)
5067 retval = -1;
5068 else
5069 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5070 break;
5071 default:
090b1795 5072 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5073 retval = -1;
5074 break;
5075 }
5076
fef45f4c
AD		 5077 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5078out:
4ae196df
AD		 5079 /* notify the VF of the results of what it sent us */
5080 if (retval)
5081 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5082 else
5083 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5084
4ae196df 5085 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5086}
4ae196df 5087
f2ca0dbe
AD		 5088static void igb_msg_task(struct igb_adapter *adapter)
5089{
5090 struct e1000_hw *hw = &adapter->hw;
5091 u32 vf;
5092
5093 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5094 /* process any reset requests */
5095 if (!igb_check_for_rst(hw, vf))
5096 igb_vf_reset_event(adapter, vf);
5097
5098 /* process any messages pending */
5099 if (!igb_check_for_msg(hw, vf))
5100 igb_rcv_msg_from_vf(adapter, vf);
5101
5102 /* process any acks */
5103 if (!igb_check_for_ack(hw, vf))
5104 igb_rcv_ack_from_vf(adapter, vf);
5105 }
4ae196df
AD		 5106}
5107
68d480c4
AD		 5108/**
5109 * igb_set_uta - Set unicast filter table address
5110 * @adapter: board private structure
5111 *
5112 * The unicast table address is a register array of 32-bit registers.
5113 * The table is meant to be used in a way similar to how the MTA is used
5114 * however due to certain limitations in the hardware it is necessary to
5115 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5116 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5117 **/
5118static void igb_set_uta(struct igb_adapter *adapter)
5119{
5120 struct e1000_hw *hw = &adapter->hw;
5121 int i;
5122
5123 /* The UTA table only exists on 82576 hardware and newer */
5124 if (hw->mac.type < e1000_82576)
5125 return;
5126
5127 /* we only need to do this if VMDq is enabled */
5128 if (!adapter->vfs_allocated_count)
5129 return;
5130
5131 for (i = 0; i < hw->mac.uta_reg_count; i++)
5132 array_wr32(E1000_UTA, i, ~0);
5133}
5134
9d5c8243
AK		 5135/**
5136 * igb_intr_msi - Interrupt Handler
5137 * @irq: interrupt number
5138 * @data: pointer to a network interface device structure
5139 **/
5140static irqreturn_t igb_intr_msi(int irq, void *data)
5141{
047e0030
AD		 5142 struct igb_adapter *adapter = data;
5143 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5144 struct e1000_hw *hw = &adapter->hw;
5145 /* read ICR disables interrupts using IAM */
5146 u32 icr = rd32(E1000_ICR);
5147
047e0030 5148 igb_write_itr(q_vector);
9d5c8243 5149
7f081d40
AD		 5150 if (icr & E1000_ICR_DRSTA)
5151 schedule_work(&adapter->reset_task);
5152
047e0030 5153 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5154 /* HW is reporting DMA is out of sync */
5155 adapter->stats.doosync++;
5156 }
5157
9d5c8243
AK		 5158 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5159 hw->mac.get_link_status = 1;
5160 if (!test_bit(__IGB_DOWN, &adapter->state))
5161 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5162 }
5163
047e0030 5164 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5165
5166 return IRQ_HANDLED;
5167}
5168
5169/**
4a3c6433 5170 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5171 * @irq: interrupt number
5172 * @data: pointer to a network interface device structure
5173 **/
5174static irqreturn_t igb_intr(int irq, void *data)
5175{
047e0030
AD		 5176 struct igb_adapter *adapter = data;
5177 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5178 struct e1000_hw *hw = &adapter->hw;
5179 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5180 * need for the IMC write */
5181 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5182 if (!icr)
5183 return IRQ_NONE; /* Not our interrupt */
5184
047e0030 5185 igb_write_itr(q_vector);
9d5c8243
AK		 5186
5187 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5188 * not set, then the adapter didn't send an interrupt */
5189 if (!(icr & E1000_ICR_INT_ASSERTED))
5190 return IRQ_NONE;
5191
7f081d40
AD		 5192 if (icr & E1000_ICR_DRSTA)
5193 schedule_work(&adapter->reset_task);
5194
047e0030 5195 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5196 /* HW is reporting DMA is out of sync */
5197 adapter->stats.doosync++;
5198 }
5199
9d5c8243
AK		 5200 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5201 hw->mac.get_link_status = 1;
5202 /* guard against interrupt when we're going down */
5203 if (!test_bit(__IGB_DOWN, &adapter->state))
5204 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5205 }
5206
047e0030 5207 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5208
5209 return IRQ_HANDLED;
5210}
5211
047e0030 5212static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5213{
047e0030 5214 struct igb_adapter *adapter = q_vector->adapter;
46544258 5215 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5216
4fc82adf
AD		 5217 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5218 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5219 if (!adapter->msix_entries)
6eb5a7f1 5220 igb_set_itr(adapter);
46544258 5221 else
047e0030 5222 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5223 }
5224
46544258
AD		 5225 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5226 if (adapter->msix_entries)
047e0030 5227 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5228 else
5229 igb_irq_enable(adapter);
5230 }
9d5c8243
AK		 5231}
5232
46544258
AD		 5233/**
5234 * igb_poll - NAPI Rx polling callback
5235 * @napi: napi polling structure
5236 * @budget: count of how many packets we should handle
5237 **/
5238static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5239{
047e0030
AD		 5240 struct igb_q_vector *q_vector = container_of(napi,
5241 struct igb_q_vector,
5242 napi);
5243 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5244
421e02f0 5245#ifdef CONFIG_IGB_DCA
047e0030
AD		 5246 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5247 igb_update_dca(q_vector);
fe4506b6 5248#endif
047e0030
AD		 5249 if (q_vector->tx_ring)
5250 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5251
047e0030
AD		 5252 if (q_vector->rx_ring)
5253 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5254
5255 if (!tx_clean_complete)
5256 work_done = budget;
46544258 5257
9d5c8243 5258 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5259 if (work_done < budget) {
288379f0 5260 napi_complete(napi);
047e0030 5261 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5262 }
5263
46544258 5264 return work_done;
9d5c8243 5265}
6d8126f9 5266
33af6bcc 5267/**
c5b9bd5e 5268 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5269 * @adapter: board private structure
c5b9bd5e
AD		 5270 * @shhwtstamps: timestamp structure to update
5271 * @regval: unsigned 64bit system time value.
5272 *
5273 * We need to convert the system time value stored in the RX/TXSTMP registers
5274 * into a hwtstamp which can be used by the upper level timestamping functions
5275 */
5276static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5277 struct skb_shared_hwtstamps *shhwtstamps,
5278 u64 regval)
5279{
5280 u64 ns;
5281
55cac248
AD		 5282 /*
5283 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5284 * 24 to match clock shift we setup earlier.
5285 */
5286 if (adapter->hw.mac.type == e1000_82580)
5287 regval <<= IGB_82580_TSYNC_SHIFT;
5288
c5b9bd5e
AD		 5289 ns = timecounter_cyc2time(&adapter->clock, regval);
5290 timecompare_update(&adapter->compare, ns);
5291 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5292 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5293 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5294}
5295
5296/**
5297 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5298 * @q_vector: pointer to q_vector containing needed info
2873957d 5299 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5300 *
5301 * If we were asked to do hardware stamping and such a time stamp is
5302 * available, then it must have been for this skb here because we only
5303 * allow only one such packet into the queue.
5304 */
2873957d 5305static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5306{
c5b9bd5e 5307 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5308 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5309 struct skb_shared_hwtstamps shhwtstamps;
5310 u64 regval;
33af6bcc 5311
c5b9bd5e 5312 /* if skb does not support hw timestamp or TX stamp not valid exit */
2873957d 5313 if (likely(!buffer_info->shtx.hardware) ||
c5b9bd5e
AD		 5314 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5315 return;
5316
5317 regval = rd32(E1000_TXSTMPL);
5318 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5319
5320 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5321 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5322}
5323
9d5c8243
AK		 5324/**
5325 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5326 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5327 * returns true if ring is completely cleaned
5328 **/
047e0030 5329static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5330{
047e0030
AD		 5331 struct igb_adapter *adapter = q_vector->adapter;
5332 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5333 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5334 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5335 struct igb_buffer *buffer_info;
0e014cb1 5336 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5337 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5338 unsigned int i, eop, count = 0;
5339 bool cleaned = false;
9d5c8243 5340
9d5c8243 5341 i = tx_ring->next_to_clean;
0e014cb1
AD		 5342 eop = tx_ring->buffer_info[i].next_to_watch;
5343 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5344
5345 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5346 (count < tx_ring->count)) {
5347 for (cleaned = false; !cleaned; count++) {
5348 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5349 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5350 cleaned = (i == eop);
9d5c8243 5351
2873957d
NN		 5352 if (buffer_info->skb) {
5353 total_bytes += buffer_info->bytecount;
9d5c8243 5354 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5355 total_packets += buffer_info->gso_segs;
5356 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5357 }
5358
80785298 5359 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5360 tx_desc->wb.status = 0;
9d5c8243
AK		 5361
5362 i++;
5363 if (i == tx_ring->count)
5364 i = 0;
9d5c8243 5365 }
0e014cb1
AD		 5366 eop = tx_ring->buffer_info[i].next_to_watch;
5367 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5368 }
5369
9d5c8243
AK		 5370 tx_ring->next_to_clean = i;
5371
fc7d345d 5372 if (unlikely(count &&
9d5c8243 5373 netif_carrier_ok(netdev) &&
c493ea45 5374 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5375 /* Make sure that anybody stopping the queue after this
5376 * sees the new next_to_clean.
5377 */
5378 smp_mb();
661086df
PWJ		 5379 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5380 !(test_bit(__IGB_DOWN, &adapter->state))) {
5381 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5382 tx_ring->tx_stats.restart_queue++;
661086df 5383 }
9d5c8243
AK		 5384 }
5385
5386 if (tx_ring->detect_tx_hung) {
5387 /* Detect a transmit hang in hardware, this serializes the
5388 * check with the clearing of time_stamp and movement of i */
5389 tx_ring->detect_tx_hung = false;
5390 if (tx_ring->buffer_info[i].time_stamp &&
5391 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5392 (adapter->tx_timeout_factor * HZ)) &&
5393 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5394
9d5c8243 5395 /* detected Tx unit hang */
59d71989 5396 dev_err(tx_ring->dev,
9d5c8243 5397 "Detected Tx Unit Hang\n"
2d064c06 5398 " Tx Queue <%d>\n"
9d5c8243
AK		 5399 " TDH <%x>\n"
5400 " TDT <%x>\n"
5401 " next_to_use <%x>\n"
5402 " next_to_clean <%x>\n"
9d5c8243
AK		 5403 "buffer_info[next_to_clean]\n"
5404 " time_stamp <%lx>\n"
0e014cb1 5405 " next_to_watch <%x>\n"
9d5c8243
AK		 5406 " jiffies <%lx>\n"
5407 " desc.status <%x>\n",
2d064c06 5408 tx_ring->queue_index,
fce99e34
AD		 5409 readl(tx_ring->head),
5410 readl(tx_ring->tail),
9d5c8243
AK		 5411 tx_ring->next_to_use,
5412 tx_ring->next_to_clean,
f7ba205e 5413 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5414 eop,
9d5c8243 5415 jiffies,
0e014cb1 5416 eop_desc->wb.status);
661086df 5417 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5418 }
5419 }
5420 tx_ring->total_bytes += total_bytes;
5421 tx_ring->total_packets += total_packets;
e21ed353
AD		 5422 tx_ring->tx_stats.bytes += total_bytes;
5423 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5424 return (count < tx_ring->count);
9d5c8243
AK		 5425}
5426
9d5c8243
AK		 5427/**
5428 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5429 * @q_vector: structure containing interrupt and ring information
5430 * @skb: packet to send up
5431 * @vlan_tag: vlan tag for packet
9d5c8243 5432 **/
047e0030
AD		 5433static void igb_receive_skb(struct igb_q_vector *q_vector,
5434 struct sk_buff *skb,
5435 u16 vlan_tag)
5436{
5437 struct igb_adapter *adapter = q_vector->adapter;
5438
31b24b95 5439 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5440 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5441 vlan_tag, skb);
182ff8df 5442 else
047e0030 5443 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5444}
5445
04a5fcaa 5446static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5447 u32 status_err, struct sk_buff *skb)
5448{
5449 skb->ip_summed = CHECKSUM_NONE;
5450
5451 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5452 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5453 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5454 return;
85ad76b2 5455
9d5c8243
AK		 5456 /* TCP/UDP checksum error bit is set */
5457 if (status_err &
5458 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5459 /*
5460 * work around errata with sctp packets where the TCPE aka
5461 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5462 * packets, (aka let the stack check the crc32c)
5463 */
85ad76b2
AD		 5464 if ((skb->len == 60) &&
5465 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5466 ring->rx_stats.csum_err++;
85ad76b2 5467
9d5c8243 5468 /* let the stack verify checksum errors */
9d5c8243
AK		 5469 return;
5470 }
5471 /* It must be a TCP or UDP packet with a valid checksum */
5472 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5473 skb->ip_summed = CHECKSUM_UNNECESSARY;
5474
59d71989 5475 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5476}
5477
757b77e2 5478static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5479 struct sk_buff *skb)
5480{
5481 struct igb_adapter *adapter = q_vector->adapter;
5482 struct e1000_hw *hw = &adapter->hw;
5483 u64 regval;
5484
5485 /*
5486 * If this bit is set, then the RX registers contain the time stamp. No
5487 * other packet will be time stamped until we read these registers, so
5488 * read the registers to make them available again. Because only one
5489 * packet can be time stamped at a time, we know that the register
5490 * values must belong to this one here and therefore we don't need to
5491 * compare any of the additional attributes stored for it.
5492 *
5493 * If nothing went wrong, then it should have a skb_shared_tx that we
5494 * can turn into a skb_shared_hwtstamps.
5495 */
757b77e2
NN		 5496 if (staterr & E1000_RXDADV_STAT_TSIP) {
5497 u32 *stamp = (u32 *)skb->data;
5498 regval = le32_to_cpu(*(stamp + 2));
5499 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5500 skb_pull(skb, IGB_TS_HDR_LEN);
5501 } else {
5502 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5503 return;
c5b9bd5e 5504
757b77e2
NN		 5505 regval = rd32(E1000_RXSTMPL);
5506 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5507 }
c5b9bd5e
AD		 5508
5509 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5510}
4c844851 5511static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5512 union e1000_adv_rx_desc *rx_desc)
5513{
5514 /* HW will not DMA in data larger than the given buffer, even if it
5515 * parses the (NFS, of course) header to be larger. In that case, it
5516 * fills the header buffer and spills the rest into the page.
5517 */
5518 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5519 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5520 if (hlen > rx_ring->rx_buffer_len)
5521 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5522 return hlen;
5523}
5524
047e0030
AD		 5525static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5526 int *work_done, int budget)
9d5c8243 5527{
047e0030 5528 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5529 struct net_device *netdev = rx_ring->netdev;
59d71989 5530 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5531 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5532 struct igb_buffer *buffer_info , *next_buffer;
5533 struct sk_buff *skb;
9d5c8243
AK		 5534 bool cleaned = false;
5535 int cleaned_count = 0;
d1eff350 5536 int current_node = numa_node_id();
9d5c8243 5537 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5538 unsigned int i;
2d94d8ab
AD		 5539 u32 staterr;
5540 u16 length;
047e0030 5541 u16 vlan_tag;
9d5c8243
AK		 5542
5543 i = rx_ring->next_to_clean;
69d3ca53 5544 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5545 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5546 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5547
5548 while (staterr & E1000_RXD_STAT_DD) {
5549 if (*work_done >= budget)
5550 break;
5551 (*work_done)++;
9d5c8243 5552
69d3ca53
AD		 5553 skb = buffer_info->skb;
5554 prefetch(skb->data - NET_IP_ALIGN);
5555 buffer_info->skb = NULL;
5556
5557 i++;
5558 if (i == rx_ring->count)
5559 i = 0;
42d0781a 5560
69d3ca53
AD		 5561 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5562 prefetch(next_rxd);
5563 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5564
5565 length = le16_to_cpu(rx_desc->wb.upper.length);
5566 cleaned = true;
5567 cleaned_count++;
5568
2d94d8ab 5569 if (buffer_info->dma) {
59d71989 5570 dma_unmap_single(dev, buffer_info->dma,
4c844851 5571 rx_ring->rx_buffer_len,
59d71989 5572 DMA_FROM_DEVICE);
91615f76 5573 buffer_info->dma = 0;
4c844851 5574 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5575 skb_put(skb, length);
5576 goto send_up;
5577 }
4c844851 5578 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5579 }
5580
5581 if (length) {
59d71989
AD		 5582 dma_unmap_page(dev, buffer_info->page_dma,
5583 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5584 buffer_info->page_dma = 0;
bf36c1a0 5585
aa913403 5586 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5587 buffer_info->page,
5588 buffer_info->page_offset,
5589 length);
5590
d1eff350
AD		 5591 if ((page_count(buffer_info->page) != 1) ||
5592 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5593 buffer_info->page = NULL;
5594 else
5595 get_page(buffer_info->page);
9d5c8243
AK		 5596
5597 skb->len += length;
5598 skb->data_len += length;
bf36c1a0 5599 skb->truesize += length;
9d5c8243 5600 }
9d5c8243 5601
bf36c1a0 5602 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5603 buffer_info->skb = next_buffer->skb;
5604 buffer_info->dma = next_buffer->dma;
5605 next_buffer->skb = skb;
5606 next_buffer->dma = 0;
bf36c1a0
AD		 5607 goto next_desc;
5608 }
69d3ca53 5609send_up:
9d5c8243
AK		 5610 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5611 dev_kfree_skb_irq(skb);
5612 goto next_desc;
5613 }
9d5c8243 5614
757b77e2
NN		 5615 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5616 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5617 total_bytes += skb->len;
5618 total_packets++;
5619
85ad76b2 5620 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5621
5622 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5623 skb_record_rx_queue(skb, rx_ring->queue_index);
5624
5625 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5626 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5627
047e0030 5628 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5629
9d5c8243
AK		 5630next_desc:
5631 rx_desc->wb.upper.status_error = 0;
5632
5633 /* return some buffers to hardware, one at a time is too slow */
5634 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5635 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5636 cleaned_count = 0;
5637 }
5638
5639 /* use prefetched values */
5640 rx_desc = next_rxd;
5641 buffer_info = next_buffer;
9d5c8243
AK		 5642 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5643 }
bf36c1a0 5644
9d5c8243 5645 rx_ring->next_to_clean = i;
c493ea45 5646 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5647
5648 if (cleaned_count)
3b644cf6 5649 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5650
5651 rx_ring->total_packets += total_packets;
5652 rx_ring->total_bytes += total_bytes;
5653 rx_ring->rx_stats.packets += total_packets;
5654 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5655 return cleaned;
5656}
5657
9d5c8243
AK		 5658/**
5659 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5660 * @adapter: address of board private structure
5661 **/
d7ee5b3a 5662void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5663{
e694e964 5664 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5665 union e1000_adv_rx_desc *rx_desc;
5666 struct igb_buffer *buffer_info;
5667 struct sk_buff *skb;
5668 unsigned int i;
db761762 5669 int bufsz;
9d5c8243
AK		 5670
5671 i = rx_ring->next_to_use;
5672 buffer_info = &rx_ring->buffer_info[i];
5673
4c844851 5674 bufsz = rx_ring->rx_buffer_len;
db761762 5675
9d5c8243
AK		 5676 while (cleaned_count--) {
5677 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5678
6ec43fe6 5679 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5680 if (!buffer_info->page) {
42d0781a 5681 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5682 if (!buffer_info->page) {
04a5fcaa 5683 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5684 goto no_buffers;
5685 }
5686 buffer_info->page_offset = 0;
5687 } else {
5688 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5689 }
5690 buffer_info->page_dma =
59d71989 5691 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5692 buffer_info->page_offset,
5693 PAGE_SIZE / 2,
59d71989
AD		 5694 DMA_FROM_DEVICE);
5695 if (dma_mapping_error(rx_ring->dev,
5696 buffer_info->page_dma)) {
42d0781a
AD		 5697 buffer_info->page_dma = 0;
5698 rx_ring->rx_stats.alloc_failed++;
5699 goto no_buffers;
5700 }
9d5c8243
AK		 5701 }
5702
42d0781a
AD		 5703 skb = buffer_info->skb;
5704 if (!skb) {
89d71a66 5705 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5706 if (!skb) {
04a5fcaa 5707 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5708 goto no_buffers;
5709 }
5710
9d5c8243 5711 buffer_info->skb = skb;
42d0781a
AD		 5712 }
5713 if (!buffer_info->dma) {
59d71989 5714 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5715 skb->data,
9d5c8243 5716 bufsz,
59d71989
AD		 5717 DMA_FROM_DEVICE);
5718 if (dma_mapping_error(rx_ring->dev,
5719 buffer_info->dma)) {
42d0781a
AD		 5720 buffer_info->dma = 0;
5721 rx_ring->rx_stats.alloc_failed++;
5722 goto no_buffers;
5723 }
9d5c8243
AK		 5724 }
5725 /* Refresh the desc even if buffer_addrs didn't change because
5726 * each write-back erases this info. */
6ec43fe6 5727 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5728 rx_desc->read.pkt_addr =
5729 cpu_to_le64(buffer_info->page_dma);
5730 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5731 } else {
42d0781a 5732 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5733 rx_desc->read.hdr_addr = 0;
5734 }
5735
5736 i++;
5737 if (i == rx_ring->count)
5738 i = 0;
5739 buffer_info = &rx_ring->buffer_info[i];
5740 }
5741
5742no_buffers:
5743 if (rx_ring->next_to_use != i) {
5744 rx_ring->next_to_use = i;
5745 if (i == 0)
5746 i = (rx_ring->count - 1);
5747 else
5748 i--;
5749
5750 /* Force memory writes to complete before letting h/w
5751 * know there are new descriptors to fetch. (Only
5752 * applicable for weak-ordered memory model archs,
5753 * such as IA-64). */
5754 wmb();
fce99e34 5755 writel(i, rx_ring->tail);
9d5c8243
AK		 5756 }
5757}
5758
5759/**
5760 * igb_mii_ioctl -
5761 * @netdev:
5762 * @ifreq:
5763 * @cmd:
5764 **/
5765static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5766{
5767 struct igb_adapter *adapter = netdev_priv(netdev);
5768 struct mii_ioctl_data *data = if_mii(ifr);
5769
5770 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5771 return -EOPNOTSUPP;
5772
5773 switch (cmd) {
5774 case SIOCGMIIPHY:
5775 data->phy_id = adapter->hw.phy.addr;
5776 break;
5777 case SIOCGMIIREG:
f5f4cf08
AD		 5778 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5779 &data->val_out))
9d5c8243
AK		 5780 return -EIO;
5781 break;
5782 case SIOCSMIIREG:
5783 default:
5784 return -EOPNOTSUPP;
5785 }
5786 return 0;
5787}
5788
c6cb090b
PO		 5789/**
5790 * igb_hwtstamp_ioctl - control hardware time stamping
5791 * @netdev:
5792 * @ifreq:
5793 * @cmd:
5794 *
33af6bcc
PO		 5795 * Outgoing time stamping can be enabled and disabled. Play nice and
5796 * disable it when requested, although it shouldn't case any overhead
5797 * when no packet needs it. At most one packet in the queue may be
5798 * marked for time stamping, otherwise it would be impossible to tell
5799 * for sure to which packet the hardware time stamp belongs.
5800 *
5801 * Incoming time stamping has to be configured via the hardware
5802 * filters. Not all combinations are supported, in particular event
5803 * type has to be specified. Matching the kind of event packet is
5804 * not supported, with the exception of "all V2 events regardless of
5805 * level 2 or 4".
5806 *
c6cb090b
PO		 5807 **/
5808static int igb_hwtstamp_ioctl(struct net_device *netdev,
5809 struct ifreq *ifr, int cmd)
5810{
33af6bcc
PO		 5811 struct igb_adapter *adapter = netdev_priv(netdev);
5812 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5813 struct hwtstamp_config config;
c5b9bd5e
AD		 5814 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5815 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5816 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5817 bool is_l4 = false;
5818 bool is_l2 = false;
33af6bcc 5819 u32 regval;
c6cb090b
PO		 5820
5821 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5822 return -EFAULT;
5823
5824 /* reserved for future extensions */
5825 if (config.flags)
5826 return -EINVAL;
5827
33af6bcc
PO		 5828 switch (config.tx_type) {
5829 case HWTSTAMP_TX_OFF:
c5b9bd5e 5830 tsync_tx_ctl = 0;
33af6bcc 5831 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5832 break;
5833 default:
5834 return -ERANGE;
5835 }
5836
5837 switch (config.rx_filter) {
5838 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5839 tsync_rx_ctl = 0;
33af6bcc
PO		 5840 break;
5841 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5842 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5843 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5844 case HWTSTAMP_FILTER_ALL:
5845 /*
5846 * register TSYNCRXCFG must be set, therefore it is not
5847 * possible to time stamp both Sync and Delay_Req messages
5848 * => fall back to time stamping all packets
5849 */
c5b9bd5e 5850 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5851 config.rx_filter = HWTSTAMP_FILTER_ALL;
5852 break;
5853 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5854 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5855 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5856 is_l4 = true;
33af6bcc
PO		 5857 break;
5858 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5859 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5860 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5861 is_l4 = true;
33af6bcc
PO		 5862 break;
5863 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5864 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5865 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5866 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5867 is_l2 = true;
5868 is_l4 = true;
33af6bcc
PO		 5869 config.rx_filter = HWTSTAMP_FILTER_SOME;
5870 break;
5871 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5872 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5873 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5874 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5875 is_l2 = true;
5876 is_l4 = true;
33af6bcc
PO		 5877 config.rx_filter = HWTSTAMP_FILTER_SOME;
5878 break;
5879 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5880 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5881 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5882 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5883 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5884 is_l2 = true;
33af6bcc
PO		 5885 break;
5886 default:
5887 return -ERANGE;
5888 }
5889
c5b9bd5e
AD		 5890 if (hw->mac.type == e1000_82575) {
5891 if (tsync_rx_ctl | tsync_tx_ctl)
5892 return -EINVAL;
5893 return 0;
5894 }
5895
757b77e2
NN		 5896 /*
5897 * Per-packet timestamping only works if all packets are
5898 * timestamped, so enable timestamping in all packets as
5899 * long as one rx filter was configured.
5900 */
5901 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5902 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5903 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5904 }
5905
33af6bcc
PO		 5906 /* enable/disable TX */
5907 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5908 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5909 regval |= tsync_tx_ctl;
33af6bcc
PO		 5910 wr32(E1000_TSYNCTXCTL, regval);
5911
c5b9bd5e 5912 /* enable/disable RX */
33af6bcc 5913 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5914 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5915 regval |= tsync_rx_ctl;
33af6bcc 5916 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5917
c5b9bd5e
AD		 5918 /* define which PTP packets are time stamped */
5919 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5920
c5b9bd5e
AD		 5921 /* define ethertype filter for timestamped packets */
5922 if (is_l2)
5923 wr32(E1000_ETQF(3),
5924 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5925 E1000_ETQF_1588 | /* enable timestamping */
5926 ETH_P_1588)); /* 1588 eth protocol type */
5927 else
5928 wr32(E1000_ETQF(3), 0);
5929
5930#define PTP_PORT 319
5931 /* L4 Queue Filter[3]: filter by destination port and protocol */
5932 if (is_l4) {
5933 u32 ftqf = (IPPROTO_UDP /* UDP */
5934 | E1000_FTQF_VF_BP /* VF not compared */
5935 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5936 | E1000_FTQF_MASK); /* mask all inputs */
5937 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5938
5939 wr32(E1000_IMIR(3), htons(PTP_PORT));
5940 wr32(E1000_IMIREXT(3),
5941 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5942 if (hw->mac.type == e1000_82576) {
5943 /* enable source port check */
5944 wr32(E1000_SPQF(3), htons(PTP_PORT));
5945 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5946 }
5947 wr32(E1000_FTQF(3), ftqf);
5948 } else {
5949 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5950 }
33af6bcc
PO		 5951 wrfl();
5952
5953 adapter->hwtstamp_config = config;
5954
5955 /* clear TX/RX time stamp registers, just to be sure */
5956 regval = rd32(E1000_TXSTMPH);
5957 regval = rd32(E1000_RXSTMPH);
c6cb090b 5958
33af6bcc
PO		 5959 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5960 -EFAULT : 0;
c6cb090b
PO		 5961}
5962
9d5c8243
AK		 5963/**
5964 * igb_ioctl -
5965 * @netdev:
5966 * @ifreq:
5967 * @cmd:
5968 **/
5969static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5970{
5971 switch (cmd) {
5972 case SIOCGMIIPHY:
5973 case SIOCGMIIREG:
5974 case SIOCSMIIREG:
5975 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5976 case SIOCSHWTSTAMP:
5977 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5978 default:
5979 return -EOPNOTSUPP;
5980 }
5981}
5982
009bc06e
AD		 5983s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5984{
5985 struct igb_adapter *adapter = hw->back;
5986 u16 cap_offset;
5987
5988 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5989 if (!cap_offset)
5990 return -E1000_ERR_CONFIG;
5991
5992 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5993
5994 return 0;
5995}
5996
5997s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5998{
5999 struct igb_adapter *adapter = hw->back;
6000 u16 cap_offset;
6001
6002 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6003 if (!cap_offset)
6004 return -E1000_ERR_CONFIG;
6005
6006 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6007
6008 return 0;
6009}
6010
9d5c8243
AK		 6011static void igb_vlan_rx_register(struct net_device *netdev,
6012 struct vlan_group *grp)
6013{
6014 struct igb_adapter *adapter = netdev_priv(netdev);
6015 struct e1000_hw *hw = &adapter->hw;
6016 u32 ctrl, rctl;
6017
6018 igb_irq_disable(adapter);
6019 adapter->vlgrp = grp;
6020
6021 if (grp) {
6022 /* enable VLAN tag insert/strip */
6023 ctrl = rd32(E1000_CTRL);
6024 ctrl |= E1000_CTRL_VME;
6025 wr32(E1000_CTRL, ctrl);
6026
51466239 6027 /* Disable CFI check */
9d5c8243 6028 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6029 rctl &= ~E1000_RCTL_CFIEN;
6030 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6031 } else {
6032 /* disable VLAN tag insert/strip */
6033 ctrl = rd32(E1000_CTRL);
6034 ctrl &= ~E1000_CTRL_VME;
6035 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6036 }
6037
e1739522
AD		 6038 igb_rlpml_set(adapter);
6039
9d5c8243
AK		 6040 if (!test_bit(__IGB_DOWN, &adapter->state))
6041 igb_irq_enable(adapter);
6042}
6043
6044static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6045{
6046 struct igb_adapter *adapter = netdev_priv(netdev);
6047 struct e1000_hw *hw = &adapter->hw;
4ae196df 6048 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6049
51466239
AD		 6050 /* attempt to add filter to vlvf array */
6051 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6052
51466239
AD		 6053 /* add the filter since PF can receive vlans w/o entry in vlvf */
6054 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6055}
6056
6057static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6058{
6059 struct igb_adapter *adapter = netdev_priv(netdev);
6060 struct e1000_hw *hw = &adapter->hw;
4ae196df 6061 int pf_id = adapter->vfs_allocated_count;
51466239 6062 s32 err;
9d5c8243
AK		 6063
6064 igb_irq_disable(adapter);
6065 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6066
6067 if (!test_bit(__IGB_DOWN, &adapter->state))
6068 igb_irq_enable(adapter);
6069
51466239
AD		 6070 /* remove vlan from VLVF table array */
6071 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6072
51466239
AD		 6073 /* if vid was not present in VLVF just remove it from table */
6074 if (err)
4ae196df 6075 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6076}
6077
6078static void igb_restore_vlan(struct igb_adapter *adapter)
6079{
6080 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6081
6082 if (adapter->vlgrp) {
6083 u16 vid;
6084 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6085 if (!vlan_group_get_device(adapter->vlgrp, vid))
6086 continue;
6087 igb_vlan_rx_add_vid(adapter->netdev, vid);
6088 }
6089 }
6090}
6091
6092int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6093{
090b1795 6094 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6095 struct e1000_mac_info *mac = &adapter->hw.mac;
6096
6097 mac->autoneg = 0;
6098
9d5c8243
AK		 6099 switch (spddplx) {
6100 case SPEED_10 + DUPLEX_HALF:
6101 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6102 break;
6103 case SPEED_10 + DUPLEX_FULL:
6104 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6105 break;
6106 case SPEED_100 + DUPLEX_HALF:
6107 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6108 break;
6109 case SPEED_100 + DUPLEX_FULL:
6110 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6111 break;
6112 case SPEED_1000 + DUPLEX_FULL:
6113 mac->autoneg = 1;
6114 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6115 break;
6116 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6117 default:
090b1795 6118 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6119 return -EINVAL;
6120 }
6121 return 0;
6122}
6123
3fe7c4c9 6124static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6125{
6126 struct net_device *netdev = pci_get_drvdata(pdev);
6127 struct igb_adapter *adapter = netdev_priv(netdev);
6128 struct e1000_hw *hw = &adapter->hw;
2d064c06 6129 u32 ctrl, rctl, status;
9d5c8243
AK		 6130 u32 wufc = adapter->wol;
6131#ifdef CONFIG_PM
6132 int retval = 0;
6133#endif
6134
6135 netif_device_detach(netdev);
6136
a88f10ec
AD		 6137 if (netif_running(netdev))
6138 igb_close(netdev);
6139
047e0030 6140 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6141
6142#ifdef CONFIG_PM
6143 retval = pci_save_state(pdev);
6144 if (retval)
6145 return retval;
6146#endif
6147
6148 status = rd32(E1000_STATUS);
6149 if (status & E1000_STATUS_LU)
6150 wufc &= ~E1000_WUFC_LNKC;
6151
6152 if (wufc) {
6153 igb_setup_rctl(adapter);
ff41f8dc 6154 igb_set_rx_mode(netdev);
9d5c8243
AK		 6155
6156 /* turn on all-multi mode if wake on multicast is enabled */
6157 if (wufc & E1000_WUFC_MC) {
6158 rctl = rd32(E1000_RCTL);
6159 rctl |= E1000_RCTL_MPE;
6160 wr32(E1000_RCTL, rctl);
6161 }
6162
6163 ctrl = rd32(E1000_CTRL);
6164 /* advertise wake from D3Cold */
6165 #define E1000_CTRL_ADVD3WUC 0x00100000
6166 /* phy power management enable */
6167 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6168 ctrl |= E1000_CTRL_ADVD3WUC;
6169 wr32(E1000_CTRL, ctrl);
6170
9d5c8243 6171 /* Allow time for pending master requests to run */
330a6d6a 6172 igb_disable_pcie_master(hw);
9d5c8243
AK		 6173
6174 wr32(E1000_WUC, E1000_WUC_PME_EN);
6175 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6176 } else {
6177 wr32(E1000_WUC, 0);
6178 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6179 }
6180
3fe7c4c9
RW		 6181 *enable_wake = wufc || adapter->en_mng_pt;
6182 if (!*enable_wake)
88a268c1
NN		 6183 igb_power_down_link(adapter);
6184 else
6185 igb_power_up_link(adapter);
9d5c8243
AK		 6186
6187 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6188 * would have already happened in close and is redundant. */
6189 igb_release_hw_control(adapter);
6190
6191 pci_disable_device(pdev);
6192
9d5c8243
AK		 6193 return 0;
6194}
6195
6196#ifdef CONFIG_PM
3fe7c4c9
RW		 6197static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6198{
6199 int retval;
6200 bool wake;
6201
6202 retval = __igb_shutdown(pdev, &wake);
6203 if (retval)
6204 return retval;
6205
6206 if (wake) {
6207 pci_prepare_to_sleep(pdev);
6208 } else {
6209 pci_wake_from_d3(pdev, false);
6210 pci_set_power_state(pdev, PCI_D3hot);
6211 }
6212
6213 return 0;
6214}
6215
9d5c8243
AK		 6216static int igb_resume(struct pci_dev *pdev)
6217{
6218 struct net_device *netdev = pci_get_drvdata(pdev);
6219 struct igb_adapter *adapter = netdev_priv(netdev);
6220 struct e1000_hw *hw = &adapter->hw;
6221 u32 err;
6222
6223 pci_set_power_state(pdev, PCI_D0);
6224 pci_restore_state(pdev);
b94f2d77 6225 pci_save_state(pdev);
42bfd33a 6226
aed5dec3 6227 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6228 if (err) {
6229 dev_err(&pdev->dev,
6230 "igb: Cannot enable PCI device from suspend\n");
6231 return err;
6232 }
6233 pci_set_master(pdev);
6234
6235 pci_enable_wake(pdev, PCI_D3hot, 0);
6236 pci_enable_wake(pdev, PCI_D3cold, 0);
6237
047e0030 6238 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6240 return -ENOMEM;
9d5c8243
AK		 6241 }
6242
9d5c8243 6243 igb_reset(adapter);
a8564f03
AD		 6244
6245 /* let the f/w know that the h/w is now under the control of the
6246 * driver. */
6247 igb_get_hw_control(adapter);
6248
9d5c8243
AK		 6249 wr32(E1000_WUS, ~0);
6250
a88f10ec
AD		 6251 if (netif_running(netdev)) {
6252 err = igb_open(netdev);
6253 if (err)
6254 return err;
6255 }
9d5c8243
AK		 6256
6257 netif_device_attach(netdev);
6258
9d5c8243
AK		 6259 return 0;
6260}
6261#endif
6262
6263static void igb_shutdown(struct pci_dev *pdev)
6264{
3fe7c4c9
RW		 6265 bool wake;
6266
6267 __igb_shutdown(pdev, &wake);
6268
6269 if (system_state == SYSTEM_POWER_OFF) {
6270 pci_wake_from_d3(pdev, wake);
6271 pci_set_power_state(pdev, PCI_D3hot);
6272 }
9d5c8243
AK		 6273}
6274
6275#ifdef CONFIG_NET_POLL_CONTROLLER
6276/*
6277 * Polling 'interrupt' - used by things like netconsole to send skbs
6278 * without having to re-enable interrupts. It's not called while
6279 * the interrupt routine is executing.
6280 */
6281static void igb_netpoll(struct net_device *netdev)
6282{
6283 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6284 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6285 int i;
9d5c8243 6286
eebbbdba 6287 if (!adapter->msix_entries) {
047e0030 6288 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6289 igb_irq_disable(adapter);
047e0030 6290 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6291 return;
6292 }
9d5c8243 6293
047e0030
AD		 6294 for (i = 0; i < adapter->num_q_vectors; i++) {
6295 struct igb_q_vector *q_vector = adapter->q_vector[i];
6296 wr32(E1000_EIMC, q_vector->eims_value);
6297 napi_schedule(&q_vector->napi);
eebbbdba 6298 }
9d5c8243
AK		 6299}
6300#endif /* CONFIG_NET_POLL_CONTROLLER */
6301
6302/**
6303 * igb_io_error_detected - called when PCI error is detected
6304 * @pdev: Pointer to PCI device
6305 * @state: The current pci connection state
6306 *
6307 * This function is called after a PCI bus error affecting
6308 * this device has been detected.
6309 */
6310static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6311 pci_channel_state_t state)
6312{
6313 struct net_device *netdev = pci_get_drvdata(pdev);
6314 struct igb_adapter *adapter = netdev_priv(netdev);
6315
6316 netif_device_detach(netdev);
6317
59ed6eec
AD		 6318 if (state == pci_channel_io_perm_failure)
6319 return PCI_ERS_RESULT_DISCONNECT;
6320
9d5c8243
AK		 6321 if (netif_running(netdev))
6322 igb_down(adapter);
6323 pci_disable_device(pdev);
6324
6325 /* Request a slot slot reset. */
6326 return PCI_ERS_RESULT_NEED_RESET;
6327}
6328
6329/**
6330 * igb_io_slot_reset - called after the pci bus has been reset.
6331 * @pdev: Pointer to PCI device
6332 *
6333 * Restart the card from scratch, as if from a cold-boot. Implementation
6334 * resembles the first-half of the igb_resume routine.
6335 */
6336static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6337{
6338 struct net_device *netdev = pci_get_drvdata(pdev);
6339 struct igb_adapter *adapter = netdev_priv(netdev);
6340 struct e1000_hw *hw = &adapter->hw;
40a914fa 6341 pci_ers_result_t result;
42bfd33a 6342 int err;
9d5c8243 6343
aed5dec3 6344 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6345 dev_err(&pdev->dev,
6346 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6347 result = PCI_ERS_RESULT_DISCONNECT;
6348 } else {
6349 pci_set_master(pdev);
6350 pci_restore_state(pdev);
b94f2d77 6351 pci_save_state(pdev);
9d5c8243 6352
40a914fa
AD		 6353 pci_enable_wake(pdev, PCI_D3hot, 0);
6354 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6355
40a914fa
AD		 6356 igb_reset(adapter);
6357 wr32(E1000_WUS, ~0);
6358 result = PCI_ERS_RESULT_RECOVERED;
6359 }
9d5c8243 6360
ea943d41
JK		 6361 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6362 if (err) {
6363 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6364 "failed 0x%0x\n", err);
6365 /* non-fatal, continue */
6366 }
40a914fa
AD		 6367
6368 return result;
9d5c8243
AK		 6369}
6370
6371/**
6372 * igb_io_resume - called when traffic can start flowing again.
6373 * @pdev: Pointer to PCI device
6374 *
6375 * This callback is called when the error recovery driver tells us that
6376 * its OK to resume normal operation. Implementation resembles the
6377 * second-half of the igb_resume routine.
6378 */
6379static void igb_io_resume(struct pci_dev *pdev)
6380{
6381 struct net_device *netdev = pci_get_drvdata(pdev);
6382 struct igb_adapter *adapter = netdev_priv(netdev);
6383
9d5c8243
AK		 6384 if (netif_running(netdev)) {
6385 if (igb_up(adapter)) {
6386 dev_err(&pdev->dev, "igb_up failed after reset\n");
6387 return;
6388 }
6389 }
6390
6391 netif_device_attach(netdev);
6392
6393 /* let the f/w know that the h/w is now under the control of the
6394 * driver. */
6395 igb_get_hw_control(adapter);
9d5c8243
AK		 6396}
6397
26ad9178
AD		 6398static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6399 u8 qsel)
6400{
6401 u32 rar_low, rar_high;
6402 struct e1000_hw *hw = &adapter->hw;
6403
6404 /* HW expects these in little endian so we reverse the byte order
6405 * from network order (big endian) to little endian
6406 */
6407 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6408 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6409 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6410
6411 /* Indicate to hardware the Address is Valid. */
6412 rar_high |= E1000_RAH_AV;
6413
6414 if (hw->mac.type == e1000_82575)
6415 rar_high |= E1000_RAH_POOL_1 * qsel;
6416 else
6417 rar_high |= E1000_RAH_POOL_1 << qsel;
6418
6419 wr32(E1000_RAL(index), rar_low);
6420 wrfl();
6421 wr32(E1000_RAH(index), rar_high);
6422 wrfl();
6423}
6424
4ae196df
AD		 6425static int igb_set_vf_mac(struct igb_adapter *adapter,
6426 int vf, unsigned char *mac_addr)
6427{
6428 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6429 /* VF MAC addresses start at end of receive addresses and moves
6430 * torwards the first, as a result a collision should not be possible */
6431 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6432
37680117 6433 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6434
26ad9178 6435 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6436
6437 return 0;
6438}
6439
8151d294
WM		 6440static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6441{
6442 struct igb_adapter *adapter = netdev_priv(netdev);
6443 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6444 return -EINVAL;
6445 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6446 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6447 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6448 " change effective.");
6449 if (test_bit(__IGB_DOWN, &adapter->state)) {
6450 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6451 " but the PF device is not up.\n");
6452 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6453 " attempting to use the VF device.\n");
6454 }
6455 return igb_set_vf_mac(adapter, vf, mac);
6456}
6457
6458static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6459{
6460 return -EOPNOTSUPP;
6461}
6462
6463static int igb_ndo_get_vf_config(struct net_device *netdev,
6464 int vf, struct ifla_vf_info *ivi)
6465{
6466 struct igb_adapter *adapter = netdev_priv(netdev);
6467 if (vf >= adapter->vfs_allocated_count)
6468 return -EINVAL;
6469 ivi->vf = vf;
6470 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6471 ivi->tx_rate = 0;
6472 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6473 ivi->qos = adapter->vf_data[vf].pf_qos;
6474 return 0;
6475}
6476
4ae196df
AD		 6477static void igb_vmm_control(struct igb_adapter *adapter)
6478{
6479 struct e1000_hw *hw = &adapter->hw;
10d8e907 6480 u32 reg;
4ae196df 6481
52a1dd4d
AD		 6482 switch (hw->mac.type) {
6483 case e1000_82575:
6484 default:
6485 /* replication is not supported for 82575 */
4ae196df 6486 return;
52a1dd4d
AD		 6487 case e1000_82576:
6488 /* notify HW that the MAC is adding vlan tags */
6489 reg = rd32(E1000_DTXCTL);
6490 reg |= E1000_DTXCTL_VLAN_ADDED;
6491 wr32(E1000_DTXCTL, reg);
6492 case e1000_82580:
6493 /* enable replication vlan tag stripping */
6494 reg = rd32(E1000_RPLOLR);
6495 reg |= E1000_RPLOLR_STRVLAN;
6496 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6497 case e1000_i350:
6498 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6499 break;
6500 }
10d8e907 6501
d4960307
AD		 6502 if (adapter->vfs_allocated_count) {
6503 igb_vmdq_set_loopback_pf(hw, true);
6504 igb_vmdq_set_replication_pf(hw, true);
6505 } else {
6506 igb_vmdq_set_loopback_pf(hw, false);
6507 igb_vmdq_set_replication_pf(hw, false);
6508 }
4ae196df
AD		 6509}
6510
9d5c8243 6511/* igb_main.c */
Clone of davem's net-next-2.6 tree