]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'pm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/suspe...
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
2d064c06 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
87 /* required last entry */
88 {0, }
89};
90
91MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
92
93void igb_reset(struct igb_adapter *);
94static int igb_setup_all_tx_resources(struct igb_adapter *);
95static int igb_setup_all_rx_resources(struct igb_adapter *);
96static void igb_free_all_tx_resources(struct igb_adapter *);
97static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 98static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 99static int igb_probe(struct pci_dev *, const struct pci_device_id *);
100static void __devexit igb_remove(struct pci_dev *pdev);
101static int igb_sw_init(struct igb_adapter *);
102static int igb_open(struct net_device *);
103static int igb_close(struct net_device *);
104static void igb_configure_tx(struct igb_adapter *);
105static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 106static void igb_clean_all_tx_rings(struct igb_adapter *);
107static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 108static void igb_clean_tx_ring(struct igb_ring *);
109static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 110static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 111static void igb_update_phy_info(unsigned long);
112static void igb_watchdog(unsigned long);
113static void igb_watchdog_task(struct work_struct *);
b1a436c3 114static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 115static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
116 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 117static int igb_change_mtu(struct net_device *, int);
118static int igb_set_mac(struct net_device *, void *);
68d480c4 119static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 120static irqreturn_t igb_intr(int irq, void *);
121static irqreturn_t igb_intr_msi(int irq, void *);
122static irqreturn_t igb_msix_other(int irq, void *);
047e0030 123static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 124#ifdef CONFIG_IGB_DCA
047e0030 125static void igb_update_dca(struct igb_q_vector *);
fe4506b6 126static void igb_setup_dca(struct igb_adapter *);
421e02f0 127#endif /* CONFIG_IGB_DCA */
047e0030 128static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 129static int igb_poll(struct napi_struct *, int);
047e0030 130static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 131static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
132static void igb_tx_timeout(struct net_device *);
133static void igb_reset_task(struct work_struct *);
134static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
135static void igb_vlan_rx_add_vid(struct net_device *, u16);
136static void igb_vlan_rx_kill_vid(struct net_device *, u16);
137static void igb_restore_vlan(struct igb_adapter *);
26ad9178 138static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 139static void igb_ping_all_vfs(struct igb_adapter *);
140static void igb_msg_task(struct igb_adapter *);
4ae196df 141static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 142static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 143static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 144static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
145static int igb_ndo_set_vf_vlan(struct net_device *netdev,
146 int vf, u16 vlan, u8 qos);
147static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
148static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
149 struct ifla_vf_info *ivi);
9d5c8243 150
9d5c8243 151#ifdef CONFIG_PM
3fe7c4c9 152static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 153static int igb_resume(struct pci_dev *);
154#endif
155static void igb_shutdown(struct pci_dev *);
421e02f0 156#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 157static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
158static struct notifier_block dca_notifier = {
159 .notifier_call = igb_notify_dca,
160 .next = NULL,
161 .priority = 0
162};
163#endif
9d5c8243
AK		 164#ifdef CONFIG_NET_POLL_CONTROLLER
165/* for netdump / net console */
166static void igb_netpoll(struct net_device *);
167#endif
37680117 168#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 169static unsigned int max_vfs = 0;
170module_param(max_vfs, uint, 0);
171MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
172 "per physical function");
173#endif /* CONFIG_PCI_IOV */
174
9d5c8243
AK		 175static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
176 pci_channel_state_t);
177static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
178static void igb_io_resume(struct pci_dev *);
179
180static struct pci_error_handlers igb_err_handler = {
181 .error_detected = igb_io_error_detected,
182 .slot_reset = igb_io_slot_reset,
183 .resume = igb_io_resume,
184};
185
186
187static struct pci_driver igb_driver = {
188 .name = igb_driver_name,
189 .id_table = igb_pci_tbl,
190 .probe = igb_probe,
191 .remove = __devexit_p(igb_remove),
192#ifdef CONFIG_PM
193 /* Power Managment Hooks */
194 .suspend = igb_suspend,
195 .resume = igb_resume,
196#endif
197 .shutdown = igb_shutdown,
198 .err_handler = &igb_err_handler
199};
200
201MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
202MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
203MODULE_LICENSE("GPL");
204MODULE_VERSION(DRV_VERSION);
205
c97ec42a
TI		 206struct igb_reg_info {
207 u32 ofs;
208 char *name;
209};
210
211static const struct igb_reg_info igb_reg_info_tbl[] = {
212
213 /* General Registers */
214 {E1000_CTRL, "CTRL"},
215 {E1000_STATUS, "STATUS"},
216 {E1000_CTRL_EXT, "CTRL_EXT"},
217
218 /* Interrupt Registers */
219 {E1000_ICR, "ICR"},
220
221 /* RX Registers */
222 {E1000_RCTL, "RCTL"},
223 {E1000_RDLEN(0), "RDLEN"},
224 {E1000_RDH(0), "RDH"},
225 {E1000_RDT(0), "RDT"},
226 {E1000_RXDCTL(0), "RXDCTL"},
227 {E1000_RDBAL(0), "RDBAL"},
228 {E1000_RDBAH(0), "RDBAH"},
229
230 /* TX Registers */
231 {E1000_TCTL, "TCTL"},
232 {E1000_TDBAL(0), "TDBAL"},
233 {E1000_TDBAH(0), "TDBAH"},
234 {E1000_TDLEN(0), "TDLEN"},
235 {E1000_TDH(0), "TDH"},
236 {E1000_TDT(0), "TDT"},
237 {E1000_TXDCTL(0), "TXDCTL"},
238 {E1000_TDFH, "TDFH"},
239 {E1000_TDFT, "TDFT"},
240 {E1000_TDFHS, "TDFHS"},
241 {E1000_TDFPC, "TDFPC"},
242
243 /* List Terminator */
244 {}
245};
246
247/*
248 * igb_regdump - register printout routine
249 */
250static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
251{
252 int n = 0;
253 char rname[16];
254 u32 regs[8];
255
256 switch (reginfo->ofs) {
257 case E1000_RDLEN(0):
258 for (n = 0; n < 4; n++)
259 regs[n] = rd32(E1000_RDLEN(n));
260 break;
261 case E1000_RDH(0):
262 for (n = 0; n < 4; n++)
263 regs[n] = rd32(E1000_RDH(n));
264 break;
265 case E1000_RDT(0):
266 for (n = 0; n < 4; n++)
267 regs[n] = rd32(E1000_RDT(n));
268 break;
269 case E1000_RXDCTL(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RXDCTL(n));
272 break;
273 case E1000_RDBAL(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDBAL(n));
276 break;
277 case E1000_RDBAH(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDBAH(n));
280 break;
281 case E1000_TDBAL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RDBAL(n));
284 break;
285 case E1000_TDBAH(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_TDBAH(n));
288 break;
289 case E1000_TDLEN(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_TDLEN(n));
292 break;
293 case E1000_TDH(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_TDH(n));
296 break;
297 case E1000_TDT(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDT(n));
300 break;
301 case E1000_TXDCTL(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TXDCTL(n));
304 break;
305 default:
306 printk(KERN_INFO "%-15s %08x\n",
307 reginfo->name, rd32(reginfo->ofs));
308 return;
309 }
310
311 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
312 printk(KERN_INFO "%-15s ", rname);
313 for (n = 0; n < 4; n++)
314 printk(KERN_CONT "%08x ", regs[n]);
315 printk(KERN_CONT "\n");
316}
317
318/*
319 * igb_dump - Print registers, tx-rings and rx-rings
320 */
321static void igb_dump(struct igb_adapter *adapter)
322{
323 struct net_device *netdev = adapter->netdev;
324 struct e1000_hw *hw = &adapter->hw;
325 struct igb_reg_info *reginfo;
326 int n = 0;
327 struct igb_ring *tx_ring;
328 union e1000_adv_tx_desc *tx_desc;
329 struct my_u0 { u64 a; u64 b; } *u0;
330 struct igb_buffer *buffer_info;
331 struct igb_ring *rx_ring;
332 union e1000_adv_rx_desc *rx_desc;
333 u32 staterr;
334 int i = 0;
335
336 if (!netif_msg_hw(adapter))
337 return;
338
339 /* Print netdevice Info */
340 if (netdev) {
341 dev_info(&adapter->pdev->dev, "Net device Info\n");
342 printk(KERN_INFO "Device Name state "
343 "trans_start last_rx\n");
344 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
345 netdev->name,
346 netdev->state,
347 netdev->trans_start,
348 netdev->last_rx);
349 }
350
351 /* Print Registers */
352 dev_info(&adapter->pdev->dev, "Register Dump\n");
353 printk(KERN_INFO " Register Name Value\n");
354 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
355 reginfo->name; reginfo++) {
356 igb_regdump(hw, reginfo);
357 }
358
359 /* Print TX Ring Summary */
360 if (!netdev || !netif_running(netdev))
361 goto exit;
362
363 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
364 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
365 " leng ntw timestamp\n");
366 for (n = 0; n < adapter->num_tx_queues; n++) {
367 tx_ring = adapter->tx_ring[n];
368 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
369 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
370 n, tx_ring->next_to_use, tx_ring->next_to_clean,
371 (u64)buffer_info->dma,
372 buffer_info->length,
373 buffer_info->next_to_watch,
374 (u64)buffer_info->time_stamp);
375 }
376
377 /* Print TX Rings */
378 if (!netif_msg_tx_done(adapter))
379 goto rx_ring_summary;
380
381 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
382
383 /* Transmit Descriptor Formats
384 *
385 * Advanced Transmit Descriptor
386 * +--------------------------------------------------------------+
387 * 0 | Buffer Address [63:0] |
388 * +--------------------------------------------------------------+
389 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
390 * +--------------------------------------------------------------+
391 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 */
393
394 for (n = 0; n < adapter->num_tx_queues; n++) {
395 tx_ring = adapter->tx_ring[n];
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "T [desc] [address 63:0 ] "
400 "[PlPOCIStDDM Ln] [bi->dma ] "
401 "leng ntw timestamp bi->skb\n");
402
403 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
404 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
405 buffer_info = &tx_ring->buffer_info[i];
406 u0 = (struct my_u0 *)tx_desc;
407 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
408 " %04X %3X %016llX %p", i,
409 le64_to_cpu(u0->a),
410 le64_to_cpu(u0->b),
411 (u64)buffer_info->dma,
412 buffer_info->length,
413 buffer_info->next_to_watch,
414 (u64)buffer_info->time_stamp,
415 buffer_info->skb);
416 if (i == tx_ring->next_to_use &&
417 i == tx_ring->next_to_clean)
418 printk(KERN_CONT " NTC/U\n");
419 else if (i == tx_ring->next_to_use)
420 printk(KERN_CONT " NTU\n");
421 else if (i == tx_ring->next_to_clean)
422 printk(KERN_CONT " NTC\n");
423 else
424 printk(KERN_CONT "\n");
425
426 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
427 print_hex_dump(KERN_INFO, "",
428 DUMP_PREFIX_ADDRESS,
429 16, 1, phys_to_virt(buffer_info->dma),
430 buffer_info->length, true);
431 }
432 }
433
434 /* Print RX Rings Summary */
435rx_ring_summary:
436 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
437 printk(KERN_INFO "Queue [NTU] [NTC]\n");
438 for (n = 0; n < adapter->num_rx_queues; n++) {
439 rx_ring = adapter->rx_ring[n];
440 printk(KERN_INFO " %5d %5X %5X\n", n,
441 rx_ring->next_to_use, rx_ring->next_to_clean);
442 }
443
444 /* Print RX Rings */
445 if (!netif_msg_rx_status(adapter))
446 goto exit;
447
448 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
449
450 /* Advanced Receive Descriptor (Read) Format
451 * 63 1 0
452 * +-----------------------------------------------------+
453 * 0 | Packet Buffer Address [63:1] |A0/NSE|
454 * +----------------------------------------------+------+
455 * 8 | Header Buffer Address [63:1] | DD |
456 * +-----------------------------------------------------+
457 *
458 *
459 * Advanced Receive Descriptor (Write-Back) Format
460 *
461 * 63 48 47 32 31 30 21 20 17 16 4 3 0
462 * +------------------------------------------------------+
463 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
464 * | Checksum Ident | | | | Type | Type |
465 * +------------------------------------------------------+
466 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
467 * +------------------------------------------------------+
468 * 63 48 47 32 31 20 19 0
469 */
470
471 for (n = 0; n < adapter->num_rx_queues; n++) {
472 rx_ring = adapter->rx_ring[n];
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "R [desc] [ PktBuf A0] "
477 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
478 "<-- Adv Rx Read format\n");
479 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
480 "[vl er S cks ln] ---------------- [bi->skb] "
481 "<-- Adv Rx Write-Back format\n");
482
483 for (i = 0; i < rx_ring->count; i++) {
484 buffer_info = &rx_ring->buffer_info[i];
485 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
486 u0 = (struct my_u0 *)rx_desc;
487 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
488 if (staterr & E1000_RXD_STAT_DD) {
489 /* Descriptor Done */
490 printk(KERN_INFO "RWB[0x%03X] %016llX "
491 "%016llX ---------------- %p", i,
492 le64_to_cpu(u0->a),
493 le64_to_cpu(u0->b),
494 buffer_info->skb);
495 } else {
496 printk(KERN_INFO "R [0x%03X] %016llX "
497 "%016llX %016llX %p", i,
498 le64_to_cpu(u0->a),
499 le64_to_cpu(u0->b),
500 (u64)buffer_info->dma,
501 buffer_info->skb);
502
503 if (netif_msg_pktdata(adapter)) {
504 print_hex_dump(KERN_INFO, "",
505 DUMP_PREFIX_ADDRESS,
506 16, 1,
507 phys_to_virt(buffer_info->dma),
508 rx_ring->rx_buffer_len, true);
509 if (rx_ring->rx_buffer_len
510 < IGB_RXBUFFER_1024)
511 print_hex_dump(KERN_INFO, "",
512 DUMP_PREFIX_ADDRESS,
513 16, 1,
514 phys_to_virt(
515 buffer_info->page_dma +
516 buffer_info->page_offset),
517 PAGE_SIZE/2, true);
518 }
519 }
520
521 if (i == rx_ring->next_to_use)
522 printk(KERN_CONT " NTU\n");
523 else if (i == rx_ring->next_to_clean)
524 printk(KERN_CONT " NTC\n");
525 else
526 printk(KERN_CONT "\n");
527
528 }
529 }
530
531exit:
532 return;
533}
534
535
38c845c7
PO		 536/**
537 * igb_read_clock - read raw cycle counter (to be used by time counter)
538 */
539static cycle_t igb_read_clock(const struct cyclecounter *tc)
540{
541 struct igb_adapter *adapter =
542 container_of(tc, struct igb_adapter, cycles);
543 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 544 u64 stamp = 0;
545 int shift = 0;
38c845c7 546
55cac248
AD		 547 /*
548 * The timestamp latches on lowest register read. For the 82580
549 * the lowest register is SYSTIMR instead of SYSTIML. However we never
550 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
551 */
552 if (hw->mac.type == e1000_82580) {
553 stamp = rd32(E1000_SYSTIMR) >> 8;
554 shift = IGB_82580_TSYNC_SHIFT;
555 }
556
c5b9bd5e
AD		 557 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
558 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 559 return stamp;
560}
561
9d5c8243 562/**
c041076a 563 * igb_get_hw_dev - return device
9d5c8243
AK		 564 * used by hardware layer to print debugging information
565 **/
c041076a 566struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 567{
568 struct igb_adapter *adapter = hw->back;
c041076a 569 return adapter->netdev;
9d5c8243 570}
38c845c7 571
9d5c8243
AK		 572/**
573 * igb_init_module - Driver Registration Routine
574 *
575 * igb_init_module is the first routine called when the driver is
576 * loaded. All it does is register with the PCI subsystem.
577 **/
578static int __init igb_init_module(void)
579{
580 int ret;
581 printk(KERN_INFO "%s - version %s\n",
582 igb_driver_string, igb_driver_version);
583
584 printk(KERN_INFO "%s\n", igb_copyright);
585
421e02f0 586#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 587 dca_register_notify(&dca_notifier);
588#endif
bbd98fe4 589 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 590 return ret;
591}
592
593module_init(igb_init_module);
594
595/**
596 * igb_exit_module - Driver Exit Cleanup Routine
597 *
598 * igb_exit_module is called just before the driver is removed
599 * from memory.
600 **/
601static void __exit igb_exit_module(void)
602{
421e02f0 603#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 604 dca_unregister_notify(&dca_notifier);
605#endif
9d5c8243
AK		 606 pci_unregister_driver(&igb_driver);
607}
608
609module_exit(igb_exit_module);
610
26bc19ec
AD		 611#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
612/**
613 * igb_cache_ring_register - Descriptor ring to register mapping
614 * @adapter: board private structure to initialize
615 *
616 * Once we know the feature-set enabled for the device, we'll cache
617 * the register offset the descriptor ring is assigned to.
618 **/
619static void igb_cache_ring_register(struct igb_adapter *adapter)
620{
ee1b9f06 621 int i = 0, j = 0;
047e0030 622 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 623
624 switch (adapter->hw.mac.type) {
625 case e1000_82576:
626 /* The queues are allocated for virtualization such that VF 0
627 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
628 * In order to avoid collision we start at the first free queue
629 * and continue consuming queues in the same sequence
630 */
ee1b9f06 631 if (adapter->vfs_allocated_count) {
a99955fc 632 for (; i < adapter->rss_queues; i++)
3025a446
AD		 633 adapter->rx_ring[i]->reg_idx = rbase_offset +
634 Q_IDX_82576(i);
ee1b9f06 635 }
26bc19ec 636 case e1000_82575:
55cac248 637 case e1000_82580:
d2ba2ed8 638 case e1000_i350:
26bc19ec 639 default:
ee1b9f06 640 for (; i < adapter->num_rx_queues; i++)
3025a446 641 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 642 for (; j < adapter->num_tx_queues; j++)
3025a446 643 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 644 break;
645 }
646}
647
047e0030
AD		 648static void igb_free_queues(struct igb_adapter *adapter)
649{
3025a446 650 int i;
047e0030 651
3025a446
AD		 652 for (i = 0; i < adapter->num_tx_queues; i++) {
653 kfree(adapter->tx_ring[i]);
654 adapter->tx_ring[i] = NULL;
655 }
656 for (i = 0; i < adapter->num_rx_queues; i++) {
657 kfree(adapter->rx_ring[i]);
658 adapter->rx_ring[i] = NULL;
659 }
047e0030
AD		 660 adapter->num_rx_queues = 0;
661 adapter->num_tx_queues = 0;
662}
663
9d5c8243
AK		 664/**
665 * igb_alloc_queues - Allocate memory for all rings
666 * @adapter: board private structure to initialize
667 *
668 * We allocate one ring per queue at run-time since we don't know the
669 * number of queues at compile-time.
670 **/
671static int igb_alloc_queues(struct igb_adapter *adapter)
672{
3025a446 673 struct igb_ring *ring;
9d5c8243
AK		 674 int i;
675
661086df 676 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 677 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 if (!ring)
679 goto err;
68fd9910 680 ring->count = adapter->tx_ring_count;
661086df 681 ring->queue_index = i;
59d71989 682 ring->dev = &adapter->pdev->dev;
e694e964 683 ring->netdev = adapter->netdev;
85ad76b2
AD		 684 /* For 82575, context index must be unique per ring. */
685 if (adapter->hw.mac.type == e1000_82575)
686 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 687 adapter->tx_ring[i] = ring;
661086df 688 }
85ad76b2 689
9d5c8243 690 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 691 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 if (!ring)
693 goto err;
68fd9910 694 ring->count = adapter->rx_ring_count;
844290e5 695 ring->queue_index = i;
59d71989 696 ring->dev = &adapter->pdev->dev;
e694e964 697 ring->netdev = adapter->netdev;
4c844851 698 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 699 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
700 /* set flag indicating ring supports SCTP checksum offload */
701 if (adapter->hw.mac.type >= e1000_82576)
702 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 703 adapter->rx_ring[i] = ring;
9d5c8243 704 }
26bc19ec
AD		 705
706 igb_cache_ring_register(adapter);
9d5c8243 707
047e0030 708 return 0;
a88f10ec 709
047e0030
AD		 710err:
711 igb_free_queues(adapter);
d1a8c9e1 712
047e0030 713 return -ENOMEM;
a88f10ec
AD		 714}
715
9d5c8243 716#define IGB_N0_QUEUE -1
047e0030 717static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 718{
719 u32 msixbm = 0;
047e0030 720 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 721 struct e1000_hw *hw = &adapter->hw;
2d064c06 722 u32 ivar, index;
047e0030
AD		 723 int rx_queue = IGB_N0_QUEUE;
724 int tx_queue = IGB_N0_QUEUE;
725
726 if (q_vector->rx_ring)
727 rx_queue = q_vector->rx_ring->reg_idx;
728 if (q_vector->tx_ring)
729 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 730
731 switch (hw->mac.type) {
732 case e1000_82575:
9d5c8243
AK		 733 /* The 82575 assigns vectors using a bitmask, which matches the
734 bitmask for the EICR/EIMS/EIMC registers. To assign one
735 or more queues to a vector, we write the appropriate bits
736 into the MSIXBM register for that vector. */
047e0030 737 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 739 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 740 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 741 if (!adapter->msix_entries && msix_vector == 0)
742 msixbm |= E1000_EIMS_OTHER;
9d5c8243 743 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 744 q_vector->eims_value = msixbm;
2d064c06
AD		 745 break;
746 case e1000_82576:
26bc19ec 747 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 748 Each queue has a single entry in the table to which we write
749 a vector number along with a "valid" bit. Sadly, the layout
750 of the table is somewhat counterintuitive. */
751 if (rx_queue > IGB_N0_QUEUE) {
047e0030 752 index = (rx_queue & 0x7);
2d064c06 753 ivar = array_rd32(E1000_IVAR0, index);
047e0030 754 if (rx_queue < 8) {
26bc19ec
AD		 755 /* vector goes into low byte of register */
756 ivar = ivar & 0xFFFFFF00;
757 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 758 } else {
759 /* vector goes into third byte of register */
760 ivar = ivar & 0xFF00FFFF;
761 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 762 }
2d064c06
AD		 763 array_wr32(E1000_IVAR0, index, ivar);
764 }
765 if (tx_queue > IGB_N0_QUEUE) {
047e0030 766 index = (tx_queue & 0x7);
2d064c06 767 ivar = array_rd32(E1000_IVAR0, index);
047e0030 768 if (tx_queue < 8) {
26bc19ec
AD		 769 /* vector goes into second byte of register */
770 ivar = ivar & 0xFFFF00FF;
771 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 772 } else {
773 /* vector goes into high byte of register */
774 ivar = ivar & 0x00FFFFFF;
775 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 776 }
2d064c06
AD		 777 array_wr32(E1000_IVAR0, index, ivar);
778 }
047e0030 779 q_vector->eims_value = 1 << msix_vector;
2d064c06 780 break;
55cac248 781 case e1000_82580:
d2ba2ed8 782 case e1000_i350:
55cac248
AD		 783 /* 82580 uses the same table-based approach as 82576 but has fewer
784 entries as a result we carry over for queues greater than 4. */
785 if (rx_queue > IGB_N0_QUEUE) {
786 index = (rx_queue >> 1);
787 ivar = array_rd32(E1000_IVAR0, index);
788 if (rx_queue & 0x1) {
789 /* vector goes into third byte of register */
790 ivar = ivar & 0xFF00FFFF;
791 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
792 } else {
793 /* vector goes into low byte of register */
794 ivar = ivar & 0xFFFFFF00;
795 ivar |= msix_vector | E1000_IVAR_VALID;
796 }
797 array_wr32(E1000_IVAR0, index, ivar);
798 }
799 if (tx_queue > IGB_N0_QUEUE) {
800 index = (tx_queue >> 1);
801 ivar = array_rd32(E1000_IVAR0, index);
802 if (tx_queue & 0x1) {
803 /* vector goes into high byte of register */
804 ivar = ivar & 0x00FFFFFF;
805 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
806 } else {
807 /* vector goes into second byte of register */
808 ivar = ivar & 0xFFFF00FF;
809 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
810 }
811 array_wr32(E1000_IVAR0, index, ivar);
812 }
813 q_vector->eims_value = 1 << msix_vector;
814 break;
2d064c06
AD		 815 default:
816 BUG();
817 break;
818 }
26b39276
AD		 819
820 /* add q_vector eims value to global eims_enable_mask */
821 adapter->eims_enable_mask |= q_vector->eims_value;
822
823 /* configure q_vector to set itr on first interrupt */
824 q_vector->set_itr = 1;
9d5c8243
AK		 825}
826
827/**
828 * igb_configure_msix - Configure MSI-X hardware
829 *
830 * igb_configure_msix sets up the hardware to properly
831 * generate MSI-X interrupts.
832 **/
833static void igb_configure_msix(struct igb_adapter *adapter)
834{
835 u32 tmp;
836 int i, vector = 0;
837 struct e1000_hw *hw = &adapter->hw;
838
839 adapter->eims_enable_mask = 0;
9d5c8243
AK		 840
841 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 842 switch (hw->mac.type) {
843 case e1000_82575:
9d5c8243
AK		 844 tmp = rd32(E1000_CTRL_EXT);
845 /* enable MSI-X PBA support*/
846 tmp |= E1000_CTRL_EXT_PBA_CLR;
847
848 /* Auto-Mask interrupts upon ICR read. */
849 tmp |= E1000_CTRL_EXT_EIAME;
850 tmp |= E1000_CTRL_EXT_IRCA;
851
852 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 853
854 /* enable msix_other interrupt */
855 array_wr32(E1000_MSIXBM(0), vector++,
856 E1000_EIMS_OTHER);
844290e5 857 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 858
2d064c06
AD		 859 break;
860
861 case e1000_82576:
55cac248 862 case e1000_82580:
d2ba2ed8 863 case e1000_i350:
047e0030
AD		 864 /* Turn on MSI-X capability first, or our settings
865 * won't stick. And it will take days to debug. */
866 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
867 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 E1000_GPIE_NSICR);
869
870 /* enable msix_other interrupt */
871 adapter->eims_other = 1 << vector;
2d064c06 872 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 873
047e0030 874 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 875 break;
876 default:
877 /* do nothing, since nothing else supports MSI-X */
878 break;
879 } /* switch (hw->mac.type) */
047e0030
AD		 880
881 adapter->eims_enable_mask |= adapter->eims_other;
882
26b39276
AD		 883 for (i = 0; i < adapter->num_q_vectors; i++)
884 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 885
9d5c8243
AK		 886 wrfl();
887}
888
889/**
890 * igb_request_msix - Initialize MSI-X interrupts
891 *
892 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 * kernel.
894 **/
895static int igb_request_msix(struct igb_adapter *adapter)
896{
897 struct net_device *netdev = adapter->netdev;
047e0030 898 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 899 int i, err = 0, vector = 0;
900
047e0030 901 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 902 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 903 if (err)
904 goto out;
905 vector++;
906
907 for (i = 0; i < adapter->num_q_vectors; i++) {
908 struct igb_q_vector *q_vector = adapter->q_vector[i];
909
910 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
911
912 if (q_vector->rx_ring && q_vector->tx_ring)
913 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
914 q_vector->rx_ring->queue_index);
915 else if (q_vector->tx_ring)
916 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
917 q_vector->tx_ring->queue_index);
918 else if (q_vector->rx_ring)
919 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
920 q_vector->rx_ring->queue_index);
9d5c8243 921 else
047e0030
AD		 922 sprintf(q_vector->name, "%s-unused", netdev->name);
923
9d5c8243 924 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 925 igb_msix_ring, 0, q_vector->name,
047e0030 926 q_vector);
9d5c8243
AK		 927 if (err)
928 goto out;
9d5c8243
AK		 929 vector++;
930 }
931
9d5c8243
AK		 932 igb_configure_msix(adapter);
933 return 0;
934out:
935 return err;
936}
937
938static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
939{
940 if (adapter->msix_entries) {
941 pci_disable_msix(adapter->pdev);
942 kfree(adapter->msix_entries);
943 adapter->msix_entries = NULL;
047e0030 944 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 945 pci_disable_msi(adapter->pdev);
047e0030 946 }
9d5c8243
AK		 947}
948
047e0030
AD		 949/**
950 * igb_free_q_vectors - Free memory allocated for interrupt vectors
951 * @adapter: board private structure to initialize
952 *
953 * This function frees the memory allocated to the q_vectors. In addition if
954 * NAPI is enabled it will delete any references to the NAPI struct prior
955 * to freeing the q_vector.
956 **/
957static void igb_free_q_vectors(struct igb_adapter *adapter)
958{
959 int v_idx;
960
961 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
962 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
963 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 964 if (!q_vector)
965 continue;
047e0030
AD		 966 netif_napi_del(&q_vector->napi);
967 kfree(q_vector);
968 }
969 adapter->num_q_vectors = 0;
970}
971
972/**
973 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
974 *
975 * This function resets the device so that it has 0 rx queues, tx queues, and
976 * MSI-X interrupts allocated.
977 */
978static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
979{
980 igb_free_queues(adapter);
981 igb_free_q_vectors(adapter);
982 igb_reset_interrupt_capability(adapter);
983}
9d5c8243
AK		 984
985/**
986 * igb_set_interrupt_capability - set MSI or MSI-X if supported
987 *
988 * Attempt to configure interrupts using the best available
989 * capabilities of the hardware and kernel.
990 **/
21adef3e 991static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 992{
993 int err;
994 int numvecs, i;
995
83b7180d 996 /* Number of supported queues. */
a99955fc 997 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 998 if (adapter->vfs_allocated_count)
999 adapter->num_tx_queues = 1;
1000 else
1001 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1002
047e0030
AD		 1003 /* start with one vector for every rx queue */
1004 numvecs = adapter->num_rx_queues;
1005
3ad2f3fb 1006 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1007 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1008 numvecs += adapter->num_tx_queues;
047e0030
AD		 1009
1010 /* store the number of vectors reserved for queues */
1011 adapter->num_q_vectors = numvecs;
1012
1013 /* add 1 vector for link status interrupts */
1014 numvecs++;
9d5c8243
AK		 1015 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1016 GFP_KERNEL);
1017 if (!adapter->msix_entries)
1018 goto msi_only;
1019
1020 for (i = 0; i < numvecs; i++)
1021 adapter->msix_entries[i].entry = i;
1022
1023 err = pci_enable_msix(adapter->pdev,
1024 adapter->msix_entries,
1025 numvecs);
1026 if (err == 0)
34a20e89 1027 goto out;
9d5c8243
AK		 1028
1029 igb_reset_interrupt_capability(adapter);
1030
1031 /* If we can't do MSI-X, try MSI */
1032msi_only:
2a3abf6d
AD		 1033#ifdef CONFIG_PCI_IOV
1034 /* disable SR-IOV for non MSI-X configurations */
1035 if (adapter->vf_data) {
1036 struct e1000_hw *hw = &adapter->hw;
1037 /* disable iov and allow time for transactions to clear */
1038 pci_disable_sriov(adapter->pdev);
1039 msleep(500);
1040
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1044 msleep(100);
1045 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 }
1047#endif
4fc82adf 1048 adapter->vfs_allocated_count = 0;
a99955fc 1049 adapter->rss_queues = 1;
4fc82adf 1050 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1051 adapter->num_rx_queues = 1;
661086df 1052 adapter->num_tx_queues = 1;
047e0030 1053 adapter->num_q_vectors = 1;
9d5c8243 1054 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1055 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1056out:
21adef3e
BH		 1057 /* Notify the stack of the (possibly) reduced queue counts. */
1058 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1059 return netif_set_real_num_rx_queues(adapter->netdev,
1060 adapter->num_rx_queues);
9d5c8243
AK		 1061}
1062
047e0030
AD		 1063/**
1064 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1065 * @adapter: board private structure to initialize
1066 *
1067 * We allocate one q_vector per queue interrupt. If allocation fails we
1068 * return -ENOMEM.
1069 **/
1070static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1071{
1072 struct igb_q_vector *q_vector;
1073 struct e1000_hw *hw = &adapter->hw;
1074 int v_idx;
1075
1076 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1077 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1078 if (!q_vector)
1079 goto err_out;
1080 q_vector->adapter = adapter;
047e0030
AD		 1081 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1082 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1083 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1084 adapter->q_vector[v_idx] = q_vector;
1085 }
1086 return 0;
1087
1088err_out:
fe0592b4 1089 igb_free_q_vectors(adapter);
047e0030
AD		 1090 return -ENOMEM;
1091}
1092
1093static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1094 int ring_idx, int v_idx)
1095{
3025a446 1096 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1097
3025a446 1098 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1099 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1100 q_vector->itr_val = adapter->rx_itr_setting;
1101 if (q_vector->itr_val && q_vector->itr_val <= 3)
1102 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1103}
1104
1105static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1111 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->tx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117/**
1118 * igb_map_ring_to_vector - maps allocated queues to vectors
1119 *
1120 * This function maps the recently allocated queues to vectors.
1121 **/
1122static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123{
1124 int i;
1125 int v_idx = 0;
1126
1127 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1128 (adapter->num_q_vectors < adapter->num_tx_queues))
1129 return -ENOMEM;
1130
1131 if (adapter->num_q_vectors >=
1132 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1133 for (i = 0; i < adapter->num_rx_queues; i++)
1134 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1135 for (i = 0; i < adapter->num_tx_queues; i++)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1137 } else {
1138 for (i = 0; i < adapter->num_rx_queues; i++) {
1139 if (i < adapter->num_tx_queues)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1141 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1142 }
1143 for (; i < adapter->num_tx_queues; i++)
1144 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1145 }
1146 return 0;
1147}
1148
1149/**
1150 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1151 *
1152 * This function initializes the interrupts and allocates all of the queues.
1153 **/
1154static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1155{
1156 struct pci_dev *pdev = adapter->pdev;
1157 int err;
1158
21adef3e
BH		 1159 err = igb_set_interrupt_capability(adapter);
1160 if (err)
1161 return err;
047e0030
AD		 1162
1163 err = igb_alloc_q_vectors(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1166 goto err_alloc_q_vectors;
1167 }
1168
1169 err = igb_alloc_queues(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1172 goto err_alloc_queues;
1173 }
1174
1175 err = igb_map_ring_to_vector(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1178 goto err_map_queues;
1179 }
1180
1181
1182 return 0;
1183err_map_queues:
1184 igb_free_queues(adapter);
1185err_alloc_queues:
1186 igb_free_q_vectors(adapter);
1187err_alloc_q_vectors:
1188 igb_reset_interrupt_capability(adapter);
1189 return err;
1190}
1191
9d5c8243
AK		 1192/**
1193 * igb_request_irq - initialize interrupts
1194 *
1195 * Attempts to configure interrupts using the best available
1196 * capabilities of the hardware and kernel.
1197 **/
1198static int igb_request_irq(struct igb_adapter *adapter)
1199{
1200 struct net_device *netdev = adapter->netdev;
047e0030 1201 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1202 int err = 0;
1203
1204 if (adapter->msix_entries) {
1205 err = igb_request_msix(adapter);
844290e5 1206 if (!err)
9d5c8243 1207 goto request_done;
9d5c8243 1208 /* fall back to MSI */
047e0030 1209 igb_clear_interrupt_scheme(adapter);
9d5c8243 1210 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1211 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1212 igb_free_all_tx_resources(adapter);
1213 igb_free_all_rx_resources(adapter);
047e0030 1214 adapter->num_tx_queues = 1;
9d5c8243 1215 adapter->num_rx_queues = 1;
047e0030
AD		 1216 adapter->num_q_vectors = 1;
1217 err = igb_alloc_q_vectors(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for vectors\n");
1221 goto request_done;
1222 }
1223 err = igb_alloc_queues(adapter);
1224 if (err) {
1225 dev_err(&pdev->dev,
1226 "Unable to allocate memory for queues\n");
1227 igb_free_q_vectors(adapter);
1228 goto request_done;
1229 }
1230 igb_setup_all_tx_resources(adapter);
1231 igb_setup_all_rx_resources(adapter);
844290e5 1232 } else {
feeb2721 1233 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1234 }
844290e5 1235
7dfc16fa 1236 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1237 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1238 netdev->name, adapter);
9d5c8243
AK		 1239 if (!err)
1240 goto request_done;
047e0030 1241
9d5c8243
AK		 1242 /* fall back to legacy interrupts */
1243 igb_reset_interrupt_capability(adapter);
7dfc16fa 1244 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1245 }
1246
a0607fd3 1247 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1248 netdev->name, adapter);
9d5c8243 1249
6cb5e577 1250 if (err)
9d5c8243
AK		 1251 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 err);
9d5c8243
AK		 1253
1254request_done:
1255 return err;
1256}
1257
1258static void igb_free_irq(struct igb_adapter *adapter)
1259{
9d5c8243
AK		 1260 if (adapter->msix_entries) {
1261 int vector = 0, i;
1262
047e0030 1263 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1264
047e0030
AD		 1265 for (i = 0; i < adapter->num_q_vectors; i++) {
1266 struct igb_q_vector *q_vector = adapter->q_vector[i];
1267 free_irq(adapter->msix_entries[vector++].vector,
1268 q_vector);
1269 }
1270 } else {
1271 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1272 }
9d5c8243
AK		 1273}
1274
1275/**
1276 * igb_irq_disable - Mask off interrupt generation on the NIC
1277 * @adapter: board private structure
1278 **/
1279static void igb_irq_disable(struct igb_adapter *adapter)
1280{
1281 struct e1000_hw *hw = &adapter->hw;
1282
25568a53
AD		 1283 /*
1284 * we need to be careful when disabling interrupts. The VFs are also
1285 * mapped into these registers and so clearing the bits can cause
1286 * issues on the VF drivers so we only need to clear what we set
1287 */
9d5c8243 1288 if (adapter->msix_entries) {
2dfd1212
AD		 1289 u32 regval = rd32(E1000_EIAM);
1290 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_EIMC, adapter->eims_enable_mask);
1292 regval = rd32(E1000_EIAC);
1293 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1294 }
844290e5
PW		 1295
1296 wr32(E1000_IAM, 0);
9d5c8243
AK		 1297 wr32(E1000_IMC, ~0);
1298 wrfl();
81a61859
ET		 1299 if (adapter->msix_entries) {
1300 int i;
1301 for (i = 0; i < adapter->num_q_vectors; i++)
1302 synchronize_irq(adapter->msix_entries[i].vector);
1303 } else {
1304 synchronize_irq(adapter->pdev->irq);
1305 }
9d5c8243
AK		 1306}
1307
1308/**
1309 * igb_irq_enable - Enable default interrupt generation settings
1310 * @adapter: board private structure
1311 **/
1312static void igb_irq_enable(struct igb_adapter *adapter)
1313{
1314 struct e1000_hw *hw = &adapter->hw;
1315
1316 if (adapter->msix_entries) {
25568a53 1317 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1318 u32 regval = rd32(E1000_EIAC);
1319 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1320 regval = rd32(E1000_EIAM);
1321 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1322 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1323 if (adapter->vfs_allocated_count) {
4ae196df 1324 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1325 ims |= E1000_IMS_VMMB;
1326 }
55cac248
AD		 1327 if (adapter->hw.mac.type == e1000_82580)
1328 ims |= E1000_IMS_DRSTA;
1329
25568a53 1330 wr32(E1000_IMS, ims);
844290e5 1331 } else {
55cac248
AD		 1332 wr32(E1000_IMS, IMS_ENABLE_MASK |
1333 E1000_IMS_DRSTA);
1334 wr32(E1000_IAM, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
844290e5 1336 }
9d5c8243
AK		 1337}
1338
1339static void igb_update_mng_vlan(struct igb_adapter *adapter)
1340{
51466239 1341 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1342 u16 vid = adapter->hw.mng_cookie.vlan_id;
1343 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1344
1345 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1346 /* add VID to filter table */
1347 igb_vfta_set(hw, vid, true);
1348 adapter->mng_vlan_id = vid;
1349 } else {
1350 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1351 }
1352
1353 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1354 (vid != old_vid) &&
1355 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1356 /* remove VID from filter table */
1357 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1358 }
1359}
1360
1361/**
1362 * igb_release_hw_control - release control of the h/w to f/w
1363 * @adapter: address of board private structure
1364 *
1365 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1366 * For ASF and Pass Through versions of f/w this means that the
1367 * driver is no longer loaded.
1368 *
1369 **/
1370static void igb_release_hw_control(struct igb_adapter *adapter)
1371{
1372 struct e1000_hw *hw = &adapter->hw;
1373 u32 ctrl_ext;
1374
1375 /* Let firmware take over control of h/w */
1376 ctrl_ext = rd32(E1000_CTRL_EXT);
1377 wr32(E1000_CTRL_EXT,
1378 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1379}
1380
9d5c8243
AK		 1381/**
1382 * igb_get_hw_control - get control of the h/w from f/w
1383 * @adapter: address of board private structure
1384 *
1385 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1386 * For ASF and Pass Through versions of f/w this means that
1387 * the driver is loaded.
1388 *
1389 **/
1390static void igb_get_hw_control(struct igb_adapter *adapter)
1391{
1392 struct e1000_hw *hw = &adapter->hw;
1393 u32 ctrl_ext;
1394
1395 /* Let firmware know the driver has taken over */
1396 ctrl_ext = rd32(E1000_CTRL_EXT);
1397 wr32(E1000_CTRL_EXT,
1398 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1399}
1400
9d5c8243
AK		 1401/**
1402 * igb_configure - configure the hardware for RX and TX
1403 * @adapter: private board structure
1404 **/
1405static void igb_configure(struct igb_adapter *adapter)
1406{
1407 struct net_device *netdev = adapter->netdev;
1408 int i;
1409
1410 igb_get_hw_control(adapter);
ff41f8dc 1411 igb_set_rx_mode(netdev);
9d5c8243
AK		 1412
1413 igb_restore_vlan(adapter);
9d5c8243 1414
85b430b4 1415 igb_setup_tctl(adapter);
06cf2666 1416 igb_setup_mrqc(adapter);
9d5c8243 1417 igb_setup_rctl(adapter);
85b430b4
AD		 1418
1419 igb_configure_tx(adapter);
9d5c8243 1420 igb_configure_rx(adapter);
662d7205
AD		 1421
1422 igb_rx_fifo_flush_82575(&adapter->hw);
1423
c493ea45 1424 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1425 * at least 1 descriptor unused to make sure
1426 * next_to_use != next_to_clean */
1427 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1428 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1429 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1430 }
9d5c8243
AK		 1431}
1432
88a268c1
NN		 1433/**
1434 * igb_power_up_link - Power up the phy/serdes link
1435 * @adapter: address of board private structure
1436 **/
1437void igb_power_up_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_up_phy_copper(&adapter->hw);
1441 else
1442 igb_power_up_serdes_link_82575(&adapter->hw);
1443}
1444
1445/**
1446 * igb_power_down_link - Power down the phy/serdes link
1447 * @adapter: address of board private structure
1448 */
1449static void igb_power_down_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_down_phy_copper_82575(&adapter->hw);
1453 else
1454 igb_shutdown_serdes_link_82575(&adapter->hw);
1455}
9d5c8243
AK		 1456
1457/**
1458 * igb_up - Open the interface and prepare it to handle traffic
1459 * @adapter: board private structure
1460 **/
9d5c8243
AK		 1461int igb_up(struct igb_adapter *adapter)
1462{
1463 struct e1000_hw *hw = &adapter->hw;
1464 int i;
1465
1466 /* hardware has been reset, we need to reload some things */
1467 igb_configure(adapter);
1468
1469 clear_bit(__IGB_DOWN, &adapter->state);
1470
047e0030
AD		 1471 for (i = 0; i < adapter->num_q_vectors; i++) {
1472 struct igb_q_vector *q_vector = adapter->q_vector[i];
1473 napi_enable(&q_vector->napi);
1474 }
844290e5 1475 if (adapter->msix_entries)
9d5c8243 1476 igb_configure_msix(adapter);
feeb2721
AD		 1477 else
1478 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1479
1480 /* Clear any pending interrupts. */
1481 rd32(E1000_ICR);
1482 igb_irq_enable(adapter);
1483
d4960307
AD		 1484 /* notify VFs that reset has been completed */
1485 if (adapter->vfs_allocated_count) {
1486 u32 reg_data = rd32(E1000_CTRL_EXT);
1487 reg_data |= E1000_CTRL_EXT_PFRSTD;
1488 wr32(E1000_CTRL_EXT, reg_data);
1489 }
1490
4cb9be7a
JB		 1491 netif_tx_start_all_queues(adapter->netdev);
1492
25568a53
AD		 1493 /* start the watchdog. */
1494 hw->mac.get_link_status = 1;
1495 schedule_work(&adapter->watchdog_task);
1496
9d5c8243
AK		 1497 return 0;
1498}
1499
1500void igb_down(struct igb_adapter *adapter)
1501{
9d5c8243 1502 struct net_device *netdev = adapter->netdev;
330a6d6a 1503 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1504 u32 tctl, rctl;
1505 int i;
1506
1507 /* signal that we're down so the interrupt handler does not
1508 * reschedule our watchdog timer */
1509 set_bit(__IGB_DOWN, &adapter->state);
1510
1511 /* disable receives in the hardware */
1512 rctl = rd32(E1000_RCTL);
1513 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1514 /* flush and sleep below */
1515
fd2ea0a7 1516 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1517
1518 /* disable transmits in the hardware */
1519 tctl = rd32(E1000_TCTL);
1520 tctl &= ~E1000_TCTL_EN;
1521 wr32(E1000_TCTL, tctl);
1522 /* flush both disables and wait for them to finish */
1523 wrfl();
1524 msleep(10);
1525
047e0030
AD		 1526 for (i = 0; i < adapter->num_q_vectors; i++) {
1527 struct igb_q_vector *q_vector = adapter->q_vector[i];
1528 napi_disable(&q_vector->napi);
1529 }
9d5c8243 1530
9d5c8243
AK		 1531 igb_irq_disable(adapter);
1532
1533 del_timer_sync(&adapter->watchdog_timer);
1534 del_timer_sync(&adapter->phy_info_timer);
1535
9d5c8243 1536 netif_carrier_off(netdev);
04fe6358
AD		 1537
1538 /* record the stats before reset*/
12dcd86b
ED		 1539 spin_lock(&adapter->stats64_lock);
1540 igb_update_stats(adapter, &adapter->stats64);
1541 spin_unlock(&adapter->stats64_lock);
04fe6358 1542
9d5c8243
AK		 1543 adapter->link_speed = 0;
1544 adapter->link_duplex = 0;
1545
3023682e
JK		 1546 if (!pci_channel_offline(adapter->pdev))
1547 igb_reset(adapter);
9d5c8243
AK		 1548 igb_clean_all_tx_rings(adapter);
1549 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1550#ifdef CONFIG_IGB_DCA
1551
1552 /* since we reset the hardware DCA settings were cleared */
1553 igb_setup_dca(adapter);
1554#endif
9d5c8243
AK		 1555}
1556
1557void igb_reinit_locked(struct igb_adapter *adapter)
1558{
1559 WARN_ON(in_interrupt());
1560 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1561 msleep(1);
1562 igb_down(adapter);
1563 igb_up(adapter);
1564 clear_bit(__IGB_RESETTING, &adapter->state);
1565}
1566
1567void igb_reset(struct igb_adapter *adapter)
1568{
090b1795 1569 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1570 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1571 struct e1000_mac_info *mac = &hw->mac;
1572 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1573 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1574 u16 hwm;
1575
1576 /* Repartition Pba for greater than 9k mtu
1577 * To take effect CTRL.RST is required.
1578 */
fa4dfae0 1579 switch (mac->type) {
d2ba2ed8 1580 case e1000_i350:
55cac248
AD		 1581 case e1000_82580:
1582 pba = rd32(E1000_RXPBS);
1583 pba = igb_rxpbs_adjust_82580(pba);
1584 break;
fa4dfae0 1585 case e1000_82576:
d249be54
AD		 1586 pba = rd32(E1000_RXPBS);
1587 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1588 break;
1589 case e1000_82575:
1590 default:
1591 pba = E1000_PBA_34K;
1592 break;
2d064c06 1593 }
9d5c8243 1594
2d064c06
AD		 1595 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1596 (mac->type < e1000_82576)) {
9d5c8243
AK		 1597 /* adjust PBA for jumbo frames */
1598 wr32(E1000_PBA, pba);
1599
1600 /* To maintain wire speed transmits, the Tx FIFO should be
1601 * large enough to accommodate two full transmit packets,
1602 * rounded up to the next 1KB and expressed in KB. Likewise,
1603 * the Rx FIFO should be large enough to accommodate at least
1604 * one full receive packet and is similarly rounded up and
1605 * expressed in KB. */
1606 pba = rd32(E1000_PBA);
1607 /* upper 16 bits has Tx packet buffer allocation size in KB */
1608 tx_space = pba >> 16;
1609 /* lower 16 bits has Rx packet buffer allocation size in KB */
1610 pba &= 0xffff;
1611 /* the tx fifo also stores 16 bytes of information about the tx
1612 * but don't include ethernet FCS because hardware appends it */
1613 min_tx_space = (adapter->max_frame_size +
85e8d004 1614 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1615 ETH_FCS_LEN) * 2;
1616 min_tx_space = ALIGN(min_tx_space, 1024);
1617 min_tx_space >>= 10;
1618 /* software strips receive CRC, so leave room for it */
1619 min_rx_space = adapter->max_frame_size;
1620 min_rx_space = ALIGN(min_rx_space, 1024);
1621 min_rx_space >>= 10;
1622
1623 /* If current Tx allocation is less than the min Tx FIFO size,
1624 * and the min Tx FIFO size is less than the current Rx FIFO
1625 * allocation, take space away from current Rx allocation */
1626 if (tx_space < min_tx_space &&
1627 ((min_tx_space - tx_space) < pba)) {
1628 pba = pba - (min_tx_space - tx_space);
1629
1630 /* if short on rx space, rx wins and must trump tx
1631 * adjustment */
1632 if (pba < min_rx_space)
1633 pba = min_rx_space;
1634 }
2d064c06 1635 wr32(E1000_PBA, pba);
9d5c8243 1636 }
9d5c8243
AK		 1637
1638 /* flow control settings */
1639 /* The high water mark must be low enough to fit one full frame
1640 * (or the size used for early receive) above it in the Rx FIFO.
1641 * Set it to the lower of:
1642 * - 90% of the Rx FIFO size, or
1643 * - the full Rx FIFO size minus one full frame */
1644 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1645 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1646
d405ea3e
AD		 1647 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1648 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1649 fc->pause_time = 0xFFFF;
1650 fc->send_xon = 1;
0cce119a 1651 fc->current_mode = fc->requested_mode;
9d5c8243 1652
4ae196df
AD		 1653 /* disable receive for all VFs and wait one second */
1654 if (adapter->vfs_allocated_count) {
1655 int i;
1656 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1657 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1658
1659 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1660 igb_ping_all_vfs(adapter);
4ae196df
AD		 1661
1662 /* disable transmits and receives */
1663 wr32(E1000_VFRE, 0);
1664 wr32(E1000_VFTE, 0);
1665 }
1666
9d5c8243 1667 /* Allow time for pending master requests to run */
330a6d6a 1668 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1669 wr32(E1000_WUC, 0);
1670
330a6d6a 1671 if (hw->mac.ops.init_hw(hw))
090b1795 1672 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1673
55cac248
AD		 1674 if (hw->mac.type == e1000_82580) {
1675 u32 reg = rd32(E1000_PCIEMISC);
1676 wr32(E1000_PCIEMISC,
1677 reg & ~E1000_PCIEMISC_LX_DECISION);
1678 }
88a268c1
NN		 1679 if (!netif_running(adapter->netdev))
1680 igb_power_down_link(adapter);
1681
9d5c8243
AK		 1682 igb_update_mng_vlan(adapter);
1683
1684 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1685 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1686
330a6d6a 1687 igb_get_phy_info(hw);
9d5c8243
AK		 1688}
1689
2e5c6922 1690static const struct net_device_ops igb_netdev_ops = {
559e9c49 1691 .ndo_open = igb_open,
2e5c6922 1692 .ndo_stop = igb_close,
00829823 1693 .ndo_start_xmit = igb_xmit_frame_adv,
12dcd86b 1694 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc
AD		 1695 .ndo_set_rx_mode = igb_set_rx_mode,
1696 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1697 .ndo_set_mac_address = igb_set_mac,
1698 .ndo_change_mtu = igb_change_mtu,
1699 .ndo_do_ioctl = igb_ioctl,
1700 .ndo_tx_timeout = igb_tx_timeout,
1701 .ndo_validate_addr = eth_validate_addr,
1702 .ndo_vlan_rx_register = igb_vlan_rx_register,
1703 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1704 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1705 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1706 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1707 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1708 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1709#ifdef CONFIG_NET_POLL_CONTROLLER
1710 .ndo_poll_controller = igb_netpoll,
1711#endif
1712};
1713
9d5c8243
AK		 1714/**
1715 * igb_probe - Device Initialization Routine
1716 * @pdev: PCI device information struct
1717 * @ent: entry in igb_pci_tbl
1718 *
1719 * Returns 0 on success, negative on failure
1720 *
1721 * igb_probe initializes an adapter identified by a pci_dev structure.
1722 * The OS initialization, configuring of the adapter private structure,
1723 * and a hardware reset occur.
1724 **/
1725static int __devinit igb_probe(struct pci_dev *pdev,
1726 const struct pci_device_id *ent)
1727{
1728 struct net_device *netdev;
1729 struct igb_adapter *adapter;
1730 struct e1000_hw *hw;
4337e993
AD		 1731 u16 eeprom_data = 0;
1732 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1733 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1734 unsigned long mmio_start, mmio_len;
2d6a5e95 1735 int err, pci_using_dac;
9d5c8243
AK		 1736 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1737 u32 part_num;
1738
bded64a7
AG		 1739 /* Catch broken hardware that put the wrong VF device ID in
1740 * the PCIe SR-IOV capability.
1741 */
1742 if (pdev->is_virtfn) {
1743 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1744 pci_name(pdev), pdev->vendor, pdev->device);
1745 return -EINVAL;
1746 }
1747
aed5dec3 1748 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1749 if (err)
1750 return err;
1751
1752 pci_using_dac = 0;
59d71989 1753 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1754 if (!err) {
59d71989 1755 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1756 if (!err)
1757 pci_using_dac = 1;
1758 } else {
59d71989 1759 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1760 if (err) {
59d71989 1761 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1762 if (err) {
1763 dev_err(&pdev->dev, "No usable DMA "
1764 "configuration, aborting\n");
1765 goto err_dma;
1766 }
1767 }
1768 }
1769
aed5dec3
AD		 1770 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1771 IORESOURCE_MEM),
1772 igb_driver_name);
9d5c8243
AK		 1773 if (err)
1774 goto err_pci_reg;
1775
19d5afd4 1776 pci_enable_pcie_error_reporting(pdev);
40a914fa 1777
9d5c8243 1778 pci_set_master(pdev);
c682fc23 1779 pci_save_state(pdev);
9d5c8243
AK		 1780
1781 err = -ENOMEM;
1bfaf07b
AD		 1782 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1783 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1784 if (!netdev)
1785 goto err_alloc_etherdev;
1786
1787 SET_NETDEV_DEV(netdev, &pdev->dev);
1788
1789 pci_set_drvdata(pdev, netdev);
1790 adapter = netdev_priv(netdev);
1791 adapter->netdev = netdev;
1792 adapter->pdev = pdev;
1793 hw = &adapter->hw;
1794 hw->back = adapter;
1795 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1796
1797 mmio_start = pci_resource_start(pdev, 0);
1798 mmio_len = pci_resource_len(pdev, 0);
1799
1800 err = -EIO;
28b0759c
AD		 1801 hw->hw_addr = ioremap(mmio_start, mmio_len);
1802 if (!hw->hw_addr)
9d5c8243
AK		 1803 goto err_ioremap;
1804
2e5c6922 1805 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1806 igb_set_ethtool_ops(netdev);
9d5c8243 1807 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1808
1809 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1810
1811 netdev->mem_start = mmio_start;
1812 netdev->mem_end = mmio_start + mmio_len;
1813
9d5c8243
AK		 1814 /* PCI config space info */
1815 hw->vendor_id = pdev->vendor;
1816 hw->device_id = pdev->device;
1817 hw->revision_id = pdev->revision;
1818 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1819 hw->subsystem_device_id = pdev->subsystem_device;
1820
9d5c8243
AK		 1821 /* Copy the default MAC, PHY and NVM function pointers */
1822 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1823 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1824 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1825 /* Initialize skew-specific constants */
1826 err = ei->get_invariants(hw);
1827 if (err)
450c87c8 1828 goto err_sw_init;
9d5c8243 1829
450c87c8 1830 /* setup the private structure */
9d5c8243
AK		 1831 err = igb_sw_init(adapter);
1832 if (err)
1833 goto err_sw_init;
1834
1835 igb_get_bus_info_pcie(hw);
1836
1837 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1838
1839 /* Copper options */
1840 if (hw->phy.media_type == e1000_media_type_copper) {
1841 hw->phy.mdix = AUTO_ALL_MODES;
1842 hw->phy.disable_polarity_correction = false;
1843 hw->phy.ms_type = e1000_ms_hw_default;
1844 }
1845
1846 if (igb_check_reset_block(hw))
1847 dev_info(&pdev->dev,
1848 "PHY reset is blocked due to SOL/IDER session.\n");
1849
1850 netdev->features = NETIF_F_SG |
7d8eb29e 1851 NETIF_F_IP_CSUM |
9d5c8243
AK		 1852 NETIF_F_HW_VLAN_TX |
1853 NETIF_F_HW_VLAN_RX |
1854 NETIF_F_HW_VLAN_FILTER;
1855
7d8eb29e 1856 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1857 netdev->features |= NETIF_F_TSO;
9d5c8243 1858 netdev->features |= NETIF_F_TSO6;
5c0999b7 1859 netdev->features |= NETIF_F_GRO;
d3352520 1860
48f29ffc
JK		 1861 netdev->vlan_features |= NETIF_F_TSO;
1862 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1863 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1864 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1865 netdev->vlan_features |= NETIF_F_SG;
1866
7b872a55 1867 if (pci_using_dac) {
9d5c8243 1868 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1869 netdev->vlan_features |= NETIF_F_HIGHDMA;
1870 }
9d5c8243 1871
5b043fb0 1872 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1873 netdev->features |= NETIF_F_SCTP_CSUM;
1874
330a6d6a 1875 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1876
1877 /* before reading the NVM, reset the controller to put the device in a
1878 * known good starting state */
1879 hw->mac.ops.reset_hw(hw);
1880
1881 /* make sure the NVM is good */
1882 if (igb_validate_nvm_checksum(hw) < 0) {
1883 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1884 err = -EIO;
1885 goto err_eeprom;
1886 }
1887
1888 /* copy the MAC address out of the NVM */
1889 if (hw->mac.ops.read_mac_addr(hw))
1890 dev_err(&pdev->dev, "NVM Read Error\n");
1891
1892 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1893 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1894
1895 if (!is_valid_ether_addr(netdev->perm_addr)) {
1896 dev_err(&pdev->dev, "Invalid MAC Address\n");
1897 err = -EIO;
1898 goto err_eeprom;
1899 }
1900
c061b18d 1901 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 1902 (unsigned long) adapter);
c061b18d 1903 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 1904 (unsigned long) adapter);
9d5c8243
AK		 1905
1906 INIT_WORK(&adapter->reset_task, igb_reset_task);
1907 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1908
450c87c8 1909 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1910 adapter->fc_autoneg = true;
1911 hw->mac.autoneg = true;
1912 hw->phy.autoneg_advertised = 0x2f;
1913
0cce119a
AD		 1914 hw->fc.requested_mode = e1000_fc_default;
1915 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1916
9d5c8243
AK		 1917 igb_validate_mdi_setting(hw);
1918
9d5c8243
AK		 1919 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1920 * enable the ACPI Magic Packet filter
1921 */
1922
a2cf8b6c 1923 if (hw->bus.func == 0)
312c75ae 1924 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1925 else if (hw->mac.type == e1000_82580)
1926 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1927 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1928 &eeprom_data);
a2cf8b6c
AD		 1929 else if (hw->bus.func == 1)
1930 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1931
1932 if (eeprom_data & eeprom_apme_mask)
1933 adapter->eeprom_wol |= E1000_WUFC_MAG;
1934
1935 /* now that we have the eeprom settings, apply the special cases where
1936 * the eeprom may be wrong or the board simply won't support wake on
1937 * lan on a particular port */
1938 switch (pdev->device) {
1939 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1940 adapter->eeprom_wol = 0;
1941 break;
1942 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1943 case E1000_DEV_ID_82576_FIBER:
1944 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1945 /* Wake events only supported on port A for dual fiber
1946 * regardless of eeprom setting */
1947 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1948 adapter->eeprom_wol = 0;
1949 break;
c8ea5ea9 1950 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1951 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1952 /* if quad port adapter, disable WoL on all but port A */
1953 if (global_quad_port_a != 0)
1954 adapter->eeprom_wol = 0;
1955 else
1956 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1957 /* Reset for multiple quad port adapters */
1958 if (++global_quad_port_a == 4)
1959 global_quad_port_a = 0;
1960 break;
9d5c8243
AK		 1961 }
1962
1963 /* initialize the wol settings based on the eeprom settings */
1964 adapter->wol = adapter->eeprom_wol;
e1b86d84 1965 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1966
1967 /* reset the hardware with the new settings */
1968 igb_reset(adapter);
1969
1970 /* let the f/w know that the h/w is now under the control of the
1971 * driver. */
1972 igb_get_hw_control(adapter);
1973
9d5c8243
AK		 1974 strcpy(netdev->name, "eth%d");
1975 err = register_netdev(netdev);
1976 if (err)
1977 goto err_register;
1978
b168dfc5
JB		 1979 /* carrier off reporting is important to ethtool even BEFORE open */
1980 netif_carrier_off(netdev);
1981
421e02f0 1982#ifdef CONFIG_IGB_DCA
bbd98fe4 1983 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1984 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1985 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1986 igb_setup_dca(adapter);
1987 }
fe4506b6 1988
38c845c7 1989#endif
9d5c8243
AK		 1990 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1991 /* print bus type/speed/width info */
7c510e4b 1992 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1993 netdev->name,
559e9c49 1994 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1995 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1996 "unknown"),
59c3de89
AD		 1997 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1998 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1999 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2000 "unknown"),
7c510e4b 2001 netdev->dev_addr);
9d5c8243
AK		 2002
2003 igb_read_part_num(hw, &part_num);
2004 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
2005 (part_num >> 8), (part_num & 0xff));
2006
2007 dev_info(&pdev->dev,
2008 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2009 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2010 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 2011 adapter->num_rx_queues, adapter->num_tx_queues);
2012
9d5c8243
AK		 2013 return 0;
2014
2015err_register:
2016 igb_release_hw_control(adapter);
2017err_eeprom:
2018 if (!igb_check_reset_block(hw))
f5f4cf08 2019 igb_reset_phy(hw);
9d5c8243
AK		 2020
2021 if (hw->flash_address)
2022 iounmap(hw->flash_address);
9d5c8243 2023err_sw_init:
047e0030 2024 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2025 iounmap(hw->hw_addr);
2026err_ioremap:
2027 free_netdev(netdev);
2028err_alloc_etherdev:
559e9c49
AD		 2029 pci_release_selected_regions(pdev,
2030 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2031err_pci_reg:
2032err_dma:
2033 pci_disable_device(pdev);
2034 return err;
2035}
2036
2037/**
2038 * igb_remove - Device Removal Routine
2039 * @pdev: PCI device information struct
2040 *
2041 * igb_remove is called by the PCI subsystem to alert the driver
2042 * that it should release a PCI device. The could be caused by a
2043 * Hot-Plug event, or because the driver is going to be removed from
2044 * memory.
2045 **/
2046static void __devexit igb_remove(struct pci_dev *pdev)
2047{
2048 struct net_device *netdev = pci_get_drvdata(pdev);
2049 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2050 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2051
2052 /* flush_scheduled work may reschedule our watchdog task, so
2053 * explicitly disable watchdog tasks from being rescheduled */
2054 set_bit(__IGB_DOWN, &adapter->state);
2055 del_timer_sync(&adapter->watchdog_timer);
2056 del_timer_sync(&adapter->phy_info_timer);
2057
2058 flush_scheduled_work();
2059
421e02f0 2060#ifdef CONFIG_IGB_DCA
7dfc16fa 2061 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2062 dev_info(&pdev->dev, "DCA disabled\n");
2063 dca_remove_requester(&pdev->dev);
7dfc16fa 2064 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2065 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2066 }
2067#endif
2068
9d5c8243
AK		 2069 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2070 * would have already happened in close and is redundant. */
2071 igb_release_hw_control(adapter);
2072
2073 unregister_netdev(netdev);
2074
047e0030 2075 igb_clear_interrupt_scheme(adapter);
9d5c8243 2076
37680117
AD		 2077#ifdef CONFIG_PCI_IOV
2078 /* reclaim resources allocated to VFs */
2079 if (adapter->vf_data) {
2080 /* disable iov and allow time for transactions to clear */
2081 pci_disable_sriov(pdev);
2082 msleep(500);
2083
2084 kfree(adapter->vf_data);
2085 adapter->vf_data = NULL;
2086 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2087 msleep(100);
2088 dev_info(&pdev->dev, "IOV Disabled\n");
2089 }
2090#endif
559e9c49 2091
28b0759c
AD		 2092 iounmap(hw->hw_addr);
2093 if (hw->flash_address)
2094 iounmap(hw->flash_address);
559e9c49
AD		 2095 pci_release_selected_regions(pdev,
2096 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2097
2098 free_netdev(netdev);
2099
19d5afd4 2100 pci_disable_pcie_error_reporting(pdev);
40a914fa 2101
9d5c8243
AK		 2102 pci_disable_device(pdev);
2103}
2104
a6b623e0
AD		 2105/**
2106 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2107 * @adapter: board private structure to initialize
2108 *
2109 * This function initializes the vf specific data storage and then attempts to
2110 * allocate the VFs. The reason for ordering it this way is because it is much
2111 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2112 * the memory for the VFs.
2113 **/
2114static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2115{
2116#ifdef CONFIG_PCI_IOV
2117 struct pci_dev *pdev = adapter->pdev;
2118
a6b623e0
AD		 2119 if (adapter->vfs_allocated_count) {
2120 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2121 sizeof(struct vf_data_storage),
2122 GFP_KERNEL);
2123 /* if allocation failed then we do not support SR-IOV */
2124 if (!adapter->vf_data) {
2125 adapter->vfs_allocated_count = 0;
2126 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2127 "Data Storage\n");
2128 }
2129 }
2130
2131 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2132 kfree(adapter->vf_data);
2133 adapter->vf_data = NULL;
2134#endif /* CONFIG_PCI_IOV */
2135 adapter->vfs_allocated_count = 0;
2136#ifdef CONFIG_PCI_IOV
2137 } else {
2138 unsigned char mac_addr[ETH_ALEN];
2139 int i;
2140 dev_info(&pdev->dev, "%d vfs allocated\n",
2141 adapter->vfs_allocated_count);
2142 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2143 random_ether_addr(mac_addr);
2144 igb_set_vf_mac(adapter, i, mac_addr);
2145 }
2146 }
2147#endif /* CONFIG_PCI_IOV */
2148}
2149
115f459a
AD		 2150
2151/**
2152 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2153 * @adapter: board private structure to initialize
2154 *
2155 * igb_init_hw_timer initializes the function pointer and values for the hw
2156 * timer found in hardware.
2157 **/
2158static void igb_init_hw_timer(struct igb_adapter *adapter)
2159{
2160 struct e1000_hw *hw = &adapter->hw;
2161
2162 switch (hw->mac.type) {
d2ba2ed8 2163 case e1000_i350:
55cac248
AD		 2164 case e1000_82580:
2165 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2166 adapter->cycles.read = igb_read_clock;
2167 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2168 adapter->cycles.mult = 1;
2169 /*
2170 * The 82580 timesync updates the system timer every 8ns by 8ns
2171 * and the value cannot be shifted. Instead we need to shift
2172 * the registers to generate a 64bit timer value. As a result
2173 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2174 * 24 in order to generate a larger value for synchronization.
2175 */
2176 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2177 /* disable system timer temporarily by setting bit 31 */
2178 wr32(E1000_TSAUXC, 0x80000000);
2179 wrfl();
2180
2181 /* Set registers so that rollover occurs soon to test this. */
2182 wr32(E1000_SYSTIMR, 0x00000000);
2183 wr32(E1000_SYSTIML, 0x80000000);
2184 wr32(E1000_SYSTIMH, 0x000000FF);
2185 wrfl();
2186
2187 /* enable system timer by clearing bit 31 */
2188 wr32(E1000_TSAUXC, 0x0);
2189 wrfl();
2190
2191 timecounter_init(&adapter->clock,
2192 &adapter->cycles,
2193 ktime_to_ns(ktime_get_real()));
2194 /*
2195 * Synchronize our NIC clock against system wall clock. NIC
2196 * time stamp reading requires ~3us per sample, each sample
2197 * was pretty stable even under load => only require 10
2198 * samples for each offset comparison.
2199 */
2200 memset(&adapter->compare, 0, sizeof(adapter->compare));
2201 adapter->compare.source = &adapter->clock;
2202 adapter->compare.target = ktime_get_real;
2203 adapter->compare.num_samples = 10;
2204 timecompare_update(&adapter->compare, 0);
2205 break;
115f459a
AD		 2206 case e1000_82576:
2207 /*
2208 * Initialize hardware timer: we keep it running just in case
2209 * that some program needs it later on.
2210 */
2211 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2212 adapter->cycles.read = igb_read_clock;
2213 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2214 adapter->cycles.mult = 1;
2215 /**
2216 * Scale the NIC clock cycle by a large factor so that
2217 * relatively small clock corrections can be added or
2218 * substracted at each clock tick. The drawbacks of a large
2219 * factor are a) that the clock register overflows more quickly
2220 * (not such a big deal) and b) that the increment per tick has
2221 * to fit into 24 bits. As a result we need to use a shift of
2222 * 19 so we can fit a value of 16 into the TIMINCA register.
2223 */
2224 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2225 wr32(E1000_TIMINCA,
2226 (1 << E1000_TIMINCA_16NS_SHIFT) |
2227 (16 << IGB_82576_TSYNC_SHIFT));
2228
2229 /* Set registers so that rollover occurs soon to test this. */
2230 wr32(E1000_SYSTIML, 0x00000000);
2231 wr32(E1000_SYSTIMH, 0xFF800000);
2232 wrfl();
2233
2234 timecounter_init(&adapter->clock,
2235 &adapter->cycles,
2236 ktime_to_ns(ktime_get_real()));
2237 /*
2238 * Synchronize our NIC clock against system wall clock. NIC
2239 * time stamp reading requires ~3us per sample, each sample
2240 * was pretty stable even under load => only require 10
2241 * samples for each offset comparison.
2242 */
2243 memset(&adapter->compare, 0, sizeof(adapter->compare));
2244 adapter->compare.source = &adapter->clock;
2245 adapter->compare.target = ktime_get_real;
2246 adapter->compare.num_samples = 10;
2247 timecompare_update(&adapter->compare, 0);
2248 break;
2249 case e1000_82575:
2250 /* 82575 does not support timesync */
2251 default:
2252 break;
2253 }
2254
2255}
2256
9d5c8243
AK		 2257/**
2258 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2259 * @adapter: board private structure to initialize
2260 *
2261 * igb_sw_init initializes the Adapter private data structure.
2262 * Fields are initialized based on PCI device information and
2263 * OS network device settings (MTU size).
2264 **/
2265static int __devinit igb_sw_init(struct igb_adapter *adapter)
2266{
2267 struct e1000_hw *hw = &adapter->hw;
2268 struct net_device *netdev = adapter->netdev;
2269 struct pci_dev *pdev = adapter->pdev;
2270
2271 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2272
68fd9910
AD		 2273 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2274 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2275 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2276 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2277
9d5c8243
AK		 2278 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2279 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2280
12dcd86b 2281 spin_lock_init(&adapter->stats64_lock);
a6b623e0
AD		 2282#ifdef CONFIG_PCI_IOV
2283 if (hw->mac.type == e1000_82576)
c0f2276f 2284 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2285
2286#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2287 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2288
2289 /*
2290 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2291 * then we should combine the queues into a queue pair in order to
2292 * conserve interrupts due to limited supply
2293 */
2294 if ((adapter->rss_queues > 4) ||
2295 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2296 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2297
a6b623e0 2298 /* This call may decrease the number of queues */
047e0030 2299 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2300 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2301 return -ENOMEM;
2302 }
2303
115f459a 2304 igb_init_hw_timer(adapter);
a6b623e0
AD		 2305 igb_probe_vfs(adapter);
2306
9d5c8243
AK		 2307 /* Explicitly disable IRQ since the NIC can be in any state. */
2308 igb_irq_disable(adapter);
2309
2310 set_bit(__IGB_DOWN, &adapter->state);
2311 return 0;
2312}
2313
2314/**
2315 * igb_open - Called when a network interface is made active
2316 * @netdev: network interface device structure
2317 *
2318 * Returns 0 on success, negative value on failure
2319 *
2320 * The open entry point is called when a network interface is made
2321 * active by the system (IFF_UP). At this point all resources needed
2322 * for transmit and receive operations are allocated, the interrupt
2323 * handler is registered with the OS, the watchdog timer is started,
2324 * and the stack is notified that the interface is ready.
2325 **/
2326static int igb_open(struct net_device *netdev)
2327{
2328 struct igb_adapter *adapter = netdev_priv(netdev);
2329 struct e1000_hw *hw = &adapter->hw;
2330 int err;
2331 int i;
2332
2333 /* disallow open during test */
2334 if (test_bit(__IGB_TESTING, &adapter->state))
2335 return -EBUSY;
2336
b168dfc5
JB		 2337 netif_carrier_off(netdev);
2338
9d5c8243
AK		 2339 /* allocate transmit descriptors */
2340 err = igb_setup_all_tx_resources(adapter);
2341 if (err)
2342 goto err_setup_tx;
2343
2344 /* allocate receive descriptors */
2345 err = igb_setup_all_rx_resources(adapter);
2346 if (err)
2347 goto err_setup_rx;
2348
88a268c1 2349 igb_power_up_link(adapter);
9d5c8243 2350
9d5c8243
AK		 2351 /* before we allocate an interrupt, we must be ready to handle it.
2352 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2353 * as soon as we call pci_request_irq, so we have to setup our
2354 * clean_rx handler before we do so. */
2355 igb_configure(adapter);
2356
2357 err = igb_request_irq(adapter);
2358 if (err)
2359 goto err_req_irq;
2360
2361 /* From here on the code is the same as igb_up() */
2362 clear_bit(__IGB_DOWN, &adapter->state);
2363
047e0030
AD		 2364 for (i = 0; i < adapter->num_q_vectors; i++) {
2365 struct igb_q_vector *q_vector = adapter->q_vector[i];
2366 napi_enable(&q_vector->napi);
2367 }
9d5c8243
AK		 2368
2369 /* Clear any pending interrupts. */
2370 rd32(E1000_ICR);
844290e5
PW		 2371
2372 igb_irq_enable(adapter);
2373
d4960307
AD		 2374 /* notify VFs that reset has been completed */
2375 if (adapter->vfs_allocated_count) {
2376 u32 reg_data = rd32(E1000_CTRL_EXT);
2377 reg_data |= E1000_CTRL_EXT_PFRSTD;
2378 wr32(E1000_CTRL_EXT, reg_data);
2379 }
2380
d55b53ff
JK		 2381 netif_tx_start_all_queues(netdev);
2382
25568a53
AD		 2383 /* start the watchdog. */
2384 hw->mac.get_link_status = 1;
2385 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2386
2387 return 0;
2388
2389err_req_irq:
2390 igb_release_hw_control(adapter);
88a268c1 2391 igb_power_down_link(adapter);
9d5c8243
AK		 2392 igb_free_all_rx_resources(adapter);
2393err_setup_rx:
2394 igb_free_all_tx_resources(adapter);
2395err_setup_tx:
2396 igb_reset(adapter);
2397
2398 return err;
2399}
2400
2401/**
2402 * igb_close - Disables a network interface
2403 * @netdev: network interface device structure
2404 *
2405 * Returns 0, this is not allowed to fail
2406 *
2407 * The close entry point is called when an interface is de-activated
2408 * by the OS. The hardware is still under the driver's control, but
2409 * needs to be disabled. A global MAC reset is issued to stop the
2410 * hardware, and all transmit and receive resources are freed.
2411 **/
2412static int igb_close(struct net_device *netdev)
2413{
2414 struct igb_adapter *adapter = netdev_priv(netdev);
2415
2416 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2417 igb_down(adapter);
2418
2419 igb_free_irq(adapter);
2420
2421 igb_free_all_tx_resources(adapter);
2422 igb_free_all_rx_resources(adapter);
2423
9d5c8243
AK		 2424 return 0;
2425}
2426
2427/**
2428 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2429 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2430 *
2431 * Return 0 on success, negative on failure
2432 **/
80785298 2433int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2434{
59d71989 2435 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2436 int size;
2437
2438 size = sizeof(struct igb_buffer) * tx_ring->count;
2439 tx_ring->buffer_info = vmalloc(size);
2440 if (!tx_ring->buffer_info)
2441 goto err;
2442 memset(tx_ring->buffer_info, 0, size);
2443
2444 /* round up to nearest 4K */
85e8d004 2445 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2446 tx_ring->size = ALIGN(tx_ring->size, 4096);
2447
59d71989
AD		 2448 tx_ring->desc = dma_alloc_coherent(dev,
2449 tx_ring->size,
2450 &tx_ring->dma,
2451 GFP_KERNEL);
9d5c8243
AK		 2452
2453 if (!tx_ring->desc)
2454 goto err;
2455
9d5c8243
AK		 2456 tx_ring->next_to_use = 0;
2457 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2458 return 0;
2459
2460err:
2461 vfree(tx_ring->buffer_info);
59d71989 2462 dev_err(dev,
9d5c8243
AK		 2463 "Unable to allocate memory for the transmit descriptor ring\n");
2464 return -ENOMEM;
2465}
2466
2467/**
2468 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2469 * (Descriptors) for all queues
2470 * @adapter: board private structure
2471 *
2472 * Return 0 on success, negative on failure
2473 **/
2474static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2475{
439705e1 2476 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2477 int i, err = 0;
2478
2479 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2480 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2481 if (err) {
439705e1 2482 dev_err(&pdev->dev,
9d5c8243
AK		 2483 "Allocation for Tx Queue %u failed\n", i);
2484 for (i--; i >= 0; i--)
3025a446 2485 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2486 break;
2487 }
2488 }
2489
a99955fc 2490 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2491 int r_idx = i % adapter->num_tx_queues;
3025a446 2492 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2493 }
9d5c8243
AK		 2494 return err;
2495}
2496
2497/**
85b430b4
AD		 2498 * igb_setup_tctl - configure the transmit control registers
2499 * @adapter: Board private structure
9d5c8243 2500 **/
d7ee5b3a 2501void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2502{
9d5c8243
AK		 2503 struct e1000_hw *hw = &adapter->hw;
2504 u32 tctl;
9d5c8243 2505
85b430b4
AD		 2506 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2507 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2508
2509 /* Program the Transmit Control Register */
9d5c8243
AK		 2510 tctl = rd32(E1000_TCTL);
2511 tctl &= ~E1000_TCTL_CT;
2512 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2513 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2514
2515 igb_config_collision_dist(hw);
2516
9d5c8243
AK		 2517 /* Enable transmits */
2518 tctl |= E1000_TCTL_EN;
2519
2520 wr32(E1000_TCTL, tctl);
2521}
2522
85b430b4
AD		 2523/**
2524 * igb_configure_tx_ring - Configure transmit ring after Reset
2525 * @adapter: board private structure
2526 * @ring: tx ring to configure
2527 *
2528 * Configure a transmit ring after a reset.
2529 **/
d7ee5b3a
AD		 2530void igb_configure_tx_ring(struct igb_adapter *adapter,
2531 struct igb_ring *ring)
85b430b4
AD		 2532{
2533 struct e1000_hw *hw = &adapter->hw;
2534 u32 txdctl;
2535 u64 tdba = ring->dma;
2536 int reg_idx = ring->reg_idx;
2537
2538 /* disable the queue */
2539 txdctl = rd32(E1000_TXDCTL(reg_idx));
2540 wr32(E1000_TXDCTL(reg_idx),
2541 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2542 wrfl();
2543 mdelay(10);
2544
2545 wr32(E1000_TDLEN(reg_idx),
2546 ring->count * sizeof(union e1000_adv_tx_desc));
2547 wr32(E1000_TDBAL(reg_idx),
2548 tdba & 0x00000000ffffffffULL);
2549 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2550
fce99e34
AD		 2551 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2552 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2553 writel(0, ring->head);
2554 writel(0, ring->tail);
85b430b4
AD		 2555
2556 txdctl |= IGB_TX_PTHRESH;
2557 txdctl |= IGB_TX_HTHRESH << 8;
2558 txdctl |= IGB_TX_WTHRESH << 16;
2559
2560 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2561 wr32(E1000_TXDCTL(reg_idx), txdctl);
2562}
2563
2564/**
2565 * igb_configure_tx - Configure transmit Unit after Reset
2566 * @adapter: board private structure
2567 *
2568 * Configure the Tx unit of the MAC after a reset.
2569 **/
2570static void igb_configure_tx(struct igb_adapter *adapter)
2571{
2572 int i;
2573
2574 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2575 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2576}
2577
9d5c8243
AK		 2578/**
2579 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2580 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2581 *
2582 * Returns 0 on success, negative on failure
2583 **/
80785298 2584int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2585{
59d71989 2586 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2587 int size, desc_len;
2588
2589 size = sizeof(struct igb_buffer) * rx_ring->count;
2590 rx_ring->buffer_info = vmalloc(size);
2591 if (!rx_ring->buffer_info)
2592 goto err;
2593 memset(rx_ring->buffer_info, 0, size);
2594
2595 desc_len = sizeof(union e1000_adv_rx_desc);
2596
2597 /* Round up to nearest 4K */
2598 rx_ring->size = rx_ring->count * desc_len;
2599 rx_ring->size = ALIGN(rx_ring->size, 4096);
2600
59d71989
AD		 2601 rx_ring->desc = dma_alloc_coherent(dev,
2602 rx_ring->size,
2603 &rx_ring->dma,
2604 GFP_KERNEL);
9d5c8243
AK		 2605
2606 if (!rx_ring->desc)
2607 goto err;
2608
2609 rx_ring->next_to_clean = 0;
2610 rx_ring->next_to_use = 0;
9d5c8243 2611
9d5c8243
AK		 2612 return 0;
2613
2614err:
2615 vfree(rx_ring->buffer_info);
439705e1 2616 rx_ring->buffer_info = NULL;
59d71989
AD		 2617 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2618 " ring\n");
9d5c8243
AK		 2619 return -ENOMEM;
2620}
2621
2622/**
2623 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2624 * (Descriptors) for all queues
2625 * @adapter: board private structure
2626 *
2627 * Return 0 on success, negative on failure
2628 **/
2629static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2630{
439705e1 2631 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2632 int i, err = 0;
2633
2634 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2635 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2636 if (err) {
439705e1 2637 dev_err(&pdev->dev,
9d5c8243
AK		 2638 "Allocation for Rx Queue %u failed\n", i);
2639 for (i--; i >= 0; i--)
3025a446 2640 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2641 break;
2642 }
2643 }
2644
2645 return err;
2646}
2647
06cf2666
AD		 2648/**
2649 * igb_setup_mrqc - configure the multiple receive queue control registers
2650 * @adapter: Board private structure
2651 **/
2652static void igb_setup_mrqc(struct igb_adapter *adapter)
2653{
2654 struct e1000_hw *hw = &adapter->hw;
2655 u32 mrqc, rxcsum;
2656 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2657 union e1000_reta {
2658 u32 dword;
2659 u8 bytes[4];
2660 } reta;
2661 static const u8 rsshash[40] = {
2662 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2663 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2664 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2665 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2666
2667 /* Fill out hash function seeds */
2668 for (j = 0; j < 10; j++) {
2669 u32 rsskey = rsshash[(j * 4)];
2670 rsskey |= rsshash[(j * 4) + 1] << 8;
2671 rsskey |= rsshash[(j * 4) + 2] << 16;
2672 rsskey |= rsshash[(j * 4) + 3] << 24;
2673 array_wr32(E1000_RSSRK(0), j, rsskey);
2674 }
2675
a99955fc 2676 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2677
2678 if (adapter->vfs_allocated_count) {
2679 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2680 switch (hw->mac.type) {
d2ba2ed8 2681 case e1000_i350:
55cac248
AD		 2682 case e1000_82580:
2683 num_rx_queues = 1;
2684 shift = 0;
2685 break;
06cf2666
AD		 2686 case e1000_82576:
2687 shift = 3;
2688 num_rx_queues = 2;
2689 break;
2690 case e1000_82575:
2691 shift = 2;
2692 shift2 = 6;
2693 default:
2694 break;
2695 }
2696 } else {
2697 if (hw->mac.type == e1000_82575)
2698 shift = 6;
2699 }
2700
2701 for (j = 0; j < (32 * 4); j++) {
2702 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2703 if (shift2)
2704 reta.bytes[j & 3] |= num_rx_queues << shift2;
2705 if ((j & 3) == 3)
2706 wr32(E1000_RETA(j >> 2), reta.dword);
2707 }
2708
2709 /*
2710 * Disable raw packet checksumming so that RSS hash is placed in
2711 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2712 * offloads as they are enabled by default
2713 */
2714 rxcsum = rd32(E1000_RXCSUM);
2715 rxcsum |= E1000_RXCSUM_PCSD;
2716
2717 if (adapter->hw.mac.type >= e1000_82576)
2718 /* Enable Receive Checksum Offload for SCTP */
2719 rxcsum |= E1000_RXCSUM_CRCOFL;
2720
2721 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2722 wr32(E1000_RXCSUM, rxcsum);
2723
2724 /* If VMDq is enabled then we set the appropriate mode for that, else
2725 * we default to RSS so that an RSS hash is calculated per packet even
2726 * if we are only using one queue */
2727 if (adapter->vfs_allocated_count) {
2728 if (hw->mac.type > e1000_82575) {
2729 /* Set the default pool for the PF's first queue */
2730 u32 vtctl = rd32(E1000_VT_CTL);
2731 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2732 E1000_VT_CTL_DISABLE_DEF_POOL);
2733 vtctl |= adapter->vfs_allocated_count <<
2734 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2735 wr32(E1000_VT_CTL, vtctl);
2736 }
a99955fc 2737 if (adapter->rss_queues > 1)
06cf2666
AD		 2738 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2739 else
2740 mrqc = E1000_MRQC_ENABLE_VMDQ;
2741 } else {
2742 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2743 }
2744 igb_vmm_control(adapter);
2745
4478a9cd
AD		 2746 /*
2747 * Generate RSS hash based on TCP port numbers and/or
2748 * IPv4/v6 src and dst addresses since UDP cannot be
2749 * hashed reliably due to IP fragmentation
2750 */
2751 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2752 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2753 E1000_MRQC_RSS_FIELD_IPV6 |
2754 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2755 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2756
2757 wr32(E1000_MRQC, mrqc);
2758}
2759
9d5c8243
AK		 2760/**
2761 * igb_setup_rctl - configure the receive control registers
2762 * @adapter: Board private structure
2763 **/
d7ee5b3a 2764void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2765{
2766 struct e1000_hw *hw = &adapter->hw;
2767 u32 rctl;
9d5c8243
AK		 2768
2769 rctl = rd32(E1000_RCTL);
2770
2771 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2772 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2773
69d728ba 2774 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2775 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2776
87cb7e8c
AK		 2777 /*
2778 * enable stripping of CRC. It's unlikely this will break BMC
2779 * redirection as it did with e1000. Newer features require
2780 * that the HW strips the CRC.
73cd78f1 2781 */
87cb7e8c 2782 rctl |= E1000_RCTL_SECRC;
9d5c8243 2783
559e9c49 2784 /* disable store bad packets and clear size bits. */
ec54d7d6 2785 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2786
6ec43fe6
AD		 2787 /* enable LPE to prevent packets larger than max_frame_size */
2788 rctl |= E1000_RCTL_LPE;
9d5c8243 2789
952f72a8
AD		 2790 /* disable queue 0 to prevent tail write w/o re-config */
2791 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2792
e1739522
AD		 2793 /* Attention!!! For SR-IOV PF driver operations you must enable
2794 * queue drop for all VF and PF queues to prevent head of line blocking
2795 * if an un-trusted VF does not provide descriptors to hardware.
2796 */
2797 if (adapter->vfs_allocated_count) {
e1739522
AD		 2798 /* set all queue drop enable bits */
2799 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2800 }
2801
9d5c8243
AK		 2802 wr32(E1000_RCTL, rctl);
2803}
2804
7d5753f0
AD		 2805static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2806 int vfn)
2807{
2808 struct e1000_hw *hw = &adapter->hw;
2809 u32 vmolr;
2810
2811 /* if it isn't the PF check to see if VFs are enabled and
2812 * increase the size to support vlan tags */
2813 if (vfn < adapter->vfs_allocated_count &&
2814 adapter->vf_data[vfn].vlans_enabled)
2815 size += VLAN_TAG_SIZE;
2816
2817 vmolr = rd32(E1000_VMOLR(vfn));
2818 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2819 vmolr |= size | E1000_VMOLR_LPE;
2820 wr32(E1000_VMOLR(vfn), vmolr);
2821
2822 return 0;
2823}
2824
e1739522
AD		 2825/**
2826 * igb_rlpml_set - set maximum receive packet size
2827 * @adapter: board private structure
2828 *
2829 * Configure maximum receivable packet size.
2830 **/
2831static void igb_rlpml_set(struct igb_adapter *adapter)
2832{
2833 u32 max_frame_size = adapter->max_frame_size;
2834 struct e1000_hw *hw = &adapter->hw;
2835 u16 pf_id = adapter->vfs_allocated_count;
2836
2837 if (adapter->vlgrp)
2838 max_frame_size += VLAN_TAG_SIZE;
2839
2840 /* if vfs are enabled we set RLPML to the largest possible request
2841 * size and set the VMOLR RLPML to the size we need */
2842 if (pf_id) {
2843 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2844 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2845 }
2846
2847 wr32(E1000_RLPML, max_frame_size);
2848}
2849
8151d294
WM		 2850static inline void igb_set_vmolr(struct igb_adapter *adapter,
2851 int vfn, bool aupe)
7d5753f0
AD		 2852{
2853 struct e1000_hw *hw = &adapter->hw;
2854 u32 vmolr;
2855
2856 /*
2857 * This register exists only on 82576 and newer so if we are older then
2858 * we should exit and do nothing
2859 */
2860 if (hw->mac.type < e1000_82576)
2861 return;
2862
2863 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2864 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2865 if (aupe)
2866 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2867 else
2868 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2869
2870 /* clear all bits that might not be set */
2871 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2872
a99955fc 2873 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2874 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2875 /*
2876 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2877 * multicast packets
2878 */
2879 if (vfn <= adapter->vfs_allocated_count)
2880 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2881
2882 wr32(E1000_VMOLR(vfn), vmolr);
2883}
2884
85b430b4
AD		 2885/**
2886 * igb_configure_rx_ring - Configure a receive ring after Reset
2887 * @adapter: board private structure
2888 * @ring: receive ring to be configured
2889 *
2890 * Configure the Rx unit of the MAC after a reset.
2891 **/
d7ee5b3a
AD		 2892void igb_configure_rx_ring(struct igb_adapter *adapter,
2893 struct igb_ring *ring)
85b430b4
AD		 2894{
2895 struct e1000_hw *hw = &adapter->hw;
2896 u64 rdba = ring->dma;
2897 int reg_idx = ring->reg_idx;
952f72a8 2898 u32 srrctl, rxdctl;
85b430b4
AD		 2899
2900 /* disable the queue */
2901 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2902 wr32(E1000_RXDCTL(reg_idx),
2903 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2904
2905 /* Set DMA base address registers */
2906 wr32(E1000_RDBAL(reg_idx),
2907 rdba & 0x00000000ffffffffULL);
2908 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2909 wr32(E1000_RDLEN(reg_idx),
2910 ring->count * sizeof(union e1000_adv_rx_desc));
2911
2912 /* initialize head and tail */
fce99e34
AD		 2913 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2914 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2915 writel(0, ring->head);
2916 writel(0, ring->tail);
85b430b4 2917
952f72a8 2918 /* set descriptor configuration */
4c844851
AD		 2919 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2920 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2921 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2922#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2923 srrctl |= IGB_RXBUFFER_16384 >>
2924 E1000_SRRCTL_BSIZEPKT_SHIFT;
2925#else
2926 srrctl |= (PAGE_SIZE / 2) >>
2927 E1000_SRRCTL_BSIZEPKT_SHIFT;
2928#endif
2929 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2930 } else {
4c844851 2931 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2932 E1000_SRRCTL_BSIZEPKT_SHIFT;
2933 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2934 }
757b77e2
NN		 2935 if (hw->mac.type == e1000_82580)
2936 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2937 /* Only set Drop Enable if we are supporting multiple queues */
2938 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2939 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2940
2941 wr32(E1000_SRRCTL(reg_idx), srrctl);
2942
7d5753f0 2943 /* set filtering for VMDQ pools */
8151d294 2944 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2945
85b430b4
AD		 2946 /* enable receive descriptor fetching */
2947 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2948 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2949 rxdctl &= 0xFFF00000;
2950 rxdctl |= IGB_RX_PTHRESH;
2951 rxdctl |= IGB_RX_HTHRESH << 8;
2952 rxdctl |= IGB_RX_WTHRESH << 16;
2953 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2954}
2955
9d5c8243
AK		 2956/**
2957 * igb_configure_rx - Configure receive Unit after Reset
2958 * @adapter: board private structure
2959 *
2960 * Configure the Rx unit of the MAC after a reset.
2961 **/
2962static void igb_configure_rx(struct igb_adapter *adapter)
2963{
9107584e 2964 int i;
9d5c8243 2965
68d480c4
AD		 2966 /* set UTA to appropriate mode */
2967 igb_set_uta(adapter);
2968
26ad9178
AD		 2969 /* set the correct pool for the PF default MAC address in entry 0 */
2970 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2971 adapter->vfs_allocated_count);
2972
06cf2666
AD		 2973 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2974 * the Base and Length of the Rx Descriptor Ring */
2975 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2976 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2977}
2978
2979/**
2980 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2981 * @tx_ring: Tx descriptor ring for a specific queue
2982 *
2983 * Free all transmit software resources
2984 **/
68fd9910 2985void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2986{
3b644cf6 2987 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2988
2989 vfree(tx_ring->buffer_info);
2990 tx_ring->buffer_info = NULL;
2991
439705e1
AD		 2992 /* if not set, then don't free */
2993 if (!tx_ring->desc)
2994 return;
2995
59d71989
AD		 2996 dma_free_coherent(tx_ring->dev, tx_ring->size,
2997 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2998
2999 tx_ring->desc = NULL;
3000}
3001
3002/**
3003 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3004 * @adapter: board private structure
3005 *
3006 * Free all transmit software resources
3007 **/
3008static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3009{
3010 int i;
3011
3012 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3013 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3014}
3015
b1a436c3
AD		 3016void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3017 struct igb_buffer *buffer_info)
9d5c8243 3018{
6366ad33
AD		 3019 if (buffer_info->dma) {
3020 if (buffer_info->mapped_as_page)
59d71989 3021 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3022 buffer_info->dma,
3023 buffer_info->length,
59d71989 3024 DMA_TO_DEVICE);
6366ad33 3025 else
59d71989 3026 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3027 buffer_info->dma,
3028 buffer_info->length,
59d71989 3029 DMA_TO_DEVICE);
6366ad33
AD		 3030 buffer_info->dma = 0;
3031 }
9d5c8243
AK		 3032 if (buffer_info->skb) {
3033 dev_kfree_skb_any(buffer_info->skb);
3034 buffer_info->skb = NULL;
3035 }
3036 buffer_info->time_stamp = 0;
6366ad33
AD		 3037 buffer_info->length = 0;
3038 buffer_info->next_to_watch = 0;
3039 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3040}
3041
3042/**
3043 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3044 * @tx_ring: ring to be cleaned
3045 **/
3b644cf6 3046static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3047{
3048 struct igb_buffer *buffer_info;
3049 unsigned long size;
3050 unsigned int i;
3051
3052 if (!tx_ring->buffer_info)
3053 return;
3054 /* Free all the Tx ring sk_buffs */
3055
3056 for (i = 0; i < tx_ring->count; i++) {
3057 buffer_info = &tx_ring->buffer_info[i];
80785298 3058 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3059 }
3060
3061 size = sizeof(struct igb_buffer) * tx_ring->count;
3062 memset(tx_ring->buffer_info, 0, size);
3063
3064 /* Zero out the descriptor ring */
9d5c8243
AK		 3065 memset(tx_ring->desc, 0, tx_ring->size);
3066
3067 tx_ring->next_to_use = 0;
3068 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3069}
3070
3071/**
3072 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3073 * @adapter: board private structure
3074 **/
3075static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3076{
3077 int i;
3078
3079 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3080 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3081}
3082
3083/**
3084 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3085 * @rx_ring: ring to clean the resources from
3086 *
3087 * Free all receive software resources
3088 **/
68fd9910 3089void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3090{
3b644cf6 3091 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3092
3093 vfree(rx_ring->buffer_info);
3094 rx_ring->buffer_info = NULL;
3095
439705e1
AD		 3096 /* if not set, then don't free */
3097 if (!rx_ring->desc)
3098 return;
3099
59d71989
AD		 3100 dma_free_coherent(rx_ring->dev, rx_ring->size,
3101 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3102
3103 rx_ring->desc = NULL;
3104}
3105
3106/**
3107 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3108 * @adapter: board private structure
3109 *
3110 * Free all receive software resources
3111 **/
3112static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3113{
3114 int i;
3115
3116 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3117 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3118}
3119
3120/**
3121 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3122 * @rx_ring: ring to free buffers from
3123 **/
3b644cf6 3124static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3125{
3126 struct igb_buffer *buffer_info;
9d5c8243
AK		 3127 unsigned long size;
3128 unsigned int i;
3129
3130 if (!rx_ring->buffer_info)
3131 return;
439705e1 3132
9d5c8243
AK		 3133 /* Free all the Rx ring sk_buffs */
3134 for (i = 0; i < rx_ring->count; i++) {
3135 buffer_info = &rx_ring->buffer_info[i];
3136 if (buffer_info->dma) {
59d71989 3137 dma_unmap_single(rx_ring->dev,
80785298 3138 buffer_info->dma,
4c844851 3139 rx_ring->rx_buffer_len,
59d71989 3140 DMA_FROM_DEVICE);
9d5c8243
AK		 3141 buffer_info->dma = 0;
3142 }
3143
3144 if (buffer_info->skb) {
3145 dev_kfree_skb(buffer_info->skb);
3146 buffer_info->skb = NULL;
3147 }
6ec43fe6 3148 if (buffer_info->page_dma) {
59d71989 3149 dma_unmap_page(rx_ring->dev,
80785298 3150 buffer_info->page_dma,
6ec43fe6 3151 PAGE_SIZE / 2,
59d71989 3152 DMA_FROM_DEVICE);
6ec43fe6
AD		 3153 buffer_info->page_dma = 0;
3154 }
9d5c8243 3155 if (buffer_info->page) {
9d5c8243
AK		 3156 put_page(buffer_info->page);
3157 buffer_info->page = NULL;
bf36c1a0 3158 buffer_info->page_offset = 0;
9d5c8243
AK		 3159 }
3160 }
3161
9d5c8243
AK		 3162 size = sizeof(struct igb_buffer) * rx_ring->count;
3163 memset(rx_ring->buffer_info, 0, size);
3164
3165 /* Zero out the descriptor ring */
3166 memset(rx_ring->desc, 0, rx_ring->size);
3167
3168 rx_ring->next_to_clean = 0;
3169 rx_ring->next_to_use = 0;
9d5c8243
AK		 3170}
3171
3172/**
3173 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3174 * @adapter: board private structure
3175 **/
3176static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3177{
3178 int i;
3179
3180 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3181 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3182}
3183
3184/**
3185 * igb_set_mac - Change the Ethernet Address of the NIC
3186 * @netdev: network interface device structure
3187 * @p: pointer to an address structure
3188 *
3189 * Returns 0 on success, negative on failure
3190 **/
3191static int igb_set_mac(struct net_device *netdev, void *p)
3192{
3193 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3194 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3195 struct sockaddr *addr = p;
3196
3197 if (!is_valid_ether_addr(addr->sa_data))
3198 return -EADDRNOTAVAIL;
3199
3200 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3201 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3202
26ad9178
AD		 3203 /* set the correct pool for the new PF MAC address in entry 0 */
3204 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3205 adapter->vfs_allocated_count);
e1739522 3206
9d5c8243
AK		 3207 return 0;
3208}
3209
3210/**
68d480c4 3211 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3212 * @netdev: network interface device structure
3213 *
68d480c4
AD		 3214 * Writes multicast address list to the MTA hash table.
3215 * Returns: -ENOMEM on failure
3216 * 0 on no addresses written
3217 * X on writing X addresses to MTA
9d5c8243 3218 **/
68d480c4 3219static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3220{
3221 struct igb_adapter *adapter = netdev_priv(netdev);
3222 struct e1000_hw *hw = &adapter->hw;
22bedad3 3223 struct netdev_hw_addr *ha;
68d480c4 3224 u8 *mta_list;
9d5c8243
AK		 3225 int i;
3226
4cd24eaf 3227 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3228 /* nothing to program, so clear mc list */
3229 igb_update_mc_addr_list(hw, NULL, 0);
3230 igb_restore_vf_multicasts(adapter);
3231 return 0;
3232 }
9d5c8243 3233
4cd24eaf 3234 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3235 if (!mta_list)
3236 return -ENOMEM;
ff41f8dc 3237
68d480c4 3238 /* The shared function expects a packed array of only addresses. */
48e2f183 3239 i = 0;
22bedad3
JP		 3240 netdev_for_each_mc_addr(ha, netdev)
3241 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3242
68d480c4
AD		 3243 igb_update_mc_addr_list(hw, mta_list, i);
3244 kfree(mta_list);
3245
4cd24eaf 3246 return netdev_mc_count(netdev);
68d480c4
AD		 3247}
3248
3249/**
3250 * igb_write_uc_addr_list - write unicast addresses to RAR table
3251 * @netdev: network interface device structure
3252 *
3253 * Writes unicast address list to the RAR table.
3254 * Returns: -ENOMEM on failure/insufficient address space
3255 * 0 on no addresses written
3256 * X on writing X addresses to the RAR table
3257 **/
3258static int igb_write_uc_addr_list(struct net_device *netdev)
3259{
3260 struct igb_adapter *adapter = netdev_priv(netdev);
3261 struct e1000_hw *hw = &adapter->hw;
3262 unsigned int vfn = adapter->vfs_allocated_count;
3263 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3264 int count = 0;
3265
3266 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3267 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3268 return -ENOMEM;
9d5c8243 3269
32e7bfc4 3270 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3271 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3272
3273 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3274 if (!rar_entries)
3275 break;
26ad9178
AD		 3276 igb_rar_set_qsel(adapter, ha->addr,
3277 rar_entries--,
68d480c4
AD		 3278 vfn);
3279 count++;
ff41f8dc
AD		 3280 }
3281 }
3282 /* write the addresses in reverse order to avoid write combining */
3283 for (; rar_entries > 0 ; rar_entries--) {
3284 wr32(E1000_RAH(rar_entries), 0);
3285 wr32(E1000_RAL(rar_entries), 0);
3286 }
3287 wrfl();
3288
68d480c4
AD		 3289 return count;
3290}
3291
3292/**
3293 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3294 * @netdev: network interface device structure
3295 *
3296 * The set_rx_mode entry point is called whenever the unicast or multicast
3297 * address lists or the network interface flags are updated. This routine is
3298 * responsible for configuring the hardware for proper unicast, multicast,
3299 * promiscuous mode, and all-multi behavior.
3300 **/
3301static void igb_set_rx_mode(struct net_device *netdev)
3302{
3303 struct igb_adapter *adapter = netdev_priv(netdev);
3304 struct e1000_hw *hw = &adapter->hw;
3305 unsigned int vfn = adapter->vfs_allocated_count;
3306 u32 rctl, vmolr = 0;
3307 int count;
3308
3309 /* Check for Promiscuous and All Multicast modes */
3310 rctl = rd32(E1000_RCTL);
3311
3312 /* clear the effected bits */
3313 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3314
3315 if (netdev->flags & IFF_PROMISC) {
3316 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3317 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3318 } else {
3319 if (netdev->flags & IFF_ALLMULTI) {
3320 rctl |= E1000_RCTL_MPE;
3321 vmolr |= E1000_VMOLR_MPME;
3322 } else {
3323 /*
3324 * Write addresses to the MTA, if the attempt fails
3325 * then we should just turn on promiscous mode so
3326 * that we can at least receive multicast traffic
3327 */
3328 count = igb_write_mc_addr_list(netdev);
3329 if (count < 0) {
3330 rctl |= E1000_RCTL_MPE;
3331 vmolr |= E1000_VMOLR_MPME;
3332 } else if (count) {
3333 vmolr |= E1000_VMOLR_ROMPE;
3334 }
3335 }
3336 /*
3337 * Write addresses to available RAR registers, if there is not
3338 * sufficient space to store all the addresses then enable
3339 * unicast promiscous mode
3340 */
3341 count = igb_write_uc_addr_list(netdev);
3342 if (count < 0) {
3343 rctl |= E1000_RCTL_UPE;
3344 vmolr |= E1000_VMOLR_ROPE;
3345 }
3346 rctl |= E1000_RCTL_VFE;
28fc06f5 3347 }
68d480c4 3348 wr32(E1000_RCTL, rctl);
28fc06f5 3349
68d480c4
AD		 3350 /*
3351 * In order to support SR-IOV and eventually VMDq it is necessary to set
3352 * the VMOLR to enable the appropriate modes. Without this workaround
3353 * we will have issues with VLAN tag stripping not being done for frames
3354 * that are only arriving because we are the default pool
3355 */
3356 if (hw->mac.type < e1000_82576)
28fc06f5 3357 return;
9d5c8243 3358
68d480c4
AD		 3359 vmolr |= rd32(E1000_VMOLR(vfn)) &
3360 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3361 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3362 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3363}
3364
3365/* Need to wait a few seconds after link up to get diagnostic information from
3366 * the phy */
3367static void igb_update_phy_info(unsigned long data)
3368{
3369 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3370 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3371}
3372
4d6b725e
AD		 3373/**
3374 * igb_has_link - check shared code for link and determine up/down
3375 * @adapter: pointer to driver private info
3376 **/
3145535a 3377bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3378{
3379 struct e1000_hw *hw = &adapter->hw;
3380 bool link_active = false;
3381 s32 ret_val = 0;
3382
3383 /* get_link_status is set on LSC (link status) interrupt or
3384 * rx sequence error interrupt. get_link_status will stay
3385 * false until the e1000_check_for_link establishes link
3386 * for copper adapters ONLY
3387 */
3388 switch (hw->phy.media_type) {
3389 case e1000_media_type_copper:
3390 if (hw->mac.get_link_status) {
3391 ret_val = hw->mac.ops.check_for_link(hw);
3392 link_active = !hw->mac.get_link_status;
3393 } else {
3394 link_active = true;
3395 }
3396 break;
4d6b725e
AD		 3397 case e1000_media_type_internal_serdes:
3398 ret_val = hw->mac.ops.check_for_link(hw);
3399 link_active = hw->mac.serdes_has_link;
3400 break;
3401 default:
3402 case e1000_media_type_unknown:
3403 break;
3404 }
3405
3406 return link_active;
3407}
3408
9d5c8243
AK		 3409/**
3410 * igb_watchdog - Timer Call-back
3411 * @data: pointer to adapter cast into an unsigned long
3412 **/
3413static void igb_watchdog(unsigned long data)
3414{
3415 struct igb_adapter *adapter = (struct igb_adapter *)data;
3416 /* Do the rest outside of interrupt context */
3417 schedule_work(&adapter->watchdog_task);
3418}
3419
3420static void igb_watchdog_task(struct work_struct *work)
3421{
3422 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3423 struct igb_adapter,
3424 watchdog_task);
9d5c8243 3425 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3426 struct net_device *netdev = adapter->netdev;
9d5c8243 3427 u32 link;
7a6ea550 3428 int i;
9d5c8243 3429
4d6b725e 3430 link = igb_has_link(adapter);
9d5c8243
AK		 3431 if (link) {
3432 if (!netif_carrier_ok(netdev)) {
3433 u32 ctrl;
330a6d6a
AD		 3434 hw->mac.ops.get_speed_and_duplex(hw,
3435 &adapter->link_speed,
3436 &adapter->link_duplex);
9d5c8243
AK		 3437
3438 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3439 /* Links status message must follow this format */
3440 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3441 "Flow Control: %s\n",
559e9c49
AD		 3442 netdev->name,
3443 adapter->link_speed,
3444 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3445 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3446 ((ctrl & E1000_CTRL_TFCE) &&
3447 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3448 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3449 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3450
d07f3e37 3451 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3452 adapter->tx_timeout_factor = 1;
3453 switch (adapter->link_speed) {
3454 case SPEED_10:
9d5c8243
AK		 3455 adapter->tx_timeout_factor = 14;
3456 break;
3457 case SPEED_100:
9d5c8243
AK		 3458 /* maybe add some timeout factor ? */
3459 break;
3460 }
3461
3462 netif_carrier_on(netdev);
9d5c8243 3463
4ae196df
AD		 3464 igb_ping_all_vfs(adapter);
3465
4b1a9877 3466 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3470 }
3471 } else {
3472 if (netif_carrier_ok(netdev)) {
3473 adapter->link_speed = 0;
3474 adapter->link_duplex = 0;
527d47c1
AD		 3475 /* Links status message must follow this format */
3476 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3477 netdev->name);
9d5c8243 3478 netif_carrier_off(netdev);
4b1a9877 3479
4ae196df
AD		 3480 igb_ping_all_vfs(adapter);
3481
4b1a9877 3482 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3483 if (!test_bit(__IGB_DOWN, &adapter->state))
3484 mod_timer(&adapter->phy_info_timer,
3485 round_jiffies(jiffies + 2 * HZ));
3486 }
3487 }
3488
12dcd86b
ED		 3489 spin_lock(&adapter->stats64_lock);
3490 igb_update_stats(adapter, &adapter->stats64);
3491 spin_unlock(&adapter->stats64_lock);
9d5c8243 3492
dbabb065 3493 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3494 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3495 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3496 /* We've lost link, so the controller stops DMA,
3497 * but we've got queued Tx work that's never going
3498 * to get done, so reset controller to flush Tx.
3499 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3500 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3501 adapter->tx_timeout_count++;
3502 schedule_work(&adapter->reset_task);
3503 /* return immediately since reset is imminent */
3504 return;
3505 }
9d5c8243 3506 }
9d5c8243 3507
dbabb065
AD		 3508 /* Force detection of hung controller every watchdog period */
3509 tx_ring->detect_tx_hung = true;
3510 }
f7ba205e 3511
9d5c8243 3512 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3513 if (adapter->msix_entries) {
047e0030
AD		 3514 u32 eics = 0;
3515 for (i = 0; i < adapter->num_q_vectors; i++) {
3516 struct igb_q_vector *q_vector = adapter->q_vector[i];
3517 eics |= q_vector->eims_value;
3518 }
7a6ea550
AD		 3519 wr32(E1000_EICS, eics);
3520 } else {
3521 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3522 }
9d5c8243 3523
9d5c8243
AK		 3524 /* Reset the timer */
3525 if (!test_bit(__IGB_DOWN, &adapter->state))
3526 mod_timer(&adapter->watchdog_timer,
3527 round_jiffies(jiffies + 2 * HZ));
3528}
3529
3530enum latency_range {
3531 lowest_latency = 0,
3532 low_latency = 1,
3533 bulk_latency = 2,
3534 latency_invalid = 255
3535};
3536
6eb5a7f1
AD		 3537/**
3538 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3539 *
3540 * Stores a new ITR value based on strictly on packet size. This
3541 * algorithm is less sophisticated than that used in igb_update_itr,
3542 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3543 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3544 * were determined based on theoretical maximum wire speed and testing
3545 * data, in order to minimize response time while increasing bulk
3546 * throughput.
3547 * This functionality is controlled by the InterruptThrottleRate module
3548 * parameter (see igb_param.c)
3549 * NOTE: This function is called only when operating in a multiqueue
3550 * receive environment.
047e0030 3551 * @q_vector: pointer to q_vector
6eb5a7f1 3552 **/
047e0030 3553static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3554{
047e0030 3555 int new_val = q_vector->itr_val;
6eb5a7f1 3556 int avg_wire_size = 0;
047e0030 3557 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3558 struct igb_ring *ring;
3559 unsigned int packets;
9d5c8243 3560
6eb5a7f1
AD		 3561 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3562 * ints/sec - ITR timer value of 120 ticks.
3563 */
3564 if (adapter->link_speed != SPEED_1000) {
047e0030 3565 new_val = 976;
6eb5a7f1 3566 goto set_itr_val;
9d5c8243 3567 }
047e0030 3568
12dcd86b
ED		 3569 ring = q_vector->rx_ring;
3570 if (ring) {
3571 packets = ACCESS_ONCE(ring->total_packets);
3572
3573 if (packets)
3574 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3575 }
3576
12dcd86b
ED		 3577 ring = q_vector->tx_ring;
3578 if (ring) {
3579 packets = ACCESS_ONCE(ring->total_packets);
3580
3581 if (packets)
3582 avg_wire_size = max_t(u32, avg_wire_size,
3583 ring->total_bytes / packets);
047e0030
AD		 3584 }
3585
3586 /* if avg_wire_size isn't set no work was done */
3587 if (!avg_wire_size)
3588 goto clear_counts;
9d5c8243 3589
6eb5a7f1
AD		 3590 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3591 avg_wire_size += 24;
3592
3593 /* Don't starve jumbo frames */
3594 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3595
6eb5a7f1
AD		 3596 /* Give a little boost to mid-size frames */
3597 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3598 new_val = avg_wire_size / 3;
3599 else
3600 new_val = avg_wire_size / 2;
9d5c8243 3601
abe1c363
NN		 3602 /* when in itr mode 3 do not exceed 20K ints/sec */
3603 if (adapter->rx_itr_setting == 3 && new_val < 196)
3604 new_val = 196;
3605
6eb5a7f1 3606set_itr_val:
047e0030
AD		 3607 if (new_val != q_vector->itr_val) {
3608 q_vector->itr_val = new_val;
3609 q_vector->set_itr = 1;
9d5c8243 3610 }
6eb5a7f1 3611clear_counts:
047e0030
AD		 3612 if (q_vector->rx_ring) {
3613 q_vector->rx_ring->total_bytes = 0;
3614 q_vector->rx_ring->total_packets = 0;
3615 }
3616 if (q_vector->tx_ring) {
3617 q_vector->tx_ring->total_bytes = 0;
3618 q_vector->tx_ring->total_packets = 0;
3619 }
9d5c8243
AK		 3620}
3621
3622/**
3623 * igb_update_itr - update the dynamic ITR value based on statistics
3624 * Stores a new ITR value based on packets and byte
3625 * counts during the last interrupt. The advantage of per interrupt
3626 * computation is faster updates and more accurate ITR for the current
3627 * traffic pattern. Constants in this function were computed
3628 * based on theoretical maximum wire speed and thresholds were set based
3629 * on testing data as well as attempting to minimize response time
3630 * while increasing bulk throughput.
3631 * this functionality is controlled by the InterruptThrottleRate module
3632 * parameter (see igb_param.c)
3633 * NOTE: These calculations are only valid when operating in a single-
3634 * queue environment.
3635 * @adapter: pointer to adapter
047e0030 3636 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3637 * @packets: the number of packets during this measurement interval
3638 * @bytes: the number of bytes during this measurement interval
3639 **/
3640static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3641 int packets, int bytes)
3642{
3643 unsigned int retval = itr_setting;
3644
3645 if (packets == 0)
3646 goto update_itr_done;
3647
3648 switch (itr_setting) {
3649 case lowest_latency:
3650 /* handle TSO and jumbo frames */
3651 if (bytes/packets > 8000)
3652 retval = bulk_latency;
3653 else if ((packets < 5) && (bytes > 512))
3654 retval = low_latency;
3655 break;
3656 case low_latency: /* 50 usec aka 20000 ints/s */
3657 if (bytes > 10000) {
3658 /* this if handles the TSO accounting */
3659 if (bytes/packets > 8000) {
3660 retval = bulk_latency;
3661 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3662 retval = bulk_latency;
3663 } else if ((packets > 35)) {
3664 retval = lowest_latency;
3665 }
3666 } else if (bytes/packets > 2000) {
3667 retval = bulk_latency;
3668 } else if (packets <= 2 && bytes < 512) {
3669 retval = lowest_latency;
3670 }
3671 break;
3672 case bulk_latency: /* 250 usec aka 4000 ints/s */
3673 if (bytes > 25000) {
3674 if (packets > 35)
3675 retval = low_latency;
1e5c3d21 3676 } else if (bytes < 1500) {
9d5c8243
AK		 3677 retval = low_latency;
3678 }
3679 break;
3680 }
3681
3682update_itr_done:
3683 return retval;
3684}
3685
6eb5a7f1 3686static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3687{
047e0030 3688 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3689 u16 current_itr;
047e0030 3690 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3691
3692 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3693 if (adapter->link_speed != SPEED_1000) {
3694 current_itr = 0;
3695 new_itr = 4000;
3696 goto set_itr_now;
3697 }
3698
3699 adapter->rx_itr = igb_update_itr(adapter,
3700 adapter->rx_itr,
3025a446
AD		 3701 q_vector->rx_ring->total_packets,
3702 q_vector->rx_ring->total_bytes);
9d5c8243 3703
047e0030
AD		 3704 adapter->tx_itr = igb_update_itr(adapter,
3705 adapter->tx_itr,
3025a446
AD		 3706 q_vector->tx_ring->total_packets,
3707 q_vector->tx_ring->total_bytes);
047e0030 3708 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3709
6eb5a7f1 3710 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3711 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3712 current_itr = low_latency;
3713
9d5c8243
AK		 3714 switch (current_itr) {
3715 /* counts and packets in update_itr are dependent on these numbers */
3716 case lowest_latency:
78b1f607 3717 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3718 break;
3719 case low_latency:
78b1f607 3720 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3721 break;
3722 case bulk_latency:
78b1f607 3723 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3724 break;
3725 default:
3726 break;
3727 }
3728
3729set_itr_now:
3025a446
AD		 3730 q_vector->rx_ring->total_bytes = 0;
3731 q_vector->rx_ring->total_packets = 0;
3732 q_vector->tx_ring->total_bytes = 0;
3733 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3734
047e0030 3735 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3736 /* this attempts to bias the interrupt rate towards Bulk
3737 * by adding intermediate steps when interrupt rate is
3738 * increasing */
047e0030
AD		 3739 new_itr = new_itr > q_vector->itr_val ?
3740 max((new_itr * q_vector->itr_val) /
3741 (new_itr + (q_vector->itr_val >> 2)),
3742 new_itr) :
9d5c8243
AK		 3743 new_itr;
3744 /* Don't write the value here; it resets the adapter's
3745 * internal timer, and causes us to delay far longer than
3746 * we should between interrupts. Instead, we write the ITR
3747 * value at the beginning of the next interrupt so the timing
3748 * ends up being correct.
3749 */
047e0030
AD		 3750 q_vector->itr_val = new_itr;
3751 q_vector->set_itr = 1;
9d5c8243 3752 }
9d5c8243
AK		 3753}
3754
9d5c8243
AK		 3755#define IGB_TX_FLAGS_CSUM 0x00000001
3756#define IGB_TX_FLAGS_VLAN 0x00000002
3757#define IGB_TX_FLAGS_TSO 0x00000004
3758#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3759#define IGB_TX_FLAGS_TSTAMP 0x00000010
3760#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3761#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3762
85ad76b2 3763static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3764 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3765{
3766 struct e1000_adv_tx_context_desc *context_desc;
3767 unsigned int i;
3768 int err;
3769 struct igb_buffer *buffer_info;
3770 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3771 u32 mss_l4len_idx;
3772 u8 l4len;
9d5c8243
AK		 3773
3774 if (skb_header_cloned(skb)) {
3775 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3776 if (err)
3777 return err;
3778 }
3779
3780 l4len = tcp_hdrlen(skb);
3781 *hdr_len += l4len;
3782
3783 if (skb->protocol == htons(ETH_P_IP)) {
3784 struct iphdr *iph = ip_hdr(skb);
3785 iph->tot_len = 0;
3786 iph->check = 0;
3787 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3788 iph->daddr, 0,
3789 IPPROTO_TCP,
3790 0);
8e1e8a47 3791 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3792 ipv6_hdr(skb)->payload_len = 0;
3793 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3794 &ipv6_hdr(skb)->daddr,
3795 0, IPPROTO_TCP, 0);
3796 }
3797
3798 i = tx_ring->next_to_use;
3799
3800 buffer_info = &tx_ring->buffer_info[i];
3801 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3802 /* VLAN MACLEN IPLEN */
3803 if (tx_flags & IGB_TX_FLAGS_VLAN)
3804 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3805 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3806 *hdr_len += skb_network_offset(skb);
3807 info |= skb_network_header_len(skb);
3808 *hdr_len += skb_network_header_len(skb);
3809 context_desc->vlan_macip_lens = cpu_to_le32(info);
3810
3811 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3812 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3813
3814 if (skb->protocol == htons(ETH_P_IP))
3815 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3816 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3817
3818 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3819
3820 /* MSS L4LEN IDX */
3821 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3822 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3823
73cd78f1 3824 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3825 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3826 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3827
3828 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3829 context_desc->seqnum_seed = 0;
3830
3831 buffer_info->time_stamp = jiffies;
0e014cb1 3832 buffer_info->next_to_watch = i;
9d5c8243
AK		 3833 buffer_info->dma = 0;
3834 i++;
3835 if (i == tx_ring->count)
3836 i = 0;
3837
3838 tx_ring->next_to_use = i;
3839
3840 return true;
3841}
3842
85ad76b2
AD		 3843static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3844 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3845{
3846 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3847 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3848 struct igb_buffer *buffer_info;
3849 u32 info = 0, tu_cmd = 0;
80785298 3850 unsigned int i;
9d5c8243
AK		 3851
3852 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3853 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3854 i = tx_ring->next_to_use;
3855 buffer_info = &tx_ring->buffer_info[i];
3856 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3857
3858 if (tx_flags & IGB_TX_FLAGS_VLAN)
3859 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3860
9d5c8243
AK		 3861 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3862 if (skb->ip_summed == CHECKSUM_PARTIAL)
3863 info |= skb_network_header_len(skb);
3864
3865 context_desc->vlan_macip_lens = cpu_to_le32(info);
3866
3867 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3868
3869 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3870 __be16 protocol;
3871
3872 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3873 const struct vlan_ethhdr *vhdr =
3874 (const struct vlan_ethhdr*)skb->data;
3875
3876 protocol = vhdr->h_vlan_encapsulated_proto;
3877 } else {
3878 protocol = skb->protocol;
3879 }
3880
3881 switch (protocol) {
09640e63 3882 case cpu_to_be16(ETH_P_IP):
9d5c8243 3883 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3884 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3885 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3886 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3887 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3888 break;
09640e63 3889 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3890 /* XXX what about other V6 headers?? */
3891 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3892 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3893 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3894 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3895 break;
3896 default:
3897 if (unlikely(net_ratelimit()))
59d71989 3898 dev_warn(dev,
44b0cda3
MW		 3899 "partial checksum but proto=%x!\n",
3900 skb->protocol);
3901 break;
3902 }
9d5c8243
AK		 3903 }
3904
3905 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3906 context_desc->seqnum_seed = 0;
85ad76b2 3907 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3908 context_desc->mss_l4len_idx =
85ad76b2 3909 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3910
3911 buffer_info->time_stamp = jiffies;
0e014cb1 3912 buffer_info->next_to_watch = i;
9d5c8243
AK		 3913 buffer_info->dma = 0;
3914
3915 i++;
3916 if (i == tx_ring->count)
3917 i = 0;
3918 tx_ring->next_to_use = i;
3919
3920 return true;
3921 }
9d5c8243
AK		 3922 return false;
3923}
3924
3925#define IGB_MAX_TXD_PWR 16
3926#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3927
80785298 3928static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3929 unsigned int first)
9d5c8243
AK		 3930{
3931 struct igb_buffer *buffer_info;
59d71989 3932 struct device *dev = tx_ring->dev;
2873957d 3933 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3934 unsigned int count = 0, i;
3935 unsigned int f;
2873957d 3936 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3937
3938 i = tx_ring->next_to_use;
3939
3940 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3941 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3942 buffer_info->length = hlen;
9d5c8243
AK		 3943 /* set time_stamp *before* dma to help avoid a possible race */
3944 buffer_info->time_stamp = jiffies;
0e014cb1 3945 buffer_info->next_to_watch = i;
2873957d 3946 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3947 DMA_TO_DEVICE);
3948 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3949 goto dma_error;
9d5c8243
AK		 3950
3951 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3952 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3953 unsigned int len = frag->size;
9d5c8243 3954
8581145f 3955 count++;
65689fef
AD		 3956 i++;
3957 if (i == tx_ring->count)
3958 i = 0;
3959
9d5c8243
AK		 3960 buffer_info = &tx_ring->buffer_info[i];
3961 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3962 buffer_info->length = len;
3963 buffer_info->time_stamp = jiffies;
0e014cb1 3964 buffer_info->next_to_watch = i;
6366ad33 3965 buffer_info->mapped_as_page = true;
59d71989 3966 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3967 frag->page,
3968 frag->page_offset,
3969 len,
59d71989
AD		 3970 DMA_TO_DEVICE);
3971 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3972 goto dma_error;
3973
9d5c8243
AK		 3974 }
3975
9d5c8243 3976 tx_ring->buffer_info[i].skb = skb;
2244d07b 3977 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 3978 /* multiply data chunks by size of headers */
3979 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3980 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3981 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3982
cdfd01fc 3983 return ++count;
6366ad33
AD		 3984
3985dma_error:
59d71989 3986 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3987
3988 /* clear timestamp and dma mappings for failed buffer_info mapping */
3989 buffer_info->dma = 0;
3990 buffer_info->time_stamp = 0;
3991 buffer_info->length = 0;
3992 buffer_info->next_to_watch = 0;
3993 buffer_info->mapped_as_page = false;
6366ad33
AD		 3994
3995 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3996 while (count--) {
3997 if (i == 0)
3998 i = tx_ring->count;
6366ad33 3999 i--;
6366ad33
AD		 4000 buffer_info = &tx_ring->buffer_info[i];
4001 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4002 }
4003
4004 return 0;
9d5c8243
AK		 4005}
4006
85ad76b2 4007static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 4008 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 4009 u8 hdr_len)
4010{
cdfd01fc 4011 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4012 struct igb_buffer *buffer_info;
4013 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4014 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4015
4016 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4017 E1000_ADVTXD_DCMD_DEXT);
4018
4019 if (tx_flags & IGB_TX_FLAGS_VLAN)
4020 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4021
33af6bcc
PO		 4022 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4023 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4024
9d5c8243
AK		 4025 if (tx_flags & IGB_TX_FLAGS_TSO) {
4026 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4027
4028 /* insert tcp checksum */
4029 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4030
4031 /* insert ip checksum */
4032 if (tx_flags & IGB_TX_FLAGS_IPV4)
4033 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4034
4035 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4036 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4037 }
4038
85ad76b2
AD		 4039 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4040 (tx_flags & (IGB_TX_FLAGS_CSUM |
4041 IGB_TX_FLAGS_TSO |
7dfc16fa 4042 IGB_TX_FLAGS_VLAN)))
85ad76b2 4043 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4044
4045 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4046
cdfd01fc 4047 do {
9d5c8243
AK		 4048 buffer_info = &tx_ring->buffer_info[i];
4049 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4050 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4051 tx_desc->read.cmd_type_len =
4052 cpu_to_le32(cmd_type_len | buffer_info->length);
4053 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4054 count--;
9d5c8243
AK		 4055 i++;
4056 if (i == tx_ring->count)
4057 i = 0;
cdfd01fc 4058 } while (count > 0);
9d5c8243 4059
85ad76b2 4060 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4061 /* Force memory writes to complete before letting h/w
4062 * know there are new descriptors to fetch. (Only
4063 * applicable for weak-ordered memory model archs,
4064 * such as IA-64). */
4065 wmb();
4066
4067 tx_ring->next_to_use = i;
fce99e34 4068 writel(i, tx_ring->tail);
9d5c8243
AK		 4069 /* we need this if more than one processor can write to our tail
4070 * at a time, it syncronizes IO on IA64/Altix systems */
4071 mmiowb();
4072}
4073
e694e964 4074static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4075{
e694e964
AD		 4076 struct net_device *netdev = tx_ring->netdev;
4077
661086df 4078 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4079
9d5c8243
AK		 4080 /* Herbert's original patch had:
4081 * smp_mb__after_netif_stop_queue();
4082 * but since that doesn't exist yet, just open code it. */
4083 smp_mb();
4084
4085 /* We need to check again in a case another CPU has just
4086 * made room available. */
c493ea45 4087 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4088 return -EBUSY;
4089
4090 /* A reprieve! */
661086df 4091 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4092
4093 u64_stats_update_begin(&tx_ring->tx_syncp2);
4094 tx_ring->tx_stats.restart_queue2++;
4095 u64_stats_update_end(&tx_ring->tx_syncp2);
4096
9d5c8243
AK		 4097 return 0;
4098}
4099
717ba089 4100static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4101{
c493ea45 4102 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4103 return 0;
e694e964 4104 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4105}
4106
b1a436c3
AD		 4107netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4108 struct igb_ring *tx_ring)
9d5c8243 4109{
cdfd01fc 4110 int tso = 0, count;
91d4ee33
NN		 4111 u32 tx_flags = 0;
4112 u16 first;
4113 u8 hdr_len = 0;
9d5c8243 4114
9d5c8243
AK		 4115 /* need: 1 descriptor per page,
4116 * + 2 desc gap to keep tail from touching head,
4117 * + 1 desc for skb->data,
4118 * + 1 desc for context descriptor,
4119 * otherwise try next time */
e694e964 4120 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4121 /* this is a hard error */
9d5c8243
AK		 4122 return NETDEV_TX_BUSY;
4123 }
33af6bcc 4124
2244d07b
OH		 4125 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4126 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4127 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4128 }
9d5c8243 4129
eab6d18d 4130 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4131 tx_flags |= IGB_TX_FLAGS_VLAN;
4132 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4133 }
4134
661086df
PWJ		 4135 if (skb->protocol == htons(ETH_P_IP))
4136 tx_flags |= IGB_TX_FLAGS_IPV4;
4137
0e014cb1 4138 first = tx_ring->next_to_use;
85ad76b2
AD		 4139 if (skb_is_gso(skb)) {
4140 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4141
85ad76b2
AD		 4142 if (tso < 0) {
4143 dev_kfree_skb_any(skb);
4144 return NETDEV_TX_OK;
4145 }
9d5c8243
AK		 4146 }
4147
4148 if (tso)
4149 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4150 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4151 (skb->ip_summed == CHECKSUM_PARTIAL))
4152 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4153
65689fef 4154 /*
cdfd01fc 4155 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4156 * has occured and we need to rewind the descriptor queue
4157 */
80785298 4158 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4159 if (!count) {
65689fef
AD		 4160 dev_kfree_skb_any(skb);
4161 tx_ring->buffer_info[first].time_stamp = 0;
4162 tx_ring->next_to_use = first;
85ad76b2 4163 return NETDEV_TX_OK;
65689fef 4164 }
9d5c8243 4165
85ad76b2
AD		 4166 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4167
4168 /* Make sure there is space in the ring for the next send. */
e694e964 4169 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4170
9d5c8243
AK		 4171 return NETDEV_TX_OK;
4172}
4173
3b29a56d
SH		 4174static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4175 struct net_device *netdev)
9d5c8243
AK		 4176{
4177 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4178 struct igb_ring *tx_ring;
661086df 4179 int r_idx = 0;
b1a436c3
AD		 4180
4181 if (test_bit(__IGB_DOWN, &adapter->state)) {
4182 dev_kfree_skb_any(skb);
4183 return NETDEV_TX_OK;
4184 }
4185
4186 if (skb->len <= 0) {
4187 dev_kfree_skb_any(skb);
4188 return NETDEV_TX_OK;
4189 }
4190
1bfaf07b 4191 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4192 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4193
4194 /* This goes back to the question of how to logically map a tx queue
4195 * to a flow. Right now, performance is impacted slightly negatively
4196 * if using multiple tx queues. If the stack breaks away from a
4197 * single qdisc implementation, we can look at this again. */
e694e964 4198 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4199}
4200
4201/**
4202 * igb_tx_timeout - Respond to a Tx Hang
4203 * @netdev: network interface device structure
4204 **/
4205static void igb_tx_timeout(struct net_device *netdev)
4206{
4207 struct igb_adapter *adapter = netdev_priv(netdev);
4208 struct e1000_hw *hw = &adapter->hw;
4209
4210 /* Do the reset outside of interrupt context */
4211 adapter->tx_timeout_count++;
f7ba205e 4212
55cac248
AD		 4213 if (hw->mac.type == e1000_82580)
4214 hw->dev_spec._82575.global_device_reset = true;
4215
9d5c8243 4216 schedule_work(&adapter->reset_task);
265de409
AD		 4217 wr32(E1000_EICS,
4218 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4219}
4220
4221static void igb_reset_task(struct work_struct *work)
4222{
4223 struct igb_adapter *adapter;
4224 adapter = container_of(work, struct igb_adapter, reset_task);
4225
c97ec42a
TI		 4226 igb_dump(adapter);
4227 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4228 igb_reinit_locked(adapter);
4229}
4230
4231/**
12dcd86b 4232 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4233 * @netdev: network interface device structure
12dcd86b 4234 * @stats: rtnl_link_stats64 pointer
9d5c8243 4235 *
9d5c8243 4236 **/
12dcd86b
ED		 4237static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4238 struct rtnl_link_stats64 *stats)
9d5c8243 4239{
12dcd86b
ED		 4240 struct igb_adapter *adapter = netdev_priv(netdev);
4241
4242 spin_lock(&adapter->stats64_lock);
4243 igb_update_stats(adapter, &adapter->stats64);
4244 memcpy(stats, &adapter->stats64, sizeof(*stats));
4245 spin_unlock(&adapter->stats64_lock);
4246
4247 return stats;
9d5c8243
AK		 4248}
4249
4250/**
4251 * igb_change_mtu - Change the Maximum Transfer Unit
4252 * @netdev: network interface device structure
4253 * @new_mtu: new value for maximum frame size
4254 *
4255 * Returns 0 on success, negative on failure
4256 **/
4257static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4258{
4259 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4260 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4261 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4262 u32 rx_buffer_len, i;
9d5c8243 4263
c809d227 4264 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4265 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4266 return -EINVAL;
4267 }
4268
9d5c8243 4269 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4270 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4271 return -EINVAL;
4272 }
4273
4274 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4275 msleep(1);
73cd78f1 4276
9d5c8243
AK		 4277 /* igb_down has a dependency on max_frame_size */
4278 adapter->max_frame_size = max_frame;
559e9c49 4279
9d5c8243
AK		 4280 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4281 * means we reserve 2 more, this pushes us to allocate from the next
4282 * larger slab size.
4283 * i.e. RXBUFFER_2048 --> size-4096 slab
4284 */
4285
757b77e2
NN		 4286 if (adapter->hw.mac.type == e1000_82580)
4287 max_frame += IGB_TS_HDR_LEN;
4288
7d95b717 4289 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4290 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4291 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4292 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4293 else
4c844851
AD		 4294 rx_buffer_len = IGB_RXBUFFER_128;
4295
757b77e2
NN		 4296 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4297 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4298 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4299
4300 if ((adapter->hw.mac.type == e1000_82580) &&
4301 (rx_buffer_len == IGB_RXBUFFER_128))
4302 rx_buffer_len += IGB_RXBUFFER_64;
4303
4c844851
AD		 4304 if (netif_running(netdev))
4305 igb_down(adapter);
9d5c8243 4306
090b1795 4307 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4308 netdev->mtu, new_mtu);
4309 netdev->mtu = new_mtu;
4310
4c844851 4311 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4312 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4313
9d5c8243
AK		 4314 if (netif_running(netdev))
4315 igb_up(adapter);
4316 else
4317 igb_reset(adapter);
4318
4319 clear_bit(__IGB_RESETTING, &adapter->state);
4320
4321 return 0;
4322}
4323
4324/**
4325 * igb_update_stats - Update the board statistics counters
4326 * @adapter: board private structure
4327 **/
4328
12dcd86b
ED		 4329void igb_update_stats(struct igb_adapter *adapter,
4330 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4331{
4332 struct e1000_hw *hw = &adapter->hw;
4333 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4334 u32 reg, mpc;
9d5c8243 4335 u16 phy_tmp;
3f9c0164
AD		 4336 int i;
4337 u64 bytes, packets;
12dcd86b
ED		 4338 unsigned int start;
4339 u64 _bytes, _packets;
9d5c8243
AK		 4340
4341#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4342
4343 /*
4344 * Prevent stats update while adapter is being reset, or if the pci
4345 * connection is down.
4346 */
4347 if (adapter->link_speed == 0)
4348 return;
4349 if (pci_channel_offline(pdev))
4350 return;
4351
3f9c0164
AD		 4352 bytes = 0;
4353 packets = 0;
4354 for (i = 0; i < adapter->num_rx_queues; i++) {
4355 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4356 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4357
3025a446 4358 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4359 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4360
4361 do {
4362 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4363 _bytes = ring->rx_stats.bytes;
4364 _packets = ring->rx_stats.packets;
4365 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4366 bytes += _bytes;
4367 packets += _packets;
3f9c0164
AD		 4368 }
4369
128e45eb
AD		 4370 net_stats->rx_bytes = bytes;
4371 net_stats->rx_packets = packets;
3f9c0164
AD		 4372
4373 bytes = 0;
4374 packets = 0;
4375 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4376 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4377 do {
4378 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4379 _bytes = ring->tx_stats.bytes;
4380 _packets = ring->tx_stats.packets;
4381 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4382 bytes += _bytes;
4383 packets += _packets;
3f9c0164 4384 }
128e45eb
AD		 4385 net_stats->tx_bytes = bytes;
4386 net_stats->tx_packets = packets;
3f9c0164
AD		 4387
4388 /* read stats registers */
9d5c8243
AK		 4389 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4390 adapter->stats.gprc += rd32(E1000_GPRC);
4391 adapter->stats.gorc += rd32(E1000_GORCL);
4392 rd32(E1000_GORCH); /* clear GORCL */
4393 adapter->stats.bprc += rd32(E1000_BPRC);
4394 adapter->stats.mprc += rd32(E1000_MPRC);
4395 adapter->stats.roc += rd32(E1000_ROC);
4396
4397 adapter->stats.prc64 += rd32(E1000_PRC64);
4398 adapter->stats.prc127 += rd32(E1000_PRC127);
4399 adapter->stats.prc255 += rd32(E1000_PRC255);
4400 adapter->stats.prc511 += rd32(E1000_PRC511);
4401 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4402 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4403 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4404 adapter->stats.sec += rd32(E1000_SEC);
4405
fa3d9a6d
MW		 4406 mpc = rd32(E1000_MPC);
4407 adapter->stats.mpc += mpc;
4408 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4409 adapter->stats.scc += rd32(E1000_SCC);
4410 adapter->stats.ecol += rd32(E1000_ECOL);
4411 adapter->stats.mcc += rd32(E1000_MCC);
4412 adapter->stats.latecol += rd32(E1000_LATECOL);
4413 adapter->stats.dc += rd32(E1000_DC);
4414 adapter->stats.rlec += rd32(E1000_RLEC);
4415 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4416 adapter->stats.xontxc += rd32(E1000_XONTXC);
4417 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4418 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4419 adapter->stats.fcruc += rd32(E1000_FCRUC);
4420 adapter->stats.gptc += rd32(E1000_GPTC);
4421 adapter->stats.gotc += rd32(E1000_GOTCL);
4422 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4423 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4424 adapter->stats.ruc += rd32(E1000_RUC);
4425 adapter->stats.rfc += rd32(E1000_RFC);
4426 adapter->stats.rjc += rd32(E1000_RJC);
4427 adapter->stats.tor += rd32(E1000_TORH);
4428 adapter->stats.tot += rd32(E1000_TOTH);
4429 adapter->stats.tpr += rd32(E1000_TPR);
4430
4431 adapter->stats.ptc64 += rd32(E1000_PTC64);
4432 adapter->stats.ptc127 += rd32(E1000_PTC127);
4433 adapter->stats.ptc255 += rd32(E1000_PTC255);
4434 adapter->stats.ptc511 += rd32(E1000_PTC511);
4435 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4436 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4437
4438 adapter->stats.mptc += rd32(E1000_MPTC);
4439 adapter->stats.bptc += rd32(E1000_BPTC);
4440
2d0b0f69
NN		 4441 adapter->stats.tpt += rd32(E1000_TPT);
4442 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4443
4444 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4445 /* read internal phy specific stats */
4446 reg = rd32(E1000_CTRL_EXT);
4447 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4448 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4449 adapter->stats.tncrs += rd32(E1000_TNCRS);
4450 }
4451
9d5c8243
AK		 4452 adapter->stats.tsctc += rd32(E1000_TSCTC);
4453 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4454
4455 adapter->stats.iac += rd32(E1000_IAC);
4456 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4457 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4458 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4459 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4460 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4461 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4462 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4463 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4464
4465 /* Fill out the OS statistics structure */
128e45eb
AD		 4466 net_stats->multicast = adapter->stats.mprc;
4467 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4468
4469 /* Rx Errors */
4470
4471 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4472 * our own version based on RUC and ROC */
128e45eb 4473 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4474 adapter->stats.crcerrs + adapter->stats.algnerrc +
4475 adapter->stats.ruc + adapter->stats.roc +
4476 adapter->stats.cexterr;
128e45eb
AD		 4477 net_stats->rx_length_errors = adapter->stats.ruc +
4478 adapter->stats.roc;
4479 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4480 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4481 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4482
4483 /* Tx Errors */
128e45eb
AD		 4484 net_stats->tx_errors = adapter->stats.ecol +
4485 adapter->stats.latecol;
4486 net_stats->tx_aborted_errors = adapter->stats.ecol;
4487 net_stats->tx_window_errors = adapter->stats.latecol;
4488 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4489
4490 /* Tx Dropped needs to be maintained elsewhere */
4491
4492 /* Phy Stats */
4493 if (hw->phy.media_type == e1000_media_type_copper) {
4494 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4495 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4496 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4497 adapter->phy_stats.idle_errors += phy_tmp;
4498 }
4499 }
4500
4501 /* Management Stats */
4502 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4503 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4504 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4505}
4506
9d5c8243
AK		 4507static irqreturn_t igb_msix_other(int irq, void *data)
4508{
047e0030 4509 struct igb_adapter *adapter = data;
9d5c8243 4510 struct e1000_hw *hw = &adapter->hw;
844290e5 4511 u32 icr = rd32(E1000_ICR);
844290e5 4512 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4513
7f081d40
AD		 4514 if (icr & E1000_ICR_DRSTA)
4515 schedule_work(&adapter->reset_task);
4516
047e0030 4517 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4518 /* HW is reporting DMA is out of sync */
4519 adapter->stats.doosync++;
4520 }
eebbbdba 4521
4ae196df
AD		 4522 /* Check for a mailbox event */
4523 if (icr & E1000_ICR_VMMB)
4524 igb_msg_task(adapter);
4525
4526 if (icr & E1000_ICR_LSC) {
4527 hw->mac.get_link_status = 1;
4528 /* guard against interrupt when we're going down */
4529 if (!test_bit(__IGB_DOWN, &adapter->state))
4530 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4531 }
4532
25568a53
AD		 4533 if (adapter->vfs_allocated_count)
4534 wr32(E1000_IMS, E1000_IMS_LSC |
4535 E1000_IMS_VMMB |
4536 E1000_IMS_DOUTSYNC);
4537 else
4538 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4539 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4540
4541 return IRQ_HANDLED;
4542}
4543
047e0030 4544static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4545{
26b39276 4546 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4547 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4548
047e0030
AD		 4549 if (!q_vector->set_itr)
4550 return;
73cd78f1 4551
047e0030
AD		 4552 if (!itr_val)
4553 itr_val = 0x4;
661086df 4554
26b39276
AD		 4555 if (adapter->hw.mac.type == e1000_82575)
4556 itr_val |= itr_val << 16;
661086df 4557 else
047e0030 4558 itr_val |= 0x8000000;
661086df 4559
047e0030
AD		 4560 writel(itr_val, q_vector->itr_register);
4561 q_vector->set_itr = 0;
6eb5a7f1
AD		 4562}
4563
047e0030 4564static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4565{
047e0030 4566 struct igb_q_vector *q_vector = data;
9d5c8243 4567
047e0030
AD		 4568 /* Write the ITR value calculated from the previous interrupt. */
4569 igb_write_itr(q_vector);
9d5c8243 4570
047e0030 4571 napi_schedule(&q_vector->napi);
844290e5 4572
047e0030 4573 return IRQ_HANDLED;
fe4506b6
JC		 4574}
4575
421e02f0 4576#ifdef CONFIG_IGB_DCA
047e0030 4577static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4578{
047e0030 4579 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4580 struct e1000_hw *hw = &adapter->hw;
4581 int cpu = get_cpu();
fe4506b6 4582
047e0030
AD		 4583 if (q_vector->cpu == cpu)
4584 goto out_no_update;
4585
4586 if (q_vector->tx_ring) {
4587 int q = q_vector->tx_ring->reg_idx;
4588 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4589 if (hw->mac.type == e1000_82575) {
4590 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4591 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4592 } else {
047e0030
AD		 4593 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4594 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4595 E1000_DCA_TXCTRL_CPUID_SHIFT;
4596 }
4597 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4598 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4599 }
4600 if (q_vector->rx_ring) {
4601 int q = q_vector->rx_ring->reg_idx;
4602 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4603 if (hw->mac.type == e1000_82575) {
2d064c06 4604 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4605 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4606 } else {
4607 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4608 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4609 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4610 }
fe4506b6
JC		 4611 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4612 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4613 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4614 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4615 }
047e0030
AD		 4616 q_vector->cpu = cpu;
4617out_no_update:
fe4506b6
JC		 4618 put_cpu();
4619}
4620
4621static void igb_setup_dca(struct igb_adapter *adapter)
4622{
7e0e99ef 4623 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4624 int i;
4625
7dfc16fa 4626 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4627 return;
4628
7e0e99ef
AD		 4629 /* Always use CB2 mode, difference is masked in the CB driver. */
4630 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4631
047e0030 4632 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4633 adapter->q_vector[i]->cpu = -1;
4634 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4635 }
4636}
4637
4638static int __igb_notify_dca(struct device *dev, void *data)
4639{
4640 struct net_device *netdev = dev_get_drvdata(dev);
4641 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4642 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4643 struct e1000_hw *hw = &adapter->hw;
4644 unsigned long event = *(unsigned long *)data;
4645
4646 switch (event) {
4647 case DCA_PROVIDER_ADD:
4648 /* if already enabled, don't do it again */
7dfc16fa 4649 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4650 break;
fe4506b6 4651 if (dca_add_requester(dev) == 0) {
bbd98fe4 4652 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4653 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4654 igb_setup_dca(adapter);
4655 break;
4656 }
4657 /* Fall Through since DCA is disabled. */
4658 case DCA_PROVIDER_REMOVE:
7dfc16fa 4659 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4660 /* without this a class_device is left
047e0030 4661 * hanging around in the sysfs model */
fe4506b6 4662 dca_remove_requester(dev);
090b1795 4663 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4664 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4665 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4666 }
4667 break;
4668 }
bbd98fe4 4669
fe4506b6 4670 return 0;
9d5c8243
AK		 4671}
4672
fe4506b6
JC		 4673static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4674 void *p)
4675{
4676 int ret_val;
4677
4678 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4679 __igb_notify_dca);
4680
4681 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4682}
421e02f0 4683#endif /* CONFIG_IGB_DCA */
9d5c8243 4684
4ae196df
AD		 4685static void igb_ping_all_vfs(struct igb_adapter *adapter)
4686{
4687 struct e1000_hw *hw = &adapter->hw;
4688 u32 ping;
4689 int i;
4690
4691 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4692 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4693 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4694 ping |= E1000_VT_MSGTYPE_CTS;
4695 igb_write_mbx(hw, &ping, 1, i);
4696 }
4697}
4698
7d5753f0
AD		 4699static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4700{
4701 struct e1000_hw *hw = &adapter->hw;
4702 u32 vmolr = rd32(E1000_VMOLR(vf));
4703 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4704
d85b9004 4705 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4706 IGB_VF_FLAG_MULTI_PROMISC);
4707 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4708
4709 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4710 vmolr |= E1000_VMOLR_MPME;
d85b9004 4711 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4712 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4713 } else {
4714 /*
4715 * if we have hashes and we are clearing a multicast promisc
4716 * flag we need to write the hashes to the MTA as this step
4717 * was previously skipped
4718 */
4719 if (vf_data->num_vf_mc_hashes > 30) {
4720 vmolr |= E1000_VMOLR_MPME;
4721 } else if (vf_data->num_vf_mc_hashes) {
4722 int j;
4723 vmolr |= E1000_VMOLR_ROMPE;
4724 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4725 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4726 }
4727 }
4728
4729 wr32(E1000_VMOLR(vf), vmolr);
4730
4731 /* there are flags left unprocessed, likely not supported */
4732 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4733 return -EINVAL;
4734
4735 return 0;
4736
4737}
4738
4ae196df
AD		 4739static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4740 u32 *msgbuf, u32 vf)
4741{
4742 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4743 u16 *hash_list = (u16 *)&msgbuf[1];
4744 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4745 int i;
4746
7d5753f0 4747 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4748 * to this VF for later use to restore when the PF multi cast
4749 * list changes
4750 */
4751 vf_data->num_vf_mc_hashes = n;
4752
7d5753f0
AD		 4753 /* only up to 30 hash values supported */
4754 if (n > 30)
4755 n = 30;
4756
4757 /* store the hashes for later use */
4ae196df 4758 for (i = 0; i < n; i++)
a419aef8 4759 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4760
4761 /* Flush and reset the mta with the new values */
ff41f8dc 4762 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4763
4764 return 0;
4765}
4766
4767static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4768{
4769 struct e1000_hw *hw = &adapter->hw;
4770 struct vf_data_storage *vf_data;
4771 int i, j;
4772
4773 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4774 u32 vmolr = rd32(E1000_VMOLR(i));
4775 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4776
4ae196df 4777 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4778
4779 if ((vf_data->num_vf_mc_hashes > 30) ||
4780 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4781 vmolr |= E1000_VMOLR_MPME;
4782 } else if (vf_data->num_vf_mc_hashes) {
4783 vmolr |= E1000_VMOLR_ROMPE;
4784 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4785 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4786 }
4787 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4788 }
4789}
4790
4791static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4792{
4793 struct e1000_hw *hw = &adapter->hw;
4794 u32 pool_mask, reg, vid;
4795 int i;
4796
4797 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4798
4799 /* Find the vlan filter for this id */
4800 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4801 reg = rd32(E1000_VLVF(i));
4802
4803 /* remove the vf from the pool */
4804 reg &= ~pool_mask;
4805
4806 /* if pool is empty then remove entry from vfta */
4807 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4808 (reg & E1000_VLVF_VLANID_ENABLE)) {
4809 reg = 0;
4810 vid = reg & E1000_VLVF_VLANID_MASK;
4811 igb_vfta_set(hw, vid, false);
4812 }
4813
4814 wr32(E1000_VLVF(i), reg);
4815 }
ae641bdc
AD		 4816
4817 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4818}
4819
4820static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4821{
4822 struct e1000_hw *hw = &adapter->hw;
4823 u32 reg, i;
4824
51466239
AD		 4825 /* The vlvf table only exists on 82576 hardware and newer */
4826 if (hw->mac.type < e1000_82576)
4827 return -1;
4828
4829 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4830 if (!adapter->vfs_allocated_count)
4831 return -1;
4832
4833 /* Find the vlan filter for this id */
4834 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4835 reg = rd32(E1000_VLVF(i));
4836 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4837 vid == (reg & E1000_VLVF_VLANID_MASK))
4838 break;
4839 }
4840
4841 if (add) {
4842 if (i == E1000_VLVF_ARRAY_SIZE) {
4843 /* Did not find a matching VLAN ID entry that was
4844 * enabled. Search for a free filter entry, i.e.
4845 * one without the enable bit set
4846 */
4847 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4848 reg = rd32(E1000_VLVF(i));
4849 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4850 break;
4851 }
4852 }
4853 if (i < E1000_VLVF_ARRAY_SIZE) {
4854 /* Found an enabled/available entry */
4855 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4856
4857 /* if !enabled we need to set this up in vfta */
4858 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4859 /* add VID to filter table */
4860 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4861 reg |= E1000_VLVF_VLANID_ENABLE;
4862 }
cad6d05f
AD		 4863 reg &= ~E1000_VLVF_VLANID_MASK;
4864 reg |= vid;
4ae196df 4865 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4866
4867 /* do not modify RLPML for PF devices */
4868 if (vf >= adapter->vfs_allocated_count)
4869 return 0;
4870
4871 if (!adapter->vf_data[vf].vlans_enabled) {
4872 u32 size;
4873 reg = rd32(E1000_VMOLR(vf));
4874 size = reg & E1000_VMOLR_RLPML_MASK;
4875 size += 4;
4876 reg &= ~E1000_VMOLR_RLPML_MASK;
4877 reg |= size;
4878 wr32(E1000_VMOLR(vf), reg);
4879 }
ae641bdc 4880
51466239 4881 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4882 return 0;
4883 }
4884 } else {
4885 if (i < E1000_VLVF_ARRAY_SIZE) {
4886 /* remove vf from the pool */
4887 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4888 /* if pool is empty then remove entry from vfta */
4889 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4890 reg = 0;
4891 igb_vfta_set(hw, vid, false);
4892 }
4893 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4894
4895 /* do not modify RLPML for PF devices */
4896 if (vf >= adapter->vfs_allocated_count)
4897 return 0;
4898
4899 adapter->vf_data[vf].vlans_enabled--;
4900 if (!adapter->vf_data[vf].vlans_enabled) {
4901 u32 size;
4902 reg = rd32(E1000_VMOLR(vf));
4903 size = reg & E1000_VMOLR_RLPML_MASK;
4904 size -= 4;
4905 reg &= ~E1000_VMOLR_RLPML_MASK;
4906 reg |= size;
4907 wr32(E1000_VMOLR(vf), reg);
4908 }
4ae196df
AD		 4909 }
4910 }
8151d294
WM		 4911 return 0;
4912}
4913
4914static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4915{
4916 struct e1000_hw *hw = &adapter->hw;
4917
4918 if (vid)
4919 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4920 else
4921 wr32(E1000_VMVIR(vf), 0);
4922}
4923
4924static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4925 int vf, u16 vlan, u8 qos)
4926{
4927 int err = 0;
4928 struct igb_adapter *adapter = netdev_priv(netdev);
4929
4930 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4931 return -EINVAL;
4932 if (vlan || qos) {
4933 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4934 if (err)
4935 goto out;
4936 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4937 igb_set_vmolr(adapter, vf, !vlan);
4938 adapter->vf_data[vf].pf_vlan = vlan;
4939 adapter->vf_data[vf].pf_qos = qos;
4940 dev_info(&adapter->pdev->dev,
4941 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4942 if (test_bit(__IGB_DOWN, &adapter->state)) {
4943 dev_warn(&adapter->pdev->dev,
4944 "The VF VLAN has been set,"
4945 " but the PF device is not up.\n");
4946 dev_warn(&adapter->pdev->dev,
4947 "Bring the PF device up before"
4948 " attempting to use the VF device.\n");
4949 }
4950 } else {
4951 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4952 false, vf);
4953 igb_set_vmvir(adapter, vlan, vf);
4954 igb_set_vmolr(adapter, vf, true);
4955 adapter->vf_data[vf].pf_vlan = 0;
4956 adapter->vf_data[vf].pf_qos = 0;
4957 }
4958out:
4959 return err;
4ae196df
AD		 4960}
4961
4962static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4963{
4964 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4965 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4966
4967 return igb_vlvf_set(adapter, vid, add, vf);
4968}
4969
f2ca0dbe 4970static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4971{
8151d294
WM		 4972 /* clear flags */
4973 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4974 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4975
4976 /* reset offloads to defaults */
8151d294 4977 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4978
4979 /* reset vlans for device */
4980 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4981 if (adapter->vf_data[vf].pf_vlan)
4982 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4983 adapter->vf_data[vf].pf_vlan,
4984 adapter->vf_data[vf].pf_qos);
4985 else
4986 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4987
4988 /* reset multicast table array for vf */
4989 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4990
4991 /* Flush and reset the mta with the new values */
ff41f8dc 4992 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4993}
4994
f2ca0dbe
AD		 4995static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4996{
4997 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4998
4999 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5000 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5001 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5002
5003 /* process remaining reset events */
5004 igb_vf_reset(adapter, vf);
5005}
5006
5007static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5008{
5009 struct e1000_hw *hw = &adapter->hw;
5010 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5011 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5012 u32 reg, msgbuf[3];
5013 u8 *addr = (u8 *)(&msgbuf[1]);
5014
5015 /* process all the same items cleared in a function level reset */
f2ca0dbe 5016 igb_vf_reset(adapter, vf);
4ae196df
AD		 5017
5018 /* set vf mac address */
26ad9178 5019 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5020
5021 /* enable transmit and receive for vf */
5022 reg = rd32(E1000_VFTE);
5023 wr32(E1000_VFTE, reg | (1 << vf));
5024 reg = rd32(E1000_VFRE);
5025 wr32(E1000_VFRE, reg | (1 << vf));
5026
f2ca0dbe 5027 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 5028
5029 /* reply to reset with ack and vf mac address */
5030 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5031 memcpy(addr, vf_mac, 6);
5032 igb_write_mbx(hw, msgbuf, 3, vf);
5033}
5034
5035static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5036{
de42edde
GR		 5037 /*
5038 * The VF MAC Address is stored in a packed array of bytes
5039 * starting at the second 32 bit word of the msg array
5040 */
f2ca0dbe
AD		 5041 unsigned char *addr = (char *)&msg[1];
5042 int err = -1;
4ae196df 5043
f2ca0dbe
AD		 5044 if (is_valid_ether_addr(addr))
5045 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5046
f2ca0dbe 5047 return err;
4ae196df
AD		 5048}
5049
5050static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5051{
5052 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5053 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5054 u32 msg = E1000_VT_MSGTYPE_NACK;
5055
5056 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5057 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5058 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5059 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5060 vf_data->last_nack = jiffies;
4ae196df
AD		 5061 }
5062}
5063
f2ca0dbe 5064static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5065{
f2ca0dbe
AD		 5066 struct pci_dev *pdev = adapter->pdev;
5067 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5068 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5069 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5070 s32 retval;
5071
f2ca0dbe 5072 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5073
fef45f4c
AD		 5074 if (retval) {
5075 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5076 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5077 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5078 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5079 return;
5080 goto out;
5081 }
4ae196df
AD		 5082
5083 /* this is a message we already processed, do nothing */
5084 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5085 return;
4ae196df
AD		 5086
5087 /*
5088 * until the vf completes a reset it should not be
5089 * allowed to start any configuration.
5090 */
5091
5092 if (msgbuf[0] == E1000_VF_RESET) {
5093 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5094 return;
4ae196df
AD		 5095 }
5096
f2ca0dbe 5097 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5098 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5099 return;
5100 retval = -1;
5101 goto out;
4ae196df
AD		 5102 }
5103
5104 switch ((msgbuf[0] & 0xFFFF)) {
5105 case E1000_VF_SET_MAC_ADDR:
5106 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5107 break;
7d5753f0
AD		 5108 case E1000_VF_SET_PROMISC:
5109 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5110 break;
4ae196df
AD		 5111 case E1000_VF_SET_MULTICAST:
5112 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5113 break;
5114 case E1000_VF_SET_LPE:
5115 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5116 break;
5117 case E1000_VF_SET_VLAN:
8151d294
WM		 5118 if (adapter->vf_data[vf].pf_vlan)
5119 retval = -1;
5120 else
5121 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5122 break;
5123 default:
090b1795 5124 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5125 retval = -1;
5126 break;
5127 }
5128
fef45f4c
AD		 5129 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5130out:
4ae196df
AD		 5131 /* notify the VF of the results of what it sent us */
5132 if (retval)
5133 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5134 else
5135 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5136
4ae196df 5137 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5138}
4ae196df 5139
f2ca0dbe
AD		 5140static void igb_msg_task(struct igb_adapter *adapter)
5141{
5142 struct e1000_hw *hw = &adapter->hw;
5143 u32 vf;
5144
5145 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5146 /* process any reset requests */
5147 if (!igb_check_for_rst(hw, vf))
5148 igb_vf_reset_event(adapter, vf);
5149
5150 /* process any messages pending */
5151 if (!igb_check_for_msg(hw, vf))
5152 igb_rcv_msg_from_vf(adapter, vf);
5153
5154 /* process any acks */
5155 if (!igb_check_for_ack(hw, vf))
5156 igb_rcv_ack_from_vf(adapter, vf);
5157 }
4ae196df
AD		 5158}
5159
68d480c4
AD		 5160/**
5161 * igb_set_uta - Set unicast filter table address
5162 * @adapter: board private structure
5163 *
5164 * The unicast table address is a register array of 32-bit registers.
5165 * The table is meant to be used in a way similar to how the MTA is used
5166 * however due to certain limitations in the hardware it is necessary to
5167 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5168 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5169 **/
5170static void igb_set_uta(struct igb_adapter *adapter)
5171{
5172 struct e1000_hw *hw = &adapter->hw;
5173 int i;
5174
5175 /* The UTA table only exists on 82576 hardware and newer */
5176 if (hw->mac.type < e1000_82576)
5177 return;
5178
5179 /* we only need to do this if VMDq is enabled */
5180 if (!adapter->vfs_allocated_count)
5181 return;
5182
5183 for (i = 0; i < hw->mac.uta_reg_count; i++)
5184 array_wr32(E1000_UTA, i, ~0);
5185}
5186
9d5c8243
AK		 5187/**
5188 * igb_intr_msi - Interrupt Handler
5189 * @irq: interrupt number
5190 * @data: pointer to a network interface device structure
5191 **/
5192static irqreturn_t igb_intr_msi(int irq, void *data)
5193{
047e0030
AD		 5194 struct igb_adapter *adapter = data;
5195 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5196 struct e1000_hw *hw = &adapter->hw;
5197 /* read ICR disables interrupts using IAM */
5198 u32 icr = rd32(E1000_ICR);
5199
047e0030 5200 igb_write_itr(q_vector);
9d5c8243 5201
7f081d40
AD		 5202 if (icr & E1000_ICR_DRSTA)
5203 schedule_work(&adapter->reset_task);
5204
047e0030 5205 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5206 /* HW is reporting DMA is out of sync */
5207 adapter->stats.doosync++;
5208 }
5209
9d5c8243
AK		 5210 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5211 hw->mac.get_link_status = 1;
5212 if (!test_bit(__IGB_DOWN, &adapter->state))
5213 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5214 }
5215
047e0030 5216 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5217
5218 return IRQ_HANDLED;
5219}
5220
5221/**
4a3c6433 5222 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5223 * @irq: interrupt number
5224 * @data: pointer to a network interface device structure
5225 **/
5226static irqreturn_t igb_intr(int irq, void *data)
5227{
047e0030
AD		 5228 struct igb_adapter *adapter = data;
5229 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5230 struct e1000_hw *hw = &adapter->hw;
5231 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5232 * need for the IMC write */
5233 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5234 if (!icr)
5235 return IRQ_NONE; /* Not our interrupt */
5236
047e0030 5237 igb_write_itr(q_vector);
9d5c8243
AK		 5238
5239 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5240 * not set, then the adapter didn't send an interrupt */
5241 if (!(icr & E1000_ICR_INT_ASSERTED))
5242 return IRQ_NONE;
5243
7f081d40
AD		 5244 if (icr & E1000_ICR_DRSTA)
5245 schedule_work(&adapter->reset_task);
5246
047e0030 5247 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5248 /* HW is reporting DMA is out of sync */
5249 adapter->stats.doosync++;
5250 }
5251
9d5c8243
AK		 5252 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5253 hw->mac.get_link_status = 1;
5254 /* guard against interrupt when we're going down */
5255 if (!test_bit(__IGB_DOWN, &adapter->state))
5256 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5257 }
5258
047e0030 5259 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5260
5261 return IRQ_HANDLED;
5262}
5263
047e0030 5264static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5265{
047e0030 5266 struct igb_adapter *adapter = q_vector->adapter;
46544258 5267 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5268
4fc82adf
AD		 5269 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5270 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5271 if (!adapter->msix_entries)
6eb5a7f1 5272 igb_set_itr(adapter);
46544258 5273 else
047e0030 5274 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5275 }
5276
46544258
AD		 5277 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5278 if (adapter->msix_entries)
047e0030 5279 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5280 else
5281 igb_irq_enable(adapter);
5282 }
9d5c8243
AK		 5283}
5284
46544258
AD		 5285/**
5286 * igb_poll - NAPI Rx polling callback
5287 * @napi: napi polling structure
5288 * @budget: count of how many packets we should handle
5289 **/
5290static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5291{
047e0030
AD		 5292 struct igb_q_vector *q_vector = container_of(napi,
5293 struct igb_q_vector,
5294 napi);
5295 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5296
421e02f0 5297#ifdef CONFIG_IGB_DCA
047e0030
AD		 5298 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5299 igb_update_dca(q_vector);
fe4506b6 5300#endif
047e0030
AD		 5301 if (q_vector->tx_ring)
5302 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5303
047e0030
AD		 5304 if (q_vector->rx_ring)
5305 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5306
5307 if (!tx_clean_complete)
5308 work_done = budget;
46544258 5309
9d5c8243 5310 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5311 if (work_done < budget) {
288379f0 5312 napi_complete(napi);
047e0030 5313 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5314 }
5315
46544258 5316 return work_done;
9d5c8243 5317}
6d8126f9 5318
33af6bcc 5319/**
c5b9bd5e 5320 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5321 * @adapter: board private structure
c5b9bd5e
AD		 5322 * @shhwtstamps: timestamp structure to update
5323 * @regval: unsigned 64bit system time value.
5324 *
5325 * We need to convert the system time value stored in the RX/TXSTMP registers
5326 * into a hwtstamp which can be used by the upper level timestamping functions
5327 */
5328static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5329 struct skb_shared_hwtstamps *shhwtstamps,
5330 u64 regval)
5331{
5332 u64 ns;
5333
55cac248
AD		 5334 /*
5335 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5336 * 24 to match clock shift we setup earlier.
5337 */
5338 if (adapter->hw.mac.type == e1000_82580)
5339 regval <<= IGB_82580_TSYNC_SHIFT;
5340
c5b9bd5e
AD		 5341 ns = timecounter_cyc2time(&adapter->clock, regval);
5342 timecompare_update(&adapter->compare, ns);
5343 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5344 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5345 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5346}
5347
5348/**
5349 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5350 * @q_vector: pointer to q_vector containing needed info
2873957d 5351 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5352 *
5353 * If we were asked to do hardware stamping and such a time stamp is
5354 * available, then it must have been for this skb here because we only
5355 * allow only one such packet into the queue.
5356 */
2873957d 5357static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5358{
c5b9bd5e 5359 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5360 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5361 struct skb_shared_hwtstamps shhwtstamps;
5362 u64 regval;
33af6bcc 5363
c5b9bd5e 5364 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5365 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5366 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5367 return;
5368
5369 regval = rd32(E1000_TXSTMPL);
5370 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5371
5372 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5373 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5374}
5375
9d5c8243
AK		 5376/**
5377 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5378 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5379 * returns true if ring is completely cleaned
5380 **/
047e0030 5381static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5382{
047e0030
AD		 5383 struct igb_adapter *adapter = q_vector->adapter;
5384 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5385 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5386 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5387 struct igb_buffer *buffer_info;
0e014cb1 5388 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5389 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5390 unsigned int i, eop, count = 0;
5391 bool cleaned = false;
9d5c8243 5392
9d5c8243 5393 i = tx_ring->next_to_clean;
0e014cb1
AD		 5394 eop = tx_ring->buffer_info[i].next_to_watch;
5395 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5396
5397 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5398 (count < tx_ring->count)) {
2d0bb1c1 5399 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5400 for (cleaned = false; !cleaned; count++) {
5401 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5402 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5403 cleaned = (i == eop);
9d5c8243 5404
2873957d
NN		 5405 if (buffer_info->skb) {
5406 total_bytes += buffer_info->bytecount;
9d5c8243 5407 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5408 total_packets += buffer_info->gso_segs;
5409 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5410 }
5411
80785298 5412 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5413 tx_desc->wb.status = 0;
9d5c8243
AK		 5414
5415 i++;
5416 if (i == tx_ring->count)
5417 i = 0;
9d5c8243 5418 }
0e014cb1
AD		 5419 eop = tx_ring->buffer_info[i].next_to_watch;
5420 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5421 }
5422
9d5c8243
AK		 5423 tx_ring->next_to_clean = i;
5424
fc7d345d 5425 if (unlikely(count &&
9d5c8243 5426 netif_carrier_ok(netdev) &&
c493ea45 5427 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5428 /* Make sure that anybody stopping the queue after this
5429 * sees the new next_to_clean.
5430 */
5431 smp_mb();
661086df
PWJ		 5432 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5433 !(test_bit(__IGB_DOWN, &adapter->state))) {
5434 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 5435
5436 u64_stats_update_begin(&tx_ring->tx_syncp);
04a5fcaa 5437 tx_ring->tx_stats.restart_queue++;
12dcd86b 5438 u64_stats_update_end(&tx_ring->tx_syncp);
661086df 5439 }
9d5c8243
AK		 5440 }
5441
5442 if (tx_ring->detect_tx_hung) {
5443 /* Detect a transmit hang in hardware, this serializes the
5444 * check with the clearing of time_stamp and movement of i */
5445 tx_ring->detect_tx_hung = false;
5446 if (tx_ring->buffer_info[i].time_stamp &&
5447 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5448 (adapter->tx_timeout_factor * HZ)) &&
5449 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5450
9d5c8243 5451 /* detected Tx unit hang */
59d71989 5452 dev_err(tx_ring->dev,
9d5c8243 5453 "Detected Tx Unit Hang\n"
2d064c06 5454 " Tx Queue <%d>\n"
9d5c8243
AK		 5455 " TDH <%x>\n"
5456 " TDT <%x>\n"
5457 " next_to_use <%x>\n"
5458 " next_to_clean <%x>\n"
9d5c8243
AK		 5459 "buffer_info[next_to_clean]\n"
5460 " time_stamp <%lx>\n"
0e014cb1 5461 " next_to_watch <%x>\n"
9d5c8243
AK		 5462 " jiffies <%lx>\n"
5463 " desc.status <%x>\n",
2d064c06 5464 tx_ring->queue_index,
fce99e34
AD		 5465 readl(tx_ring->head),
5466 readl(tx_ring->tail),
9d5c8243
AK		 5467 tx_ring->next_to_use,
5468 tx_ring->next_to_clean,
f7ba205e 5469 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5470 eop,
9d5c8243 5471 jiffies,
0e014cb1 5472 eop_desc->wb.status);
661086df 5473 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5474 }
5475 }
5476 tx_ring->total_bytes += total_bytes;
5477 tx_ring->total_packets += total_packets;
12dcd86b 5478 u64_stats_update_begin(&tx_ring->tx_syncp);
e21ed353
AD		 5479 tx_ring->tx_stats.bytes += total_bytes;
5480 tx_ring->tx_stats.packets += total_packets;
12dcd86b 5481 u64_stats_update_end(&tx_ring->tx_syncp);
807540ba 5482 return count < tx_ring->count;
9d5c8243
AK		 5483}
5484
9d5c8243
AK		 5485/**
5486 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5487 * @q_vector: structure containing interrupt and ring information
5488 * @skb: packet to send up
5489 * @vlan_tag: vlan tag for packet
9d5c8243 5490 **/
047e0030
AD		 5491static void igb_receive_skb(struct igb_q_vector *q_vector,
5492 struct sk_buff *skb,
5493 u16 vlan_tag)
5494{
5495 struct igb_adapter *adapter = q_vector->adapter;
5496
31b24b95 5497 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5498 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5499 vlan_tag, skb);
182ff8df 5500 else
047e0030 5501 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5502}
5503
04a5fcaa 5504static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5505 u32 status_err, struct sk_buff *skb)
5506{
bc8acf2c 5507 skb_checksum_none_assert(skb);
9d5c8243
AK		 5508
5509 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5510 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5511 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5512 return;
85ad76b2 5513
9d5c8243
AK		 5514 /* TCP/UDP checksum error bit is set */
5515 if (status_err &
5516 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5517 /*
5518 * work around errata with sctp packets where the TCPE aka
5519 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5520 * packets, (aka let the stack check the crc32c)
5521 */
85ad76b2 5522 if ((skb->len == 60) &&
12dcd86b
ED		 5523 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5524 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5525 ring->rx_stats.csum_err++;
12dcd86b
ED		 5526 u64_stats_update_end(&ring->rx_syncp);
5527 }
9d5c8243 5528 /* let the stack verify checksum errors */
9d5c8243
AK		 5529 return;
5530 }
5531 /* It must be a TCP or UDP packet with a valid checksum */
5532 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5533 skb->ip_summed = CHECKSUM_UNNECESSARY;
5534
59d71989 5535 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5536}
5537
757b77e2 5538static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5539 struct sk_buff *skb)
5540{
5541 struct igb_adapter *adapter = q_vector->adapter;
5542 struct e1000_hw *hw = &adapter->hw;
5543 u64 regval;
5544
5545 /*
5546 * If this bit is set, then the RX registers contain the time stamp. No
5547 * other packet will be time stamped until we read these registers, so
5548 * read the registers to make them available again. Because only one
5549 * packet can be time stamped at a time, we know that the register
5550 * values must belong to this one here and therefore we don't need to
5551 * compare any of the additional attributes stored for it.
5552 *
2244d07b 5553 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5554 * can turn into a skb_shared_hwtstamps.
5555 */
757b77e2
NN		 5556 if (staterr & E1000_RXDADV_STAT_TSIP) {
5557 u32 *stamp = (u32 *)skb->data;
5558 regval = le32_to_cpu(*(stamp + 2));
5559 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5560 skb_pull(skb, IGB_TS_HDR_LEN);
5561 } else {
5562 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5563 return;
c5b9bd5e 5564
757b77e2
NN		 5565 regval = rd32(E1000_RXSTMPL);
5566 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5567 }
c5b9bd5e
AD		 5568
5569 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5570}
4c844851 5571static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5572 union e1000_adv_rx_desc *rx_desc)
5573{
5574 /* HW will not DMA in data larger than the given buffer, even if it
5575 * parses the (NFS, of course) header to be larger. In that case, it
5576 * fills the header buffer and spills the rest into the page.
5577 */
5578 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5579 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5580 if (hlen > rx_ring->rx_buffer_len)
5581 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5582 return hlen;
5583}
5584
047e0030
AD		 5585static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5586 int *work_done, int budget)
9d5c8243 5587{
047e0030 5588 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5589 struct net_device *netdev = rx_ring->netdev;
59d71989 5590 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5591 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5592 struct igb_buffer *buffer_info , *next_buffer;
5593 struct sk_buff *skb;
9d5c8243
AK		 5594 bool cleaned = false;
5595 int cleaned_count = 0;
d1eff350 5596 int current_node = numa_node_id();
9d5c8243 5597 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5598 unsigned int i;
2d94d8ab
AD		 5599 u32 staterr;
5600 u16 length;
047e0030 5601 u16 vlan_tag;
9d5c8243
AK		 5602
5603 i = rx_ring->next_to_clean;
69d3ca53 5604 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5605 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5606 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5607
5608 while (staterr & E1000_RXD_STAT_DD) {
5609 if (*work_done >= budget)
5610 break;
5611 (*work_done)++;
2d0bb1c1 5612 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5613
69d3ca53
AD		 5614 skb = buffer_info->skb;
5615 prefetch(skb->data - NET_IP_ALIGN);
5616 buffer_info->skb = NULL;
5617
5618 i++;
5619 if (i == rx_ring->count)
5620 i = 0;
42d0781a 5621
69d3ca53
AD		 5622 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5623 prefetch(next_rxd);
5624 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5625
5626 length = le16_to_cpu(rx_desc->wb.upper.length);
5627 cleaned = true;
5628 cleaned_count++;
5629
2d94d8ab 5630 if (buffer_info->dma) {
59d71989 5631 dma_unmap_single(dev, buffer_info->dma,
4c844851 5632 rx_ring->rx_buffer_len,
59d71989 5633 DMA_FROM_DEVICE);
91615f76 5634 buffer_info->dma = 0;
4c844851 5635 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5636 skb_put(skb, length);
5637 goto send_up;
5638 }
4c844851 5639 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5640 }
5641
5642 if (length) {
59d71989
AD		 5643 dma_unmap_page(dev, buffer_info->page_dma,
5644 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5645 buffer_info->page_dma = 0;
bf36c1a0 5646
aa913403 5647 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5648 buffer_info->page,
5649 buffer_info->page_offset,
5650 length);
5651
d1eff350
AD		 5652 if ((page_count(buffer_info->page) != 1) ||
5653 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5654 buffer_info->page = NULL;
5655 else
5656 get_page(buffer_info->page);
9d5c8243
AK		 5657
5658 skb->len += length;
5659 skb->data_len += length;
bf36c1a0 5660 skb->truesize += length;
9d5c8243 5661 }
9d5c8243 5662
bf36c1a0 5663 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5664 buffer_info->skb = next_buffer->skb;
5665 buffer_info->dma = next_buffer->dma;
5666 next_buffer->skb = skb;
5667 next_buffer->dma = 0;
bf36c1a0
AD		 5668 goto next_desc;
5669 }
69d3ca53 5670send_up:
9d5c8243
AK		 5671 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5672 dev_kfree_skb_irq(skb);
5673 goto next_desc;
5674 }
9d5c8243 5675
757b77e2
NN		 5676 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5677 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5678 total_bytes += skb->len;
5679 total_packets++;
5680
85ad76b2 5681 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5682
5683 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5684 skb_record_rx_queue(skb, rx_ring->queue_index);
5685
5686 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5687 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5688
047e0030 5689 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5690
9d5c8243
AK		 5691next_desc:
5692 rx_desc->wb.upper.status_error = 0;
5693
5694 /* return some buffers to hardware, one at a time is too slow */
5695 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5696 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5697 cleaned_count = 0;
5698 }
5699
5700 /* use prefetched values */
5701 rx_desc = next_rxd;
5702 buffer_info = next_buffer;
9d5c8243
AK		 5703 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5704 }
bf36c1a0 5705
9d5c8243 5706 rx_ring->next_to_clean = i;
c493ea45 5707 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5708
5709 if (cleaned_count)
3b644cf6 5710 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5711
5712 rx_ring->total_packets += total_packets;
5713 rx_ring->total_bytes += total_bytes;
12dcd86b 5714 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5715 rx_ring->rx_stats.packets += total_packets;
5716 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5717 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5718 return cleaned;
5719}
5720
9d5c8243
AK		 5721/**
5722 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5723 * @adapter: address of board private structure
5724 **/
d7ee5b3a 5725void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5726{
e694e964 5727 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5728 union e1000_adv_rx_desc *rx_desc;
5729 struct igb_buffer *buffer_info;
5730 struct sk_buff *skb;
5731 unsigned int i;
db761762 5732 int bufsz;
9d5c8243
AK		 5733
5734 i = rx_ring->next_to_use;
5735 buffer_info = &rx_ring->buffer_info[i];
5736
4c844851 5737 bufsz = rx_ring->rx_buffer_len;
db761762 5738
9d5c8243
AK		 5739 while (cleaned_count--) {
5740 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5741
6ec43fe6 5742 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5743 if (!buffer_info->page) {
42d0781a 5744 buffer_info->page = netdev_alloc_page(netdev);
12dcd86b
ED		 5745 if (unlikely(!buffer_info->page)) {
5746 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5747 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5748 u64_stats_update_end(&rx_ring->rx_syncp);
bf36c1a0
AD		 5749 goto no_buffers;
5750 }
5751 buffer_info->page_offset = 0;
5752 } else {
5753 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5754 }
5755 buffer_info->page_dma =
59d71989 5756 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5757 buffer_info->page_offset,
5758 PAGE_SIZE / 2,
59d71989
AD		 5759 DMA_FROM_DEVICE);
5760 if (dma_mapping_error(rx_ring->dev,
5761 buffer_info->page_dma)) {
42d0781a 5762 buffer_info->page_dma = 0;
12dcd86b 5763 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5764 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5765 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5766 goto no_buffers;
5767 }
9d5c8243
AK		 5768 }
5769
42d0781a
AD		 5770 skb = buffer_info->skb;
5771 if (!skb) {
89d71a66 5772 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
12dcd86b
ED		 5773 if (unlikely(!skb)) {
5774 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5775 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5776 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5777 goto no_buffers;
5778 }
5779
9d5c8243 5780 buffer_info->skb = skb;
42d0781a
AD		 5781 }
5782 if (!buffer_info->dma) {
59d71989 5783 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5784 skb->data,
9d5c8243 5785 bufsz,
59d71989
AD		 5786 DMA_FROM_DEVICE);
5787 if (dma_mapping_error(rx_ring->dev,
5788 buffer_info->dma)) {
42d0781a 5789 buffer_info->dma = 0;
12dcd86b 5790 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5791 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5792 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5793 goto no_buffers;
5794 }
9d5c8243
AK		 5795 }
5796 /* Refresh the desc even if buffer_addrs didn't change because
5797 * each write-back erases this info. */
6ec43fe6 5798 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5799 rx_desc->read.pkt_addr =
5800 cpu_to_le64(buffer_info->page_dma);
5801 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5802 } else {
42d0781a 5803 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5804 rx_desc->read.hdr_addr = 0;
5805 }
5806
5807 i++;
5808 if (i == rx_ring->count)
5809 i = 0;
5810 buffer_info = &rx_ring->buffer_info[i];
5811 }
5812
5813no_buffers:
5814 if (rx_ring->next_to_use != i) {
5815 rx_ring->next_to_use = i;
5816 if (i == 0)
5817 i = (rx_ring->count - 1);
5818 else
5819 i--;
5820
5821 /* Force memory writes to complete before letting h/w
5822 * know there are new descriptors to fetch. (Only
5823 * applicable for weak-ordered memory model archs,
5824 * such as IA-64). */
5825 wmb();
fce99e34 5826 writel(i, rx_ring->tail);
9d5c8243
AK		 5827 }
5828}
5829
5830/**
5831 * igb_mii_ioctl -
5832 * @netdev:
5833 * @ifreq:
5834 * @cmd:
5835 **/
5836static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5837{
5838 struct igb_adapter *adapter = netdev_priv(netdev);
5839 struct mii_ioctl_data *data = if_mii(ifr);
5840
5841 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5842 return -EOPNOTSUPP;
5843
5844 switch (cmd) {
5845 case SIOCGMIIPHY:
5846 data->phy_id = adapter->hw.phy.addr;
5847 break;
5848 case SIOCGMIIREG:
f5f4cf08
AD		 5849 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5850 &data->val_out))
9d5c8243
AK		 5851 return -EIO;
5852 break;
5853 case SIOCSMIIREG:
5854 default:
5855 return -EOPNOTSUPP;
5856 }
5857 return 0;
5858}
5859
c6cb090b
PO		 5860/**
5861 * igb_hwtstamp_ioctl - control hardware time stamping
5862 * @netdev:
5863 * @ifreq:
5864 * @cmd:
5865 *
33af6bcc
PO		 5866 * Outgoing time stamping can be enabled and disabled. Play nice and
5867 * disable it when requested, although it shouldn't case any overhead
5868 * when no packet needs it. At most one packet in the queue may be
5869 * marked for time stamping, otherwise it would be impossible to tell
5870 * for sure to which packet the hardware time stamp belongs.
5871 *
5872 * Incoming time stamping has to be configured via the hardware
5873 * filters. Not all combinations are supported, in particular event
5874 * type has to be specified. Matching the kind of event packet is
5875 * not supported, with the exception of "all V2 events regardless of
5876 * level 2 or 4".
5877 *
c6cb090b
PO		 5878 **/
5879static int igb_hwtstamp_ioctl(struct net_device *netdev,
5880 struct ifreq *ifr, int cmd)
5881{
33af6bcc
PO		 5882 struct igb_adapter *adapter = netdev_priv(netdev);
5883 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5884 struct hwtstamp_config config;
c5b9bd5e
AD		 5885 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5886 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5887 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5888 bool is_l4 = false;
5889 bool is_l2 = false;
33af6bcc 5890 u32 regval;
c6cb090b
PO		 5891
5892 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5893 return -EFAULT;
5894
5895 /* reserved for future extensions */
5896 if (config.flags)
5897 return -EINVAL;
5898
33af6bcc
PO		 5899 switch (config.tx_type) {
5900 case HWTSTAMP_TX_OFF:
c5b9bd5e 5901 tsync_tx_ctl = 0;
33af6bcc 5902 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5903 break;
5904 default:
5905 return -ERANGE;
5906 }
5907
5908 switch (config.rx_filter) {
5909 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5910 tsync_rx_ctl = 0;
33af6bcc
PO		 5911 break;
5912 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5913 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5914 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5915 case HWTSTAMP_FILTER_ALL:
5916 /*
5917 * register TSYNCRXCFG must be set, therefore it is not
5918 * possible to time stamp both Sync and Delay_Req messages
5919 * => fall back to time stamping all packets
5920 */
c5b9bd5e 5921 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5922 config.rx_filter = HWTSTAMP_FILTER_ALL;
5923 break;
5924 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5925 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5926 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5927 is_l4 = true;
33af6bcc
PO		 5928 break;
5929 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5930 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5931 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5932 is_l4 = true;
33af6bcc
PO		 5933 break;
5934 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5935 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5936 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5937 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5938 is_l2 = true;
5939 is_l4 = true;
33af6bcc
PO		 5940 config.rx_filter = HWTSTAMP_FILTER_SOME;
5941 break;
5942 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5943 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5944 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5945 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5946 is_l2 = true;
5947 is_l4 = true;
33af6bcc
PO		 5948 config.rx_filter = HWTSTAMP_FILTER_SOME;
5949 break;
5950 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5951 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5952 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5953 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5954 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5955 is_l2 = true;
33af6bcc
PO		 5956 break;
5957 default:
5958 return -ERANGE;
5959 }
5960
c5b9bd5e
AD		 5961 if (hw->mac.type == e1000_82575) {
5962 if (tsync_rx_ctl | tsync_tx_ctl)
5963 return -EINVAL;
5964 return 0;
5965 }
5966
757b77e2
NN		 5967 /*
5968 * Per-packet timestamping only works if all packets are
5969 * timestamped, so enable timestamping in all packets as
5970 * long as one rx filter was configured.
5971 */
5972 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5973 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5974 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5975 }
5976
33af6bcc
PO		 5977 /* enable/disable TX */
5978 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5979 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5980 regval |= tsync_tx_ctl;
33af6bcc
PO		 5981 wr32(E1000_TSYNCTXCTL, regval);
5982
c5b9bd5e 5983 /* enable/disable RX */
33af6bcc 5984 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5985 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5986 regval |= tsync_rx_ctl;
33af6bcc 5987 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5988
c5b9bd5e
AD		 5989 /* define which PTP packets are time stamped */
5990 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5991
c5b9bd5e
AD		 5992 /* define ethertype filter for timestamped packets */
5993 if (is_l2)
5994 wr32(E1000_ETQF(3),
5995 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5996 E1000_ETQF_1588 | /* enable timestamping */
5997 ETH_P_1588)); /* 1588 eth protocol type */
5998 else
5999 wr32(E1000_ETQF(3), 0);
6000
6001#define PTP_PORT 319
6002 /* L4 Queue Filter[3]: filter by destination port and protocol */
6003 if (is_l4) {
6004 u32 ftqf = (IPPROTO_UDP /* UDP */
6005 | E1000_FTQF_VF_BP /* VF not compared */
6006 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6007 | E1000_FTQF_MASK); /* mask all inputs */
6008 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6009
6010 wr32(E1000_IMIR(3), htons(PTP_PORT));
6011 wr32(E1000_IMIREXT(3),
6012 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6013 if (hw->mac.type == e1000_82576) {
6014 /* enable source port check */
6015 wr32(E1000_SPQF(3), htons(PTP_PORT));
6016 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6017 }
6018 wr32(E1000_FTQF(3), ftqf);
6019 } else {
6020 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6021 }
33af6bcc
PO		 6022 wrfl();
6023
6024 adapter->hwtstamp_config = config;
6025
6026 /* clear TX/RX time stamp registers, just to be sure */
6027 regval = rd32(E1000_TXSTMPH);
6028 regval = rd32(E1000_RXSTMPH);
c6cb090b 6029
33af6bcc
PO		 6030 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6031 -EFAULT : 0;
c6cb090b
PO		 6032}
6033
9d5c8243
AK		 6034/**
6035 * igb_ioctl -
6036 * @netdev:
6037 * @ifreq:
6038 * @cmd:
6039 **/
6040static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6041{
6042 switch (cmd) {
6043 case SIOCGMIIPHY:
6044 case SIOCGMIIREG:
6045 case SIOCSMIIREG:
6046 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6047 case SIOCSHWTSTAMP:
6048 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6049 default:
6050 return -EOPNOTSUPP;
6051 }
6052}
6053
009bc06e
AD		 6054s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6055{
6056 struct igb_adapter *adapter = hw->back;
6057 u16 cap_offset;
6058
6059 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6060 if (!cap_offset)
6061 return -E1000_ERR_CONFIG;
6062
6063 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6064
6065 return 0;
6066}
6067
6068s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6069{
6070 struct igb_adapter *adapter = hw->back;
6071 u16 cap_offset;
6072
6073 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6074 if (!cap_offset)
6075 return -E1000_ERR_CONFIG;
6076
6077 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6078
6079 return 0;
6080}
6081
9d5c8243
AK		 6082static void igb_vlan_rx_register(struct net_device *netdev,
6083 struct vlan_group *grp)
6084{
6085 struct igb_adapter *adapter = netdev_priv(netdev);
6086 struct e1000_hw *hw = &adapter->hw;
6087 u32 ctrl, rctl;
6088
6089 igb_irq_disable(adapter);
6090 adapter->vlgrp = grp;
6091
6092 if (grp) {
6093 /* enable VLAN tag insert/strip */
6094 ctrl = rd32(E1000_CTRL);
6095 ctrl |= E1000_CTRL_VME;
6096 wr32(E1000_CTRL, ctrl);
6097
51466239 6098 /* Disable CFI check */
9d5c8243 6099 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6100 rctl &= ~E1000_RCTL_CFIEN;
6101 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6102 } else {
6103 /* disable VLAN tag insert/strip */
6104 ctrl = rd32(E1000_CTRL);
6105 ctrl &= ~E1000_CTRL_VME;
6106 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6107 }
6108
e1739522
AD		 6109 igb_rlpml_set(adapter);
6110
9d5c8243
AK		 6111 if (!test_bit(__IGB_DOWN, &adapter->state))
6112 igb_irq_enable(adapter);
6113}
6114
6115static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6116{
6117 struct igb_adapter *adapter = netdev_priv(netdev);
6118 struct e1000_hw *hw = &adapter->hw;
4ae196df 6119 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6120
51466239
AD		 6121 /* attempt to add filter to vlvf array */
6122 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6123
51466239
AD		 6124 /* add the filter since PF can receive vlans w/o entry in vlvf */
6125 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6126}
6127
6128static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6129{
6130 struct igb_adapter *adapter = netdev_priv(netdev);
6131 struct e1000_hw *hw = &adapter->hw;
4ae196df 6132 int pf_id = adapter->vfs_allocated_count;
51466239 6133 s32 err;
9d5c8243
AK		 6134
6135 igb_irq_disable(adapter);
6136 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6137
6138 if (!test_bit(__IGB_DOWN, &adapter->state))
6139 igb_irq_enable(adapter);
6140
51466239
AD		 6141 /* remove vlan from VLVF table array */
6142 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6143
51466239
AD		 6144 /* if vid was not present in VLVF just remove it from table */
6145 if (err)
4ae196df 6146 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6147}
6148
6149static void igb_restore_vlan(struct igb_adapter *adapter)
6150{
6151 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6152
6153 if (adapter->vlgrp) {
6154 u16 vid;
b738127d 6155 for (vid = 0; vid < VLAN_N_VID; vid++) {
9d5c8243
AK		 6156 if (!vlan_group_get_device(adapter->vlgrp, vid))
6157 continue;
6158 igb_vlan_rx_add_vid(adapter->netdev, vid);
6159 }
6160 }
6161}
6162
6163int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6164{
090b1795 6165 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6166 struct e1000_mac_info *mac = &adapter->hw.mac;
6167
6168 mac->autoneg = 0;
6169
cd2638a8
CW		 6170 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6171 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6172 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6173 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6174 return -EINVAL;
6175 }
6176
9d5c8243
AK		 6177 switch (spddplx) {
6178 case SPEED_10 + DUPLEX_HALF:
6179 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6180 break;
6181 case SPEED_10 + DUPLEX_FULL:
6182 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6183 break;
6184 case SPEED_100 + DUPLEX_HALF:
6185 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6186 break;
6187 case SPEED_100 + DUPLEX_FULL:
6188 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6189 break;
6190 case SPEED_1000 + DUPLEX_FULL:
6191 mac->autoneg = 1;
6192 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6193 break;
6194 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6195 default:
090b1795 6196 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6197 return -EINVAL;
6198 }
6199 return 0;
6200}
6201
3fe7c4c9 6202static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6203{
6204 struct net_device *netdev = pci_get_drvdata(pdev);
6205 struct igb_adapter *adapter = netdev_priv(netdev);
6206 struct e1000_hw *hw = &adapter->hw;
2d064c06 6207 u32 ctrl, rctl, status;
9d5c8243
AK		 6208 u32 wufc = adapter->wol;
6209#ifdef CONFIG_PM
6210 int retval = 0;
6211#endif
6212
6213 netif_device_detach(netdev);
6214
a88f10ec
AD		 6215 if (netif_running(netdev))
6216 igb_close(netdev);
6217
047e0030 6218 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6219
6220#ifdef CONFIG_PM
6221 retval = pci_save_state(pdev);
6222 if (retval)
6223 return retval;
6224#endif
6225
6226 status = rd32(E1000_STATUS);
6227 if (status & E1000_STATUS_LU)
6228 wufc &= ~E1000_WUFC_LNKC;
6229
6230 if (wufc) {
6231 igb_setup_rctl(adapter);
ff41f8dc 6232 igb_set_rx_mode(netdev);
9d5c8243
AK		 6233
6234 /* turn on all-multi mode if wake on multicast is enabled */
6235 if (wufc & E1000_WUFC_MC) {
6236 rctl = rd32(E1000_RCTL);
6237 rctl |= E1000_RCTL_MPE;
6238 wr32(E1000_RCTL, rctl);
6239 }
6240
6241 ctrl = rd32(E1000_CTRL);
6242 /* advertise wake from D3Cold */
6243 #define E1000_CTRL_ADVD3WUC 0x00100000
6244 /* phy power management enable */
6245 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6246 ctrl |= E1000_CTRL_ADVD3WUC;
6247 wr32(E1000_CTRL, ctrl);
6248
9d5c8243 6249 /* Allow time for pending master requests to run */
330a6d6a 6250 igb_disable_pcie_master(hw);
9d5c8243
AK		 6251
6252 wr32(E1000_WUC, E1000_WUC_PME_EN);
6253 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6254 } else {
6255 wr32(E1000_WUC, 0);
6256 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6257 }
6258
3fe7c4c9
RW		 6259 *enable_wake = wufc || adapter->en_mng_pt;
6260 if (!*enable_wake)
88a268c1
NN		 6261 igb_power_down_link(adapter);
6262 else
6263 igb_power_up_link(adapter);
9d5c8243
AK		 6264
6265 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6266 * would have already happened in close and is redundant. */
6267 igb_release_hw_control(adapter);
6268
6269 pci_disable_device(pdev);
6270
9d5c8243
AK		 6271 return 0;
6272}
6273
6274#ifdef CONFIG_PM
3fe7c4c9
RW		 6275static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6276{
6277 int retval;
6278 bool wake;
6279
6280 retval = __igb_shutdown(pdev, &wake);
6281 if (retval)
6282 return retval;
6283
6284 if (wake) {
6285 pci_prepare_to_sleep(pdev);
6286 } else {
6287 pci_wake_from_d3(pdev, false);
6288 pci_set_power_state(pdev, PCI_D3hot);
6289 }
6290
6291 return 0;
6292}
6293
9d5c8243
AK		 6294static int igb_resume(struct pci_dev *pdev)
6295{
6296 struct net_device *netdev = pci_get_drvdata(pdev);
6297 struct igb_adapter *adapter = netdev_priv(netdev);
6298 struct e1000_hw *hw = &adapter->hw;
6299 u32 err;
6300
6301 pci_set_power_state(pdev, PCI_D0);
6302 pci_restore_state(pdev);
b94f2d77 6303 pci_save_state(pdev);
42bfd33a 6304
aed5dec3 6305 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6306 if (err) {
6307 dev_err(&pdev->dev,
6308 "igb: Cannot enable PCI device from suspend\n");
6309 return err;
6310 }
6311 pci_set_master(pdev);
6312
6313 pci_enable_wake(pdev, PCI_D3hot, 0);
6314 pci_enable_wake(pdev, PCI_D3cold, 0);
6315
047e0030 6316 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6317 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6318 return -ENOMEM;
9d5c8243
AK		 6319 }
6320
9d5c8243 6321 igb_reset(adapter);
a8564f03
AD		 6322
6323 /* let the f/w know that the h/w is now under the control of the
6324 * driver. */
6325 igb_get_hw_control(adapter);
6326
9d5c8243
AK		 6327 wr32(E1000_WUS, ~0);
6328
a88f10ec
AD		 6329 if (netif_running(netdev)) {
6330 err = igb_open(netdev);
6331 if (err)
6332 return err;
6333 }
9d5c8243
AK		 6334
6335 netif_device_attach(netdev);
6336
9d5c8243
AK		 6337 return 0;
6338}
6339#endif
6340
6341static void igb_shutdown(struct pci_dev *pdev)
6342{
3fe7c4c9
RW		 6343 bool wake;
6344
6345 __igb_shutdown(pdev, &wake);
6346
6347 if (system_state == SYSTEM_POWER_OFF) {
6348 pci_wake_from_d3(pdev, wake);
6349 pci_set_power_state(pdev, PCI_D3hot);
6350 }
9d5c8243
AK		 6351}
6352
6353#ifdef CONFIG_NET_POLL_CONTROLLER
6354/*
6355 * Polling 'interrupt' - used by things like netconsole to send skbs
6356 * without having to re-enable interrupts. It's not called while
6357 * the interrupt routine is executing.
6358 */
6359static void igb_netpoll(struct net_device *netdev)
6360{
6361 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6362 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6363 int i;
9d5c8243 6364
eebbbdba 6365 if (!adapter->msix_entries) {
047e0030 6366 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6367 igb_irq_disable(adapter);
047e0030 6368 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6369 return;
6370 }
9d5c8243 6371
047e0030
AD		 6372 for (i = 0; i < adapter->num_q_vectors; i++) {
6373 struct igb_q_vector *q_vector = adapter->q_vector[i];
6374 wr32(E1000_EIMC, q_vector->eims_value);
6375 napi_schedule(&q_vector->napi);
eebbbdba 6376 }
9d5c8243
AK		 6377}
6378#endif /* CONFIG_NET_POLL_CONTROLLER */
6379
6380/**
6381 * igb_io_error_detected - called when PCI error is detected
6382 * @pdev: Pointer to PCI device
6383 * @state: The current pci connection state
6384 *
6385 * This function is called after a PCI bus error affecting
6386 * this device has been detected.
6387 */
6388static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6389 pci_channel_state_t state)
6390{
6391 struct net_device *netdev = pci_get_drvdata(pdev);
6392 struct igb_adapter *adapter = netdev_priv(netdev);
6393
6394 netif_device_detach(netdev);
6395
59ed6eec
AD		 6396 if (state == pci_channel_io_perm_failure)
6397 return PCI_ERS_RESULT_DISCONNECT;
6398
9d5c8243
AK		 6399 if (netif_running(netdev))
6400 igb_down(adapter);
6401 pci_disable_device(pdev);
6402
6403 /* Request a slot slot reset. */
6404 return PCI_ERS_RESULT_NEED_RESET;
6405}
6406
6407/**
6408 * igb_io_slot_reset - called after the pci bus has been reset.
6409 * @pdev: Pointer to PCI device
6410 *
6411 * Restart the card from scratch, as if from a cold-boot. Implementation
6412 * resembles the first-half of the igb_resume routine.
6413 */
6414static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6415{
6416 struct net_device *netdev = pci_get_drvdata(pdev);
6417 struct igb_adapter *adapter = netdev_priv(netdev);
6418 struct e1000_hw *hw = &adapter->hw;
40a914fa 6419 pci_ers_result_t result;
42bfd33a 6420 int err;
9d5c8243 6421
aed5dec3 6422 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6423 dev_err(&pdev->dev,
6424 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6425 result = PCI_ERS_RESULT_DISCONNECT;
6426 } else {
6427 pci_set_master(pdev);
6428 pci_restore_state(pdev);
b94f2d77 6429 pci_save_state(pdev);
9d5c8243 6430
40a914fa
AD		 6431 pci_enable_wake(pdev, PCI_D3hot, 0);
6432 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6433
40a914fa
AD		 6434 igb_reset(adapter);
6435 wr32(E1000_WUS, ~0);
6436 result = PCI_ERS_RESULT_RECOVERED;
6437 }
9d5c8243 6438
ea943d41
JK		 6439 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6440 if (err) {
6441 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6442 "failed 0x%0x\n", err);
6443 /* non-fatal, continue */
6444 }
40a914fa
AD		 6445
6446 return result;
9d5c8243
AK		 6447}
6448
6449/**
6450 * igb_io_resume - called when traffic can start flowing again.
6451 * @pdev: Pointer to PCI device
6452 *
6453 * This callback is called when the error recovery driver tells us that
6454 * its OK to resume normal operation. Implementation resembles the
6455 * second-half of the igb_resume routine.
6456 */
6457static void igb_io_resume(struct pci_dev *pdev)
6458{
6459 struct net_device *netdev = pci_get_drvdata(pdev);
6460 struct igb_adapter *adapter = netdev_priv(netdev);
6461
9d5c8243
AK		 6462 if (netif_running(netdev)) {
6463 if (igb_up(adapter)) {
6464 dev_err(&pdev->dev, "igb_up failed after reset\n");
6465 return;
6466 }
6467 }
6468
6469 netif_device_attach(netdev);
6470
6471 /* let the f/w know that the h/w is now under the control of the
6472 * driver. */
6473 igb_get_hw_control(adapter);
9d5c8243
AK		 6474}
6475
26ad9178
AD		 6476static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6477 u8 qsel)
6478{
6479 u32 rar_low, rar_high;
6480 struct e1000_hw *hw = &adapter->hw;
6481
6482 /* HW expects these in little endian so we reverse the byte order
6483 * from network order (big endian) to little endian
6484 */
6485 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6486 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6487 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6488
6489 /* Indicate to hardware the Address is Valid. */
6490 rar_high |= E1000_RAH_AV;
6491
6492 if (hw->mac.type == e1000_82575)
6493 rar_high |= E1000_RAH_POOL_1 * qsel;
6494 else
6495 rar_high |= E1000_RAH_POOL_1 << qsel;
6496
6497 wr32(E1000_RAL(index), rar_low);
6498 wrfl();
6499 wr32(E1000_RAH(index), rar_high);
6500 wrfl();
6501}
6502
4ae196df
AD		 6503static int igb_set_vf_mac(struct igb_adapter *adapter,
6504 int vf, unsigned char *mac_addr)
6505{
6506 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6507 /* VF MAC addresses start at end of receive addresses and moves
6508 * torwards the first, as a result a collision should not be possible */
6509 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6510
37680117 6511 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6512
26ad9178 6513 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6514
6515 return 0;
6516}
6517
8151d294
WM		 6518static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6519{
6520 struct igb_adapter *adapter = netdev_priv(netdev);
6521 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6522 return -EINVAL;
6523 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6524 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6525 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6526 " change effective.");
6527 if (test_bit(__IGB_DOWN, &adapter->state)) {
6528 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6529 " but the PF device is not up.\n");
6530 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6531 " attempting to use the VF device.\n");
6532 }
6533 return igb_set_vf_mac(adapter, vf, mac);
6534}
6535
6536static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6537{
6538 return -EOPNOTSUPP;
6539}
6540
6541static int igb_ndo_get_vf_config(struct net_device *netdev,
6542 int vf, struct ifla_vf_info *ivi)
6543{
6544 struct igb_adapter *adapter = netdev_priv(netdev);
6545 if (vf >= adapter->vfs_allocated_count)
6546 return -EINVAL;
6547 ivi->vf = vf;
6548 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6549 ivi->tx_rate = 0;
6550 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6551 ivi->qos = adapter->vf_data[vf].pf_qos;
6552 return 0;
6553}
6554
4ae196df
AD		 6555static void igb_vmm_control(struct igb_adapter *adapter)
6556{
6557 struct e1000_hw *hw = &adapter->hw;
10d8e907 6558 u32 reg;
4ae196df 6559
52a1dd4d
AD		 6560 switch (hw->mac.type) {
6561 case e1000_82575:
6562 default:
6563 /* replication is not supported for 82575 */
4ae196df 6564 return;
52a1dd4d
AD		 6565 case e1000_82576:
6566 /* notify HW that the MAC is adding vlan tags */
6567 reg = rd32(E1000_DTXCTL);
6568 reg |= E1000_DTXCTL_VLAN_ADDED;
6569 wr32(E1000_DTXCTL, reg);
6570 case e1000_82580:
6571 /* enable replication vlan tag stripping */
6572 reg = rd32(E1000_RPLOLR);
6573 reg |= E1000_RPLOLR_STRVLAN;
6574 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6575 case e1000_i350:
6576 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6577 break;
6578 }
10d8e907 6579
d4960307
AD		 6580 if (adapter->vfs_allocated_count) {
6581 igb_vmdq_set_loopback_pf(hw, true);
6582 igb_vmdq_set_replication_pf(hw, true);
6583 } else {
6584 igb_vmdq_set_loopback_pf(hw, false);
6585 igb_vmdq_set_replication_pf(hw, false);
6586 }
4ae196df
AD		 6587}
6588
9d5c8243 6589/* igb_main.c */
Clone of davem's net-next-2.6 tree