]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: setup vlan tag replication stripping in igb_vmm_control
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
85b430b4 94static void igb_setup_tctl(struct igb_adapter *);
9d5c8243
AK		 95static void igb_setup_rctl(struct igb_adapter *);
96static void igb_clean_all_tx_rings(struct igb_adapter *);
97static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 98static void igb_clean_tx_ring(struct igb_ring *);
99static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 100static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 101static void igb_update_phy_info(unsigned long);
102static void igb_watchdog(unsigned long);
103static void igb_watchdog_task(struct work_struct *);
3b29a56d 104static netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *,
3b29a56d
SH		 105 struct igb_ring *);
106static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
107 struct net_device *);
9d5c8243
AK		 108static struct net_device_stats *igb_get_stats(struct net_device *);
109static int igb_change_mtu(struct net_device *, int);
110static int igb_set_mac(struct net_device *, void *);
68d480c4 111static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 112static irqreturn_t igb_intr(int irq, void *);
113static irqreturn_t igb_intr_msi(int irq, void *);
114static irqreturn_t igb_msix_other(int irq, void *);
047e0030 115static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 116#ifdef CONFIG_IGB_DCA
047e0030 117static void igb_update_dca(struct igb_q_vector *);
fe4506b6 118static void igb_setup_dca(struct igb_adapter *);
421e02f0 119#endif /* CONFIG_IGB_DCA */
047e0030 120static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 121static int igb_poll(struct napi_struct *, int);
047e0030 122static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
3b644cf6 123static void igb_alloc_rx_buffers_adv(struct igb_ring *, int);
9d5c8243
AK		 124static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
125static void igb_tx_timeout(struct net_device *);
126static void igb_reset_task(struct work_struct *);
127static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
128static void igb_vlan_rx_add_vid(struct net_device *, u16);
129static void igb_vlan_rx_kill_vid(struct net_device *, u16);
130static void igb_restore_vlan(struct igb_adapter *);
26ad9178 131static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 132static void igb_ping_all_vfs(struct igb_adapter *);
133static void igb_msg_task(struct igb_adapter *);
134static int igb_rcv_msg_from_vf(struct igb_adapter *, u32);
4ae196df 135static void igb_vmm_control(struct igb_adapter *);
4ae196df
AD		 136static int igb_set_vf_mac(struct igb_adapter *adapter, int, unsigned char *);
137static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 138
c8159b2d
ED		 139static inline void igb_set_vmolr(struct e1000_hw *hw, int vfn)
140{
141 u32 reg_data;
142
143 reg_data = rd32(E1000_VMOLR(vfn));
144 reg_data |= E1000_VMOLR_BAM | /* Accept broadcast */
c8159b2d
ED		 145 E1000_VMOLR_ROMPE | /* Accept packets matched in MTA */
146 E1000_VMOLR_AUPE | /* Accept untagged packets */
147 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
148 wr32(E1000_VMOLR(vfn), reg_data);
149}
150
151static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
152 int vfn)
153{
154 struct e1000_hw *hw = &adapter->hw;
155 u32 vmolr;
156
ae641bdc
AD		 157 /* if it isn't the PF check to see if VFs are enabled and
158 * increase the size to support vlan tags */
159 if (vfn < adapter->vfs_allocated_count &&
160 adapter->vf_data[vfn].vlans_enabled)
161 size += VLAN_TAG_SIZE;
162
c8159b2d
ED		 163 vmolr = rd32(E1000_VMOLR(vfn));
164 vmolr &= ~E1000_VMOLR_RLPML_MASK;
165 vmolr |= size | E1000_VMOLR_LPE;
166 wr32(E1000_VMOLR(vfn), vmolr);
167
168 return 0;
169}
170
9d5c8243 171#ifdef CONFIG_PM
3fe7c4c9 172static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 173static int igb_resume(struct pci_dev *);
174#endif
175static void igb_shutdown(struct pci_dev *);
421e02f0 176#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 177static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
178static struct notifier_block dca_notifier = {
179 .notifier_call = igb_notify_dca,
180 .next = NULL,
181 .priority = 0
182};
183#endif
9d5c8243
AK		 184#ifdef CONFIG_NET_POLL_CONTROLLER
185/* for netdump / net console */
186static void igb_netpoll(struct net_device *);
187#endif
37680117 188#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 189static unsigned int max_vfs = 0;
190module_param(max_vfs, uint, 0);
191MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
192 "per physical function");
193#endif /* CONFIG_PCI_IOV */
194
9d5c8243
AK		 195static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
196 pci_channel_state_t);
197static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
198static void igb_io_resume(struct pci_dev *);
199
200static struct pci_error_handlers igb_err_handler = {
201 .error_detected = igb_io_error_detected,
202 .slot_reset = igb_io_slot_reset,
203 .resume = igb_io_resume,
204};
205
206
207static struct pci_driver igb_driver = {
208 .name = igb_driver_name,
209 .id_table = igb_pci_tbl,
210 .probe = igb_probe,
211 .remove = __devexit_p(igb_remove),
212#ifdef CONFIG_PM
213 /* Power Managment Hooks */
214 .suspend = igb_suspend,
215 .resume = igb_resume,
216#endif
217 .shutdown = igb_shutdown,
218 .err_handler = &igb_err_handler
219};
220
7dfc16fa
AD		 221static int global_quad_port_a; /* global quad port a indication */
222
9d5c8243
AK		 223MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
224MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
225MODULE_LICENSE("GPL");
226MODULE_VERSION(DRV_VERSION);
227
38c845c7
PO		 228/**
229 * Scale the NIC clock cycle by a large factor so that
230 * relatively small clock corrections can be added or
231 * substracted at each clock tick. The drawbacks of a
232 * large factor are a) that the clock register overflows
233 * more quickly (not such a big deal) and b) that the
234 * increment per tick has to fit into 24 bits.
235 *
236 * Note that
237 * TIMINCA = IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS *
238 * IGB_TSYNC_SCALE
239 * TIMINCA += TIMINCA * adjustment [ppm] / 1e9
240 *
241 * The base scale factor is intentionally a power of two
242 * so that the division in %struct timecounter can be done with
243 * a shift.
244 */
245#define IGB_TSYNC_SHIFT (19)
246#define IGB_TSYNC_SCALE (1<<IGB_TSYNC_SHIFT)
247
248/**
249 * The duration of one clock cycle of the NIC.
250 *
251 * @todo This hard-coded value is part of the specification and might change
252 * in future hardware revisions. Add revision check.
253 */
254#define IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS 16
255
256#if (IGB_TSYNC_SCALE * IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS) >= (1<<24)
257# error IGB_TSYNC_SCALE and/or IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS are too large to fit into TIMINCA
258#endif
259
260/**
261 * igb_read_clock - read raw cycle counter (to be used by time counter)
262 */
263static cycle_t igb_read_clock(const struct cyclecounter *tc)
264{
265 struct igb_adapter *adapter =
266 container_of(tc, struct igb_adapter, cycles);
267 struct e1000_hw *hw = &adapter->hw;
268 u64 stamp;
269
270 stamp = rd32(E1000_SYSTIML);
271 stamp |= (u64)rd32(E1000_SYSTIMH) << 32ULL;
272
273 return stamp;
274}
275
9d5c8243
AK		 276#ifdef DEBUG
277/**
278 * igb_get_hw_dev_name - return device name string
279 * used by hardware layer to print debugging information
280 **/
281char *igb_get_hw_dev_name(struct e1000_hw *hw)
282{
283 struct igb_adapter *adapter = hw->back;
284 return adapter->netdev->name;
285}
38c845c7
PO		 286
287/**
288 * igb_get_time_str - format current NIC and system time as string
289 */
290static char *igb_get_time_str(struct igb_adapter *adapter,
291 char buffer[160])
292{
293 cycle_t hw = adapter->cycles.read(&adapter->cycles);
294 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
295 struct timespec sys;
296 struct timespec delta;
297 getnstimeofday(&sys);
298
299 delta = timespec_sub(nic, sys);
300
301 sprintf(buffer,
33af6bcc
PO		 302 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
303 hw,
38c845c7
PO		 304 (long)nic.tv_sec, nic.tv_nsec,
305 (long)sys.tv_sec, sys.tv_nsec,
306 (long)delta.tv_sec, delta.tv_nsec);
307
308 return buffer;
309}
9d5c8243
AK		 310#endif
311
c493ea45
AD		 312/**
313 * igb_desc_unused - calculate if we have unused descriptors
314 **/
315static int igb_desc_unused(struct igb_ring *ring)
316{
317 if (ring->next_to_clean > ring->next_to_use)
318 return ring->next_to_clean - ring->next_to_use - 1;
319
320 return ring->count + ring->next_to_clean - ring->next_to_use - 1;
321}
322
9d5c8243
AK		 323/**
324 * igb_init_module - Driver Registration Routine
325 *
326 * igb_init_module is the first routine called when the driver is
327 * loaded. All it does is register with the PCI subsystem.
328 **/
329static int __init igb_init_module(void)
330{
331 int ret;
332 printk(KERN_INFO "%s - version %s\n",
333 igb_driver_string, igb_driver_version);
334
335 printk(KERN_INFO "%s\n", igb_copyright);
336
7dfc16fa
AD		 337 global_quad_port_a = 0;
338
421e02f0 339#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 340 dca_register_notify(&dca_notifier);
341#endif
bbd98fe4
AD		 342
343 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 344 return ret;
345}
346
347module_init(igb_init_module);
348
349/**
350 * igb_exit_module - Driver Exit Cleanup Routine
351 *
352 * igb_exit_module is called just before the driver is removed
353 * from memory.
354 **/
355static void __exit igb_exit_module(void)
356{
421e02f0 357#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 358 dca_unregister_notify(&dca_notifier);
359#endif
9d5c8243
AK		 360 pci_unregister_driver(&igb_driver);
361}
362
363module_exit(igb_exit_module);
364
26bc19ec
AD		 365#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
366/**
367 * igb_cache_ring_register - Descriptor ring to register mapping
368 * @adapter: board private structure to initialize
369 *
370 * Once we know the feature-set enabled for the device, we'll cache
371 * the register offset the descriptor ring is assigned to.
372 **/
373static void igb_cache_ring_register(struct igb_adapter *adapter)
374{
375 int i;
047e0030 376 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 377
378 switch (adapter->hw.mac.type) {
379 case e1000_82576:
380 /* The queues are allocated for virtualization such that VF 0
381 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
382 * In order to avoid collision we start at the first free queue
383 * and continue consuming queues in the same sequence
384 */
385 for (i = 0; i < adapter->num_rx_queues; i++)
1bfaf07b
AD		 386 adapter->rx_ring[i].reg_idx = rbase_offset +
387 Q_IDX_82576(i);
26bc19ec 388 for (i = 0; i < adapter->num_tx_queues; i++)
1bfaf07b
AD		 389 adapter->tx_ring[i].reg_idx = rbase_offset +
390 Q_IDX_82576(i);
26bc19ec
AD		 391 break;
392 case e1000_82575:
393 default:
394 for (i = 0; i < adapter->num_rx_queues; i++)
395 adapter->rx_ring[i].reg_idx = i;
396 for (i = 0; i < adapter->num_tx_queues; i++)
397 adapter->tx_ring[i].reg_idx = i;
398 break;
399 }
400}
401
047e0030
AD		 402static void igb_free_queues(struct igb_adapter *adapter)
403{
404 kfree(adapter->tx_ring);
405 kfree(adapter->rx_ring);
406
407 adapter->tx_ring = NULL;
408 adapter->rx_ring = NULL;
409
410 adapter->num_rx_queues = 0;
411 adapter->num_tx_queues = 0;
412}
413
9d5c8243
AK		 414/**
415 * igb_alloc_queues - Allocate memory for all rings
416 * @adapter: board private structure to initialize
417 *
418 * We allocate one ring per queue at run-time since we don't know the
419 * number of queues at compile-time.
420 **/
421static int igb_alloc_queues(struct igb_adapter *adapter)
422{
423 int i;
424
425 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
426 sizeof(struct igb_ring), GFP_KERNEL);
427 if (!adapter->tx_ring)
047e0030 428 goto err;
9d5c8243
AK		 429
430 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
431 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 432 if (!adapter->rx_ring)
433 goto err;
6eb5a7f1 434
661086df
PWJ		 435 for (i = 0; i < adapter->num_tx_queues; i++) {
436 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 437 ring->count = adapter->tx_ring_count;
661086df 438 ring->queue_index = i;
80785298 439 ring->pdev = adapter->pdev;
e694e964 440 ring->netdev = adapter->netdev;
85ad76b2
AD		 441 /* For 82575, context index must be unique per ring. */
442 if (adapter->hw.mac.type == e1000_82575)
443 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 444 }
85ad76b2 445
9d5c8243
AK		 446 for (i = 0; i < adapter->num_rx_queues; i++) {
447 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 448 ring->count = adapter->rx_ring_count;
844290e5 449 ring->queue_index = i;
80785298 450 ring->pdev = adapter->pdev;
e694e964 451 ring->netdev = adapter->netdev;
4c844851 452 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 453 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
454 /* set flag indicating ring supports SCTP checksum offload */
455 if (adapter->hw.mac.type >= e1000_82576)
456 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 457 }
26bc19ec
AD		 458
459 igb_cache_ring_register(adapter);
9d5c8243 460
047e0030 461 return 0;
a88f10ec 462
047e0030
AD		 463err:
464 igb_free_queues(adapter);
d1a8c9e1 465
047e0030 466 return -ENOMEM;
a88f10ec
AD		 467}
468
9d5c8243 469#define IGB_N0_QUEUE -1
047e0030 470static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 471{
472 u32 msixbm = 0;
047e0030 473 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 474 struct e1000_hw *hw = &adapter->hw;
2d064c06 475 u32 ivar, index;
047e0030
AD		 476 int rx_queue = IGB_N0_QUEUE;
477 int tx_queue = IGB_N0_QUEUE;
478
479 if (q_vector->rx_ring)
480 rx_queue = q_vector->rx_ring->reg_idx;
481 if (q_vector->tx_ring)
482 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 483
484 switch (hw->mac.type) {
485 case e1000_82575:
9d5c8243
AK		 486 /* The 82575 assigns vectors using a bitmask, which matches the
487 bitmask for the EICR/EIMS/EIMC registers. To assign one
488 or more queues to a vector, we write the appropriate bits
489 into the MSIXBM register for that vector. */
047e0030 490 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 491 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 492 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 493 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 494 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 495 q_vector->eims_value = msixbm;
2d064c06
AD		 496 break;
497 case e1000_82576:
26bc19ec 498 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 499 Each queue has a single entry in the table to which we write
500 a vector number along with a "valid" bit. Sadly, the layout
501 of the table is somewhat counterintuitive. */
502 if (rx_queue > IGB_N0_QUEUE) {
047e0030 503 index = (rx_queue & 0x7);
2d064c06 504 ivar = array_rd32(E1000_IVAR0, index);
047e0030 505 if (rx_queue < 8) {
26bc19ec
AD		 506 /* vector goes into low byte of register */
507 ivar = ivar & 0xFFFFFF00;
508 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 509 } else {
510 /* vector goes into third byte of register */
511 ivar = ivar & 0xFF00FFFF;
512 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 513 }
2d064c06
AD		 514 array_wr32(E1000_IVAR0, index, ivar);
515 }
516 if (tx_queue > IGB_N0_QUEUE) {
047e0030 517 index = (tx_queue & 0x7);
2d064c06 518 ivar = array_rd32(E1000_IVAR0, index);
047e0030 519 if (tx_queue < 8) {
26bc19ec
AD		 520 /* vector goes into second byte of register */
521 ivar = ivar & 0xFFFF00FF;
522 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 523 } else {
524 /* vector goes into high byte of register */
525 ivar = ivar & 0x00FFFFFF;
526 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 527 }
2d064c06
AD		 528 array_wr32(E1000_IVAR0, index, ivar);
529 }
047e0030 530 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 531 break;
532 default:
533 BUG();
534 break;
535 }
9d5c8243
AK		 536}
537
538/**
539 * igb_configure_msix - Configure MSI-X hardware
540 *
541 * igb_configure_msix sets up the hardware to properly
542 * generate MSI-X interrupts.
543 **/
544static void igb_configure_msix(struct igb_adapter *adapter)
545{
546 u32 tmp;
547 int i, vector = 0;
548 struct e1000_hw *hw = &adapter->hw;
549
550 adapter->eims_enable_mask = 0;
9d5c8243
AK		 551
552 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 553 switch (hw->mac.type) {
554 case e1000_82575:
9d5c8243
AK		 555 tmp = rd32(E1000_CTRL_EXT);
556 /* enable MSI-X PBA support*/
557 tmp |= E1000_CTRL_EXT_PBA_CLR;
558
559 /* Auto-Mask interrupts upon ICR read. */
560 tmp |= E1000_CTRL_EXT_EIAME;
561 tmp |= E1000_CTRL_EXT_IRCA;
562
563 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 564
565 /* enable msix_other interrupt */
566 array_wr32(E1000_MSIXBM(0), vector++,
567 E1000_EIMS_OTHER);
844290e5 568 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 569
2d064c06
AD		 570 break;
571
572 case e1000_82576:
047e0030
AD		 573 /* Turn on MSI-X capability first, or our settings
574 * won't stick. And it will take days to debug. */
575 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
576 E1000_GPIE_PBA | E1000_GPIE_EIAME |
577 E1000_GPIE_NSICR);
578
579 /* enable msix_other interrupt */
580 adapter->eims_other = 1 << vector;
2d064c06 581 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 582
047e0030 583 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 584 break;
585 default:
586 /* do nothing, since nothing else supports MSI-X */
587 break;
588 } /* switch (hw->mac.type) */
047e0030
AD		 589
590 adapter->eims_enable_mask |= adapter->eims_other;
591
592 for (i = 0; i < adapter->num_q_vectors; i++) {
593 struct igb_q_vector *q_vector = adapter->q_vector[i];
594 igb_assign_vector(q_vector, vector++);
595 adapter->eims_enable_mask |= q_vector->eims_value;
596 }
597
9d5c8243
AK		 598 wrfl();
599}
600
601/**
602 * igb_request_msix - Initialize MSI-X interrupts
603 *
604 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
605 * kernel.
606 **/
607static int igb_request_msix(struct igb_adapter *adapter)
608{
609 struct net_device *netdev = adapter->netdev;
047e0030 610 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 611 int i, err = 0, vector = 0;
612
047e0030
AD		 613 err = request_irq(adapter->msix_entries[vector].vector,
614 &igb_msix_other, 0, netdev->name, adapter);
615 if (err)
616 goto out;
617 vector++;
618
619 for (i = 0; i < adapter->num_q_vectors; i++) {
620 struct igb_q_vector *q_vector = adapter->q_vector[i];
621
622 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
623
624 if (q_vector->rx_ring && q_vector->tx_ring)
625 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
626 q_vector->rx_ring->queue_index);
627 else if (q_vector->tx_ring)
628 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
629 q_vector->tx_ring->queue_index);
630 else if (q_vector->rx_ring)
631 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
632 q_vector->rx_ring->queue_index);
9d5c8243 633 else
047e0030
AD		 634 sprintf(q_vector->name, "%s-unused", netdev->name);
635
9d5c8243 636 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 637 &igb_msix_ring, 0, q_vector->name,
638 q_vector);
9d5c8243
AK		 639 if (err)
640 goto out;
9d5c8243
AK		 641 vector++;
642 }
643
9d5c8243
AK		 644 igb_configure_msix(adapter);
645 return 0;
646out:
647 return err;
648}
649
650static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
651{
652 if (adapter->msix_entries) {
653 pci_disable_msix(adapter->pdev);
654 kfree(adapter->msix_entries);
655 adapter->msix_entries = NULL;
047e0030 656 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 657 pci_disable_msi(adapter->pdev);
047e0030 658 }
9d5c8243
AK		 659}
660
047e0030
AD		 661/**
662 * igb_free_q_vectors - Free memory allocated for interrupt vectors
663 * @adapter: board private structure to initialize
664 *
665 * This function frees the memory allocated to the q_vectors. In addition if
666 * NAPI is enabled it will delete any references to the NAPI struct prior
667 * to freeing the q_vector.
668 **/
669static void igb_free_q_vectors(struct igb_adapter *adapter)
670{
671 int v_idx;
672
673 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
674 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
675 adapter->q_vector[v_idx] = NULL;
676 netif_napi_del(&q_vector->napi);
677 kfree(q_vector);
678 }
679 adapter->num_q_vectors = 0;
680}
681
682/**
683 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
684 *
685 * This function resets the device so that it has 0 rx queues, tx queues, and
686 * MSI-X interrupts allocated.
687 */
688static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
689{
690 igb_free_queues(adapter);
691 igb_free_q_vectors(adapter);
692 igb_reset_interrupt_capability(adapter);
693}
9d5c8243
AK		 694
695/**
696 * igb_set_interrupt_capability - set MSI or MSI-X if supported
697 *
698 * Attempt to configure interrupts using the best available
699 * capabilities of the hardware and kernel.
700 **/
701static void igb_set_interrupt_capability(struct igb_adapter *adapter)
702{
703 int err;
704 int numvecs, i;
705
83b7180d 706 /* Number of supported queues. */
83b7180d
AD		 707 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
708 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
709
047e0030
AD		 710 /* start with one vector for every rx queue */
711 numvecs = adapter->num_rx_queues;
712
713 /* if tx handler is seperate add 1 for every tx queue */
714 numvecs += adapter->num_tx_queues;
715
716 /* store the number of vectors reserved for queues */
717 adapter->num_q_vectors = numvecs;
718
719 /* add 1 vector for link status interrupts */
720 numvecs++;
9d5c8243
AK		 721 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
722 GFP_KERNEL);
723 if (!adapter->msix_entries)
724 goto msi_only;
725
726 for (i = 0; i < numvecs; i++)
727 adapter->msix_entries[i].entry = i;
728
729 err = pci_enable_msix(adapter->pdev,
730 adapter->msix_entries,
731 numvecs);
732 if (err == 0)
34a20e89 733 goto out;
9d5c8243
AK		 734
735 igb_reset_interrupt_capability(adapter);
736
737 /* If we can't do MSI-X, try MSI */
738msi_only:
2a3abf6d
AD		 739#ifdef CONFIG_PCI_IOV
740 /* disable SR-IOV for non MSI-X configurations */
741 if (adapter->vf_data) {
742 struct e1000_hw *hw = &adapter->hw;
743 /* disable iov and allow time for transactions to clear */
744 pci_disable_sriov(adapter->pdev);
745 msleep(500);
746
747 kfree(adapter->vf_data);
748 adapter->vf_data = NULL;
749 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
750 msleep(100);
751 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
752 }
753#endif
9d5c8243 754 adapter->num_rx_queues = 1;
661086df 755 adapter->num_tx_queues = 1;
047e0030 756 adapter->num_q_vectors = 1;
9d5c8243 757 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 758 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 759out:
661086df 760 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 761 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 762 return;
763}
764
047e0030
AD		 765/**
766 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
767 * @adapter: board private structure to initialize
768 *
769 * We allocate one q_vector per queue interrupt. If allocation fails we
770 * return -ENOMEM.
771 **/
772static int igb_alloc_q_vectors(struct igb_adapter *adapter)
773{
774 struct igb_q_vector *q_vector;
775 struct e1000_hw *hw = &adapter->hw;
776 int v_idx;
777
778 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
779 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
780 if (!q_vector)
781 goto err_out;
782 q_vector->adapter = adapter;
783 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
784 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
785 q_vector->itr_val = IGB_START_ITR;
786 q_vector->set_itr = 1;
787 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
788 adapter->q_vector[v_idx] = q_vector;
789 }
790 return 0;
791
792err_out:
793 while (v_idx) {
794 v_idx--;
795 q_vector = adapter->q_vector[v_idx];
796 netif_napi_del(&q_vector->napi);
797 kfree(q_vector);
798 adapter->q_vector[v_idx] = NULL;
799 }
800 return -ENOMEM;
801}
802
803static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
804 int ring_idx, int v_idx)
805{
806 struct igb_q_vector *q_vector;
807
808 q_vector = adapter->q_vector[v_idx];
809 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
810 q_vector->rx_ring->q_vector = q_vector;
811 q_vector->itr_val = adapter->itr;
812}
813
814static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
815 int ring_idx, int v_idx)
816{
817 struct igb_q_vector *q_vector;
818
819 q_vector = adapter->q_vector[v_idx];
820 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
821 q_vector->tx_ring->q_vector = q_vector;
822 q_vector->itr_val = adapter->itr;
823}
824
825/**
826 * igb_map_ring_to_vector - maps allocated queues to vectors
827 *
828 * This function maps the recently allocated queues to vectors.
829 **/
830static int igb_map_ring_to_vector(struct igb_adapter *adapter)
831{
832 int i;
833 int v_idx = 0;
834
835 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
836 (adapter->num_q_vectors < adapter->num_tx_queues))
837 return -ENOMEM;
838
839 if (adapter->num_q_vectors >=
840 (adapter->num_rx_queues + adapter->num_tx_queues)) {
841 for (i = 0; i < adapter->num_rx_queues; i++)
842 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
843 for (i = 0; i < adapter->num_tx_queues; i++)
844 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
845 } else {
846 for (i = 0; i < adapter->num_rx_queues; i++) {
847 if (i < adapter->num_tx_queues)
848 igb_map_tx_ring_to_vector(adapter, i, v_idx);
849 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
850 }
851 for (; i < adapter->num_tx_queues; i++)
852 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
853 }
854 return 0;
855}
856
857/**
858 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
859 *
860 * This function initializes the interrupts and allocates all of the queues.
861 **/
862static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
863{
864 struct pci_dev *pdev = adapter->pdev;
865 int err;
866
867 igb_set_interrupt_capability(adapter);
868
869 err = igb_alloc_q_vectors(adapter);
870 if (err) {
871 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
872 goto err_alloc_q_vectors;
873 }
874
875 err = igb_alloc_queues(adapter);
876 if (err) {
877 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
878 goto err_alloc_queues;
879 }
880
881 err = igb_map_ring_to_vector(adapter);
882 if (err) {
883 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
884 goto err_map_queues;
885 }
886
887
888 return 0;
889err_map_queues:
890 igb_free_queues(adapter);
891err_alloc_queues:
892 igb_free_q_vectors(adapter);
893err_alloc_q_vectors:
894 igb_reset_interrupt_capability(adapter);
895 return err;
896}
897
9d5c8243
AK		 898/**
899 * igb_request_irq - initialize interrupts
900 *
901 * Attempts to configure interrupts using the best available
902 * capabilities of the hardware and kernel.
903 **/
904static int igb_request_irq(struct igb_adapter *adapter)
905{
906 struct net_device *netdev = adapter->netdev;
047e0030 907 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 908 struct e1000_hw *hw = &adapter->hw;
909 int err = 0;
910
911 if (adapter->msix_entries) {
912 err = igb_request_msix(adapter);
844290e5 913 if (!err)
9d5c8243 914 goto request_done;
9d5c8243 915 /* fall back to MSI */
047e0030 916 igb_clear_interrupt_scheme(adapter);
9d5c8243 917 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 918 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 919 igb_free_all_tx_resources(adapter);
920 igb_free_all_rx_resources(adapter);
047e0030 921 adapter->num_tx_queues = 1;
9d5c8243 922 adapter->num_rx_queues = 1;
047e0030
AD		 923 adapter->num_q_vectors = 1;
924 err = igb_alloc_q_vectors(adapter);
925 if (err) {
926 dev_err(&pdev->dev,
927 "Unable to allocate memory for vectors\n");
928 goto request_done;
929 }
930 err = igb_alloc_queues(adapter);
931 if (err) {
932 dev_err(&pdev->dev,
933 "Unable to allocate memory for queues\n");
934 igb_free_q_vectors(adapter);
935 goto request_done;
936 }
937 igb_setup_all_tx_resources(adapter);
938 igb_setup_all_rx_resources(adapter);
844290e5 939 } else {
2d064c06
AD		 940 switch (hw->mac.type) {
941 case e1000_82575:
942 wr32(E1000_MSIXBM(0),
047e0030
AD		 943 (E1000_EICR_RX_QUEUE0 |
944 E1000_EICR_TX_QUEUE0 |
945 E1000_EIMS_OTHER));
2d064c06
AD		 946 break;
947 case e1000_82576:
948 wr32(E1000_IVAR0, E1000_IVAR_VALID);
949 break;
950 default:
951 break;
952 }
9d5c8243 953 }
844290e5 954
7dfc16fa 955 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 956 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 957 netdev->name, adapter);
9d5c8243
AK		 958 if (!err)
959 goto request_done;
047e0030 960
9d5c8243
AK		 961 /* fall back to legacy interrupts */
962 igb_reset_interrupt_capability(adapter);
7dfc16fa 963 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 964 }
965
966 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 967 netdev->name, adapter);
9d5c8243 968
6cb5e577 969 if (err)
9d5c8243
AK		 970 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
971 err);
9d5c8243
AK		 972
973request_done:
974 return err;
975}
976
977static void igb_free_irq(struct igb_adapter *adapter)
978{
9d5c8243
AK		 979 if (adapter->msix_entries) {
980 int vector = 0, i;
981
047e0030 982 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 983
047e0030
AD		 984 for (i = 0; i < adapter->num_q_vectors; i++) {
985 struct igb_q_vector *q_vector = adapter->q_vector[i];
986 free_irq(adapter->msix_entries[vector++].vector,
987 q_vector);
988 }
989 } else {
990 free_irq(adapter->pdev->irq, adapter);
9d5c8243 991 }
9d5c8243
AK		 992}
993
994/**
995 * igb_irq_disable - Mask off interrupt generation on the NIC
996 * @adapter: board private structure
997 **/
998static void igb_irq_disable(struct igb_adapter *adapter)
999{
1000 struct e1000_hw *hw = &adapter->hw;
1001
1002 if (adapter->msix_entries) {
2dfd1212
AD		 1003 u32 regval = rd32(E1000_EIAM);
1004 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1005 wr32(E1000_EIMC, adapter->eims_enable_mask);
1006 regval = rd32(E1000_EIAC);
1007 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1008 }
844290e5
PW		 1009
1010 wr32(E1000_IAM, 0);
9d5c8243
AK		 1011 wr32(E1000_IMC, ~0);
1012 wrfl();
1013 synchronize_irq(adapter->pdev->irq);
1014}
1015
1016/**
1017 * igb_irq_enable - Enable default interrupt generation settings
1018 * @adapter: board private structure
1019 **/
1020static void igb_irq_enable(struct igb_adapter *adapter)
1021{
1022 struct e1000_hw *hw = &adapter->hw;
1023
1024 if (adapter->msix_entries) {
2dfd1212
AD		 1025 u32 regval = rd32(E1000_EIAC);
1026 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1027 regval = rd32(E1000_EIAM);
1028 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1029 wr32(E1000_EIMS, adapter->eims_enable_mask);
4ae196df
AD		 1030 if (adapter->vfs_allocated_count)
1031 wr32(E1000_MBVFIMR, 0xFF);
1032 wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB |
1033 E1000_IMS_DOUTSYNC));
844290e5
PW		 1034 } else {
1035 wr32(E1000_IMS, IMS_ENABLE_MASK);
1036 wr32(E1000_IAM, IMS_ENABLE_MASK);
1037 }
9d5c8243
AK		 1038}
1039
1040static void igb_update_mng_vlan(struct igb_adapter *adapter)
1041{
1042 struct net_device *netdev = adapter->netdev;
1043 u16 vid = adapter->hw.mng_cookie.vlan_id;
1044 u16 old_vid = adapter->mng_vlan_id;
1045 if (adapter->vlgrp) {
1046 if (!vlan_group_get_device(adapter->vlgrp, vid)) {
1047 if (adapter->hw.mng_cookie.status &
1048 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1049 igb_vlan_rx_add_vid(netdev, vid);
1050 adapter->mng_vlan_id = vid;
1051 } else
1052 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1053
1054 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1055 (vid != old_vid) &&
1056 !vlan_group_get_device(adapter->vlgrp, old_vid))
1057 igb_vlan_rx_kill_vid(netdev, old_vid);
1058 } else
1059 adapter->mng_vlan_id = vid;
1060 }
1061}
1062
1063/**
1064 * igb_release_hw_control - release control of the h/w to f/w
1065 * @adapter: address of board private structure
1066 *
1067 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1068 * For ASF and Pass Through versions of f/w this means that the
1069 * driver is no longer loaded.
1070 *
1071 **/
1072static void igb_release_hw_control(struct igb_adapter *adapter)
1073{
1074 struct e1000_hw *hw = &adapter->hw;
1075 u32 ctrl_ext;
1076
1077 /* Let firmware take over control of h/w */
1078 ctrl_ext = rd32(E1000_CTRL_EXT);
1079 wr32(E1000_CTRL_EXT,
1080 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1081}
1082
1083
1084/**
1085 * igb_get_hw_control - get control of the h/w from f/w
1086 * @adapter: address of board private structure
1087 *
1088 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1089 * For ASF and Pass Through versions of f/w this means that
1090 * the driver is loaded.
1091 *
1092 **/
1093static void igb_get_hw_control(struct igb_adapter *adapter)
1094{
1095 struct e1000_hw *hw = &adapter->hw;
1096 u32 ctrl_ext;
1097
1098 /* Let firmware know the driver has taken over */
1099 ctrl_ext = rd32(E1000_CTRL_EXT);
1100 wr32(E1000_CTRL_EXT,
1101 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1102}
1103
9d5c8243
AK		 1104/**
1105 * igb_configure - configure the hardware for RX and TX
1106 * @adapter: private board structure
1107 **/
1108static void igb_configure(struct igb_adapter *adapter)
1109{
1110 struct net_device *netdev = adapter->netdev;
1111 int i;
1112
1113 igb_get_hw_control(adapter);
ff41f8dc 1114 igb_set_rx_mode(netdev);
9d5c8243
AK		 1115
1116 igb_restore_vlan(adapter);
9d5c8243 1117
85b430b4 1118 igb_setup_tctl(adapter);
06cf2666 1119 igb_setup_mrqc(adapter);
9d5c8243 1120 igb_setup_rctl(adapter);
85b430b4
AD		 1121
1122 igb_configure_tx(adapter);
9d5c8243 1123 igb_configure_rx(adapter);
662d7205
AD		 1124
1125 igb_rx_fifo_flush_82575(&adapter->hw);
1126
c493ea45 1127 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1128 * at least 1 descriptor unused to make sure
1129 * next_to_use != next_to_clean */
1130 for (i = 0; i < adapter->num_rx_queues; i++) {
1131 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1132 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1133 }
1134
1135
1136 adapter->tx_queue_len = netdev->tx_queue_len;
1137}
1138
1139
1140/**
1141 * igb_up - Open the interface and prepare it to handle traffic
1142 * @adapter: board private structure
1143 **/
1144
1145int igb_up(struct igb_adapter *adapter)
1146{
1147 struct e1000_hw *hw = &adapter->hw;
1148 int i;
1149
1150 /* hardware has been reset, we need to reload some things */
1151 igb_configure(adapter);
1152
1153 clear_bit(__IGB_DOWN, &adapter->state);
1154
047e0030
AD		 1155 for (i = 0; i < adapter->num_q_vectors; i++) {
1156 struct igb_q_vector *q_vector = adapter->q_vector[i];
1157 napi_enable(&q_vector->napi);
1158 }
844290e5 1159 if (adapter->msix_entries)
9d5c8243 1160 igb_configure_msix(adapter);
9d5c8243 1161
e1739522
AD		 1162 igb_set_vmolr(hw, adapter->vfs_allocated_count);
1163
9d5c8243
AK		 1164 /* Clear any pending interrupts. */
1165 rd32(E1000_ICR);
1166 igb_irq_enable(adapter);
1167
d4960307
AD		 1168 /* notify VFs that reset has been completed */
1169 if (adapter->vfs_allocated_count) {
1170 u32 reg_data = rd32(E1000_CTRL_EXT);
1171 reg_data |= E1000_CTRL_EXT_PFRSTD;
1172 wr32(E1000_CTRL_EXT, reg_data);
1173 }
1174
4cb9be7a
JB		 1175 netif_tx_start_all_queues(adapter->netdev);
1176
9d5c8243
AK		 1177 /* Fire a link change interrupt to start the watchdog. */
1178 wr32(E1000_ICS, E1000_ICS_LSC);
1179 return 0;
1180}
1181
1182void igb_down(struct igb_adapter *adapter)
1183{
1184 struct e1000_hw *hw = &adapter->hw;
1185 struct net_device *netdev = adapter->netdev;
1186 u32 tctl, rctl;
1187 int i;
1188
1189 /* signal that we're down so the interrupt handler does not
1190 * reschedule our watchdog timer */
1191 set_bit(__IGB_DOWN, &adapter->state);
1192
1193 /* disable receives in the hardware */
1194 rctl = rd32(E1000_RCTL);
1195 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1196 /* flush and sleep below */
1197
fd2ea0a7 1198 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1199
1200 /* disable transmits in the hardware */
1201 tctl = rd32(E1000_TCTL);
1202 tctl &= ~E1000_TCTL_EN;
1203 wr32(E1000_TCTL, tctl);
1204 /* flush both disables and wait for them to finish */
1205 wrfl();
1206 msleep(10);
1207
047e0030
AD		 1208 for (i = 0; i < adapter->num_q_vectors; i++) {
1209 struct igb_q_vector *q_vector = adapter->q_vector[i];
1210 napi_disable(&q_vector->napi);
1211 }
9d5c8243 1212
9d5c8243
AK		 1213 igb_irq_disable(adapter);
1214
1215 del_timer_sync(&adapter->watchdog_timer);
1216 del_timer_sync(&adapter->phy_info_timer);
1217
1218 netdev->tx_queue_len = adapter->tx_queue_len;
1219 netif_carrier_off(netdev);
04fe6358
AD		 1220
1221 /* record the stats before reset*/
1222 igb_update_stats(adapter);
1223
9d5c8243
AK		 1224 adapter->link_speed = 0;
1225 adapter->link_duplex = 0;
1226
3023682e
JK		 1227 if (!pci_channel_offline(adapter->pdev))
1228 igb_reset(adapter);
9d5c8243
AK		 1229 igb_clean_all_tx_rings(adapter);
1230 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1231#ifdef CONFIG_IGB_DCA
1232
1233 /* since we reset the hardware DCA settings were cleared */
1234 igb_setup_dca(adapter);
1235#endif
9d5c8243
AK		 1236}
1237
1238void igb_reinit_locked(struct igb_adapter *adapter)
1239{
1240 WARN_ON(in_interrupt());
1241 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1242 msleep(1);
1243 igb_down(adapter);
1244 igb_up(adapter);
1245 clear_bit(__IGB_RESETTING, &adapter->state);
1246}
1247
1248void igb_reset(struct igb_adapter *adapter)
1249{
1250 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1251 struct e1000_mac_info *mac = &hw->mac;
1252 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1253 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1254 u16 hwm;
1255
1256 /* Repartition Pba for greater than 9k mtu
1257 * To take effect CTRL.RST is required.
1258 */
fa4dfae0
AD		 1259 switch (mac->type) {
1260 case e1000_82576:
2d064c06 1261 pba = E1000_PBA_64K;
fa4dfae0
AD		 1262 break;
1263 case e1000_82575:
1264 default:
1265 pba = E1000_PBA_34K;
1266 break;
2d064c06 1267 }
9d5c8243 1268
2d064c06
AD		 1269 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1270 (mac->type < e1000_82576)) {
9d5c8243
AK		 1271 /* adjust PBA for jumbo frames */
1272 wr32(E1000_PBA, pba);
1273
1274 /* To maintain wire speed transmits, the Tx FIFO should be
1275 * large enough to accommodate two full transmit packets,
1276 * rounded up to the next 1KB and expressed in KB. Likewise,
1277 * the Rx FIFO should be large enough to accommodate at least
1278 * one full receive packet and is similarly rounded up and
1279 * expressed in KB. */
1280 pba = rd32(E1000_PBA);
1281 /* upper 16 bits has Tx packet buffer allocation size in KB */
1282 tx_space = pba >> 16;
1283 /* lower 16 bits has Rx packet buffer allocation size in KB */
1284 pba &= 0xffff;
1285 /* the tx fifo also stores 16 bytes of information about the tx
1286 * but don't include ethernet FCS because hardware appends it */
1287 min_tx_space = (adapter->max_frame_size +
85e8d004 1288 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1289 ETH_FCS_LEN) * 2;
1290 min_tx_space = ALIGN(min_tx_space, 1024);
1291 min_tx_space >>= 10;
1292 /* software strips receive CRC, so leave room for it */
1293 min_rx_space = adapter->max_frame_size;
1294 min_rx_space = ALIGN(min_rx_space, 1024);
1295 min_rx_space >>= 10;
1296
1297 /* If current Tx allocation is less than the min Tx FIFO size,
1298 * and the min Tx FIFO size is less than the current Rx FIFO
1299 * allocation, take space away from current Rx allocation */
1300 if (tx_space < min_tx_space &&
1301 ((min_tx_space - tx_space) < pba)) {
1302 pba = pba - (min_tx_space - tx_space);
1303
1304 /* if short on rx space, rx wins and must trump tx
1305 * adjustment */
1306 if (pba < min_rx_space)
1307 pba = min_rx_space;
1308 }
2d064c06 1309 wr32(E1000_PBA, pba);
9d5c8243 1310 }
9d5c8243
AK		 1311
1312 /* flow control settings */
1313 /* The high water mark must be low enough to fit one full frame
1314 * (or the size used for early receive) above it in the Rx FIFO.
1315 * Set it to the lower of:
1316 * - 90% of the Rx FIFO size, or
1317 * - the full Rx FIFO size minus one full frame */
1318 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1319 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1320
2d064c06
AD		 1321 if (mac->type < e1000_82576) {
1322 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1323 fc->low_water = fc->high_water - 8;
1324 } else {
1325 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1326 fc->low_water = fc->high_water - 16;
1327 }
9d5c8243
AK		 1328 fc->pause_time = 0xFFFF;
1329 fc->send_xon = 1;
0cce119a 1330 fc->current_mode = fc->requested_mode;
9d5c8243 1331
4ae196df
AD		 1332 /* disable receive for all VFs and wait one second */
1333 if (adapter->vfs_allocated_count) {
1334 int i;
1335 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1336 adapter->vf_data[i].clear_to_send = false;
1337
1338 /* ping all the active vfs to let them know we are going down */
1339 igb_ping_all_vfs(adapter);
1340
1341 /* disable transmits and receives */
1342 wr32(E1000_VFRE, 0);
1343 wr32(E1000_VFTE, 0);
1344 }
1345
9d5c8243
AK		 1346 /* Allow time for pending master requests to run */
1347 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1348 wr32(E1000_WUC, 0);
1349
1350 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1351 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1352
1353 igb_update_mng_vlan(adapter);
1354
1355 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1356 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1357
1358 igb_reset_adaptive(&adapter->hw);
f5f4cf08 1359 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 1360}
1361
2e5c6922
SH		 1362static const struct net_device_ops igb_netdev_ops = {
1363 .ndo_open = igb_open,
1364 .ndo_stop = igb_close,
00829823 1365 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1366 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1367 .ndo_set_rx_mode = igb_set_rx_mode,
1368 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1369 .ndo_set_mac_address = igb_set_mac,
1370 .ndo_change_mtu = igb_change_mtu,
1371 .ndo_do_ioctl = igb_ioctl,
1372 .ndo_tx_timeout = igb_tx_timeout,
1373 .ndo_validate_addr = eth_validate_addr,
1374 .ndo_vlan_rx_register = igb_vlan_rx_register,
1375 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1376 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1377#ifdef CONFIG_NET_POLL_CONTROLLER
1378 .ndo_poll_controller = igb_netpoll,
1379#endif
1380};
1381
9d5c8243
AK		 1382/**
1383 * igb_probe - Device Initialization Routine
1384 * @pdev: PCI device information struct
1385 * @ent: entry in igb_pci_tbl
1386 *
1387 * Returns 0 on success, negative on failure
1388 *
1389 * igb_probe initializes an adapter identified by a pci_dev structure.
1390 * The OS initialization, configuring of the adapter private structure,
1391 * and a hardware reset occur.
1392 **/
1393static int __devinit igb_probe(struct pci_dev *pdev,
1394 const struct pci_device_id *ent)
1395{
1396 struct net_device *netdev;
1397 struct igb_adapter *adapter;
1398 struct e1000_hw *hw;
1399 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1400 unsigned long mmio_start, mmio_len;
2d6a5e95 1401 int err, pci_using_dac;
682337fe 1402 u16 eeprom_data = 0;
9d5c8243
AK		 1403 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1404 u32 part_num;
1405
aed5dec3 1406 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1407 if (err)
1408 return err;
1409
1410 pci_using_dac = 0;
6a35528a 1411 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1412 if (!err) {
6a35528a 1413 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1414 if (!err)
1415 pci_using_dac = 1;
1416 } else {
284901a9 1417 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1418 if (err) {
284901a9 1419 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1420 if (err) {
1421 dev_err(&pdev->dev, "No usable DMA "
1422 "configuration, aborting\n");
1423 goto err_dma;
1424 }
1425 }
1426 }
1427
aed5dec3
AD		 1428 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1429 IORESOURCE_MEM),
1430 igb_driver_name);
9d5c8243
AK		 1431 if (err)
1432 goto err_pci_reg;
1433
19d5afd4 1434 pci_enable_pcie_error_reporting(pdev);
40a914fa 1435
9d5c8243 1436 pci_set_master(pdev);
c682fc23 1437 pci_save_state(pdev);
9d5c8243
AK		 1438
1439 err = -ENOMEM;
1bfaf07b
AD		 1440 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1441 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1442 if (!netdev)
1443 goto err_alloc_etherdev;
1444
1445 SET_NETDEV_DEV(netdev, &pdev->dev);
1446
1447 pci_set_drvdata(pdev, netdev);
1448 adapter = netdev_priv(netdev);
1449 adapter->netdev = netdev;
1450 adapter->pdev = pdev;
1451 hw = &adapter->hw;
1452 hw->back = adapter;
1453 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1454
1455 mmio_start = pci_resource_start(pdev, 0);
1456 mmio_len = pci_resource_len(pdev, 0);
1457
1458 err = -EIO;
28b0759c
AD		 1459 hw->hw_addr = ioremap(mmio_start, mmio_len);
1460 if (!hw->hw_addr)
9d5c8243
AK		 1461 goto err_ioremap;
1462
2e5c6922 1463 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1464 igb_set_ethtool_ops(netdev);
9d5c8243 1465 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1466
1467 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1468
1469 netdev->mem_start = mmio_start;
1470 netdev->mem_end = mmio_start + mmio_len;
1471
9d5c8243
AK		 1472 /* PCI config space info */
1473 hw->vendor_id = pdev->vendor;
1474 hw->device_id = pdev->device;
1475 hw->revision_id = pdev->revision;
1476 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1477 hw->subsystem_device_id = pdev->subsystem_device;
1478
1479 /* setup the private structure */
1480 hw->back = adapter;
1481 /* Copy the default MAC, PHY and NVM function pointers */
1482 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1483 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1484 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1485 /* Initialize skew-specific constants */
1486 err = ei->get_invariants(hw);
1487 if (err)
450c87c8 1488 goto err_sw_init;
9d5c8243 1489
2a3abf6d
AD		 1490#ifdef CONFIG_PCI_IOV
1491 /* since iov functionality isn't critical to base device function we
1492 * can accept failure. If it fails we don't allow iov to be enabled */
1493 if (hw->mac.type == e1000_82576) {
1494 /* 82576 supports a maximum of 7 VFs in addition to the PF */
1495 unsigned int num_vfs = (max_vfs > 7) ? 7 : max_vfs;
1496 int i;
1497 unsigned char mac_addr[ETH_ALEN];
1498
9ca046d5 1499 if (num_vfs) {
2a3abf6d
AD		 1500 adapter->vf_data = kcalloc(num_vfs,
1501 sizeof(struct vf_data_storage),
1502 GFP_KERNEL);
9ca046d5
AD		 1503 if (!adapter->vf_data) {
1504 dev_err(&pdev->dev,
1505 "Could not allocate VF private data - "
1506 "IOV enable failed\n");
2a3abf6d 1507 } else {
9ca046d5
AD		 1508 err = pci_enable_sriov(pdev, num_vfs);
1509 if (!err) {
1510 adapter->vfs_allocated_count = num_vfs;
1511 dev_info(&pdev->dev,
1512 "%d vfs allocated\n",
1513 num_vfs);
1514 for (i = 0;
1515 i < adapter->vfs_allocated_count;
1516 i++) {
1517 random_ether_addr(mac_addr);
1518 igb_set_vf_mac(adapter, i,
1519 mac_addr);
1520 }
1521 } else {
1522 kfree(adapter->vf_data);
1523 adapter->vf_data = NULL;
1524 }
2a3abf6d
AD		 1525 }
1526 }
1527 }
1528
1529#endif
450c87c8 1530 /* setup the private structure */
9d5c8243
AK		 1531 err = igb_sw_init(adapter);
1532 if (err)
1533 goto err_sw_init;
1534
1535 igb_get_bus_info_pcie(hw);
1536
1537 hw->phy.autoneg_wait_to_complete = false;
1538 hw->mac.adaptive_ifs = true;
1539
1540 /* Copper options */
1541 if (hw->phy.media_type == e1000_media_type_copper) {
1542 hw->phy.mdix = AUTO_ALL_MODES;
1543 hw->phy.disable_polarity_correction = false;
1544 hw->phy.ms_type = e1000_ms_hw_default;
1545 }
1546
1547 if (igb_check_reset_block(hw))
1548 dev_info(&pdev->dev,
1549 "PHY reset is blocked due to SOL/IDER session.\n");
1550
1551 netdev->features = NETIF_F_SG |
7d8eb29e 1552 NETIF_F_IP_CSUM |
9d5c8243
AK		 1553 NETIF_F_HW_VLAN_TX |
1554 NETIF_F_HW_VLAN_RX |
1555 NETIF_F_HW_VLAN_FILTER;
1556
7d8eb29e 1557 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1558 netdev->features |= NETIF_F_TSO;
9d5c8243 1559 netdev->features |= NETIF_F_TSO6;
48f29ffc 1560
5c0999b7 1561 netdev->features |= NETIF_F_GRO;
d3352520 1562
48f29ffc
JK		 1563 netdev->vlan_features |= NETIF_F_TSO;
1564 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1565 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1566 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1567 netdev->vlan_features |= NETIF_F_SG;
1568
9d5c8243
AK		 1569 if (pci_using_dac)
1570 netdev->features |= NETIF_F_HIGHDMA;
1571
b9473560
JB		 1572 if (adapter->hw.mac.type == e1000_82576)
1573 netdev->features |= NETIF_F_SCTP_CSUM;
1574
9d5c8243
AK		 1575 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1576
1577 /* before reading the NVM, reset the controller to put the device in a
1578 * known good starting state */
1579 hw->mac.ops.reset_hw(hw);
1580
1581 /* make sure the NVM is good */
1582 if (igb_validate_nvm_checksum(hw) < 0) {
1583 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1584 err = -EIO;
1585 goto err_eeprom;
1586 }
1587
1588 /* copy the MAC address out of the NVM */
1589 if (hw->mac.ops.read_mac_addr(hw))
1590 dev_err(&pdev->dev, "NVM Read Error\n");
1591
1592 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1593 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1594
1595 if (!is_valid_ether_addr(netdev->perm_addr)) {
1596 dev_err(&pdev->dev, "Invalid MAC Address\n");
1597 err = -EIO;
1598 goto err_eeprom;
1599 }
1600
0e340485
AD		 1601 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1602 (unsigned long) adapter);
1603 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1604 (unsigned long) adapter);
9d5c8243
AK		 1605
1606 INIT_WORK(&adapter->reset_task, igb_reset_task);
1607 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1608
450c87c8 1609 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1610 adapter->fc_autoneg = true;
1611 hw->mac.autoneg = true;
1612 hw->phy.autoneg_advertised = 0x2f;
1613
0cce119a
AD		 1614 hw->fc.requested_mode = e1000_fc_default;
1615 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1616
cbd347ad 1617 adapter->itr_setting = IGB_DEFAULT_ITR;
9d5c8243
AK		 1618 adapter->itr = IGB_START_ITR;
1619
1620 igb_validate_mdi_setting(hw);
1621
9d5c8243
AK		 1622 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1623 * enable the ACPI Magic Packet filter
1624 */
1625
a2cf8b6c 1626 if (hw->bus.func == 0)
312c75ae 1627 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1628 else if (hw->bus.func == 1)
1629 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1630
1631 if (eeprom_data & eeprom_apme_mask)
1632 adapter->eeprom_wol |= E1000_WUFC_MAG;
1633
1634 /* now that we have the eeprom settings, apply the special cases where
1635 * the eeprom may be wrong or the board simply won't support wake on
1636 * lan on a particular port */
1637 switch (pdev->device) {
1638 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1639 adapter->eeprom_wol = 0;
1640 break;
1641 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1642 case E1000_DEV_ID_82576_FIBER:
1643 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1644 /* Wake events only supported on port A for dual fiber
1645 * regardless of eeprom setting */
1646 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1647 adapter->eeprom_wol = 0;
1648 break;
c8ea5ea9
AD		 1649 case E1000_DEV_ID_82576_QUAD_COPPER:
1650 /* if quad port adapter, disable WoL on all but port A */
1651 if (global_quad_port_a != 0)
1652 adapter->eeprom_wol = 0;
1653 else
1654 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1655 /* Reset for multiple quad port adapters */
1656 if (++global_quad_port_a == 4)
1657 global_quad_port_a = 0;
1658 break;
9d5c8243
AK		 1659 }
1660
1661 /* initialize the wol settings based on the eeprom settings */
1662 adapter->wol = adapter->eeprom_wol;
e1b86d84 1663 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1664
1665 /* reset the hardware with the new settings */
1666 igb_reset(adapter);
1667
1668 /* let the f/w know that the h/w is now under the control of the
1669 * driver. */
1670 igb_get_hw_control(adapter);
1671
9d5c8243
AK		 1672 strcpy(netdev->name, "eth%d");
1673 err = register_netdev(netdev);
1674 if (err)
1675 goto err_register;
1676
b168dfc5
JB		 1677 /* carrier off reporting is important to ethtool even BEFORE open */
1678 netif_carrier_off(netdev);
1679
421e02f0 1680#ifdef CONFIG_IGB_DCA
bbd98fe4 1681 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1682 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1683 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1684 igb_setup_dca(adapter);
1685 }
1686#endif
1687
38c845c7
PO		 1688 /*
1689 * Initialize hardware timer: we keep it running just in case
1690 * that some program needs it later on.
1691 */
1692 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1693 adapter->cycles.read = igb_read_clock;
1694 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1695 adapter->cycles.mult = 1;
1696 adapter->cycles.shift = IGB_TSYNC_SHIFT;
1697 wr32(E1000_TIMINCA,
1698 (1<<24) |
1699 IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS * IGB_TSYNC_SCALE);
1700#if 0
1701 /*
1702 * Avoid rollover while we initialize by resetting the time counter.
1703 */
1704 wr32(E1000_SYSTIML, 0x00000000);
1705 wr32(E1000_SYSTIMH, 0x00000000);
1706#else
1707 /*
1708 * Set registers so that rollover occurs soon to test this.
1709 */
1710 wr32(E1000_SYSTIML, 0x00000000);
1711 wr32(E1000_SYSTIMH, 0xFF800000);
1712#endif
1713 wrfl();
1714 timecounter_init(&adapter->clock,
1715 &adapter->cycles,
1716 ktime_to_ns(ktime_get_real()));
1717
33af6bcc
PO		 1718 /*
1719 * Synchronize our NIC clock against system wall clock. NIC
1720 * time stamp reading requires ~3us per sample, each sample
1721 * was pretty stable even under load => only require 10
1722 * samples for each offset comparison.
1723 */
1724 memset(&adapter->compare, 0, sizeof(adapter->compare));
1725 adapter->compare.source = &adapter->clock;
1726 adapter->compare.target = ktime_get_real;
1727 adapter->compare.num_samples = 10;
1728 timecompare_update(&adapter->compare, 0);
1729
38c845c7
PO		 1730#ifdef DEBUG
1731 {
1732 char buffer[160];
1733 printk(KERN_DEBUG
1734 "igb: %s: hw %p initialized timer\n",
1735 igb_get_time_str(adapter, buffer),
1736 &adapter->hw);
1737 }
1738#endif
1739
9d5c8243
AK		 1740 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1741 /* print bus type/speed/width info */
7c510e4b 1742 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1743 netdev->name,
1744 ((hw->bus.speed == e1000_bus_speed_2500)
1745 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1746 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1747 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1748 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1749 "unknown"),
7c510e4b 1750 netdev->dev_addr);
9d5c8243
AK		 1751
1752 igb_read_part_num(hw, &part_num);
1753 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1754 (part_num >> 8), (part_num & 0xff));
1755
1756 dev_info(&pdev->dev,
1757 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1758 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1759 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1760 adapter->num_rx_queues, adapter->num_tx_queues);
1761
9d5c8243
AK		 1762 return 0;
1763
1764err_register:
1765 igb_release_hw_control(adapter);
1766err_eeprom:
1767 if (!igb_check_reset_block(hw))
f5f4cf08 1768 igb_reset_phy(hw);
9d5c8243
AK		 1769
1770 if (hw->flash_address)
1771 iounmap(hw->flash_address);
9d5c8243 1772err_sw_init:
047e0030 1773 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1774 iounmap(hw->hw_addr);
1775err_ioremap:
1776 free_netdev(netdev);
1777err_alloc_etherdev:
aed5dec3
AD		 1778 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1779 IORESOURCE_MEM));
9d5c8243
AK		 1780err_pci_reg:
1781err_dma:
1782 pci_disable_device(pdev);
1783 return err;
1784}
1785
1786/**
1787 * igb_remove - Device Removal Routine
1788 * @pdev: PCI device information struct
1789 *
1790 * igb_remove is called by the PCI subsystem to alert the driver
1791 * that it should release a PCI device. The could be caused by a
1792 * Hot-Plug event, or because the driver is going to be removed from
1793 * memory.
1794 **/
1795static void __devexit igb_remove(struct pci_dev *pdev)
1796{
1797 struct net_device *netdev = pci_get_drvdata(pdev);
1798 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1799 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1800
1801 /* flush_scheduled work may reschedule our watchdog task, so
1802 * explicitly disable watchdog tasks from being rescheduled */
1803 set_bit(__IGB_DOWN, &adapter->state);
1804 del_timer_sync(&adapter->watchdog_timer);
1805 del_timer_sync(&adapter->phy_info_timer);
1806
1807 flush_scheduled_work();
1808
421e02f0 1809#ifdef CONFIG_IGB_DCA
7dfc16fa 1810 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1811 dev_info(&pdev->dev, "DCA disabled\n");
1812 dca_remove_requester(&pdev->dev);
7dfc16fa 1813 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1814 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1815 }
1816#endif
1817
9d5c8243
AK		 1818 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1819 * would have already happened in close and is redundant. */
1820 igb_release_hw_control(adapter);
1821
1822 unregister_netdev(netdev);
1823
f5f4cf08
AD		 1824 if (!igb_check_reset_block(&adapter->hw))
1825 igb_reset_phy(&adapter->hw);
9d5c8243 1826
047e0030 1827 igb_clear_interrupt_scheme(adapter);
9d5c8243 1828
37680117
AD		 1829#ifdef CONFIG_PCI_IOV
1830 /* reclaim resources allocated to VFs */
1831 if (adapter->vf_data) {
1832 /* disable iov and allow time for transactions to clear */
1833 pci_disable_sriov(pdev);
1834 msleep(500);
1835
1836 kfree(adapter->vf_data);
1837 adapter->vf_data = NULL;
1838 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1839 msleep(100);
1840 dev_info(&pdev->dev, "IOV Disabled\n");
1841 }
1842#endif
28b0759c
AD		 1843 iounmap(hw->hw_addr);
1844 if (hw->flash_address)
1845 iounmap(hw->flash_address);
aed5dec3
AD		 1846 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1847 IORESOURCE_MEM));
9d5c8243
AK		 1848
1849 free_netdev(netdev);
1850
19d5afd4 1851 pci_disable_pcie_error_reporting(pdev);
40a914fa 1852
9d5c8243
AK		 1853 pci_disable_device(pdev);
1854}
1855
1856/**
1857 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1858 * @adapter: board private structure to initialize
1859 *
1860 * igb_sw_init initializes the Adapter private data structure.
1861 * Fields are initialized based on PCI device information and
1862 * OS network device settings (MTU size).
1863 **/
1864static int __devinit igb_sw_init(struct igb_adapter *adapter)
1865{
1866 struct e1000_hw *hw = &adapter->hw;
1867 struct net_device *netdev = adapter->netdev;
1868 struct pci_dev *pdev = adapter->pdev;
1869
1870 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1871
68fd9910
AD		 1872 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1873 adapter->rx_ring_count = IGB_DEFAULT_RXD;
9d5c8243
AK		 1874 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1875 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1876
661086df
PWJ		 1877 /* This call may decrease the number of queues depending on
1878 * interrupt mode. */
047e0030 1879 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1880 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1881 return -ENOMEM;
1882 }
1883
1884 /* Explicitly disable IRQ since the NIC can be in any state. */
1885 igb_irq_disable(adapter);
1886
1887 set_bit(__IGB_DOWN, &adapter->state);
1888 return 0;
1889}
1890
1891/**
1892 * igb_open - Called when a network interface is made active
1893 * @netdev: network interface device structure
1894 *
1895 * Returns 0 on success, negative value on failure
1896 *
1897 * The open entry point is called when a network interface is made
1898 * active by the system (IFF_UP). At this point all resources needed
1899 * for transmit and receive operations are allocated, the interrupt
1900 * handler is registered with the OS, the watchdog timer is started,
1901 * and the stack is notified that the interface is ready.
1902 **/
1903static int igb_open(struct net_device *netdev)
1904{
1905 struct igb_adapter *adapter = netdev_priv(netdev);
1906 struct e1000_hw *hw = &adapter->hw;
1907 int err;
1908 int i;
1909
1910 /* disallow open during test */
1911 if (test_bit(__IGB_TESTING, &adapter->state))
1912 return -EBUSY;
1913
b168dfc5
JB		 1914 netif_carrier_off(netdev);
1915
9d5c8243
AK		 1916 /* allocate transmit descriptors */
1917 err = igb_setup_all_tx_resources(adapter);
1918 if (err)
1919 goto err_setup_tx;
1920
1921 /* allocate receive descriptors */
1922 err = igb_setup_all_rx_resources(adapter);
1923 if (err)
1924 goto err_setup_rx;
1925
1926 /* e1000_power_up_phy(adapter); */
1927
1928 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1929 if ((adapter->hw.mng_cookie.status &
1930 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
1931 igb_update_mng_vlan(adapter);
1932
1933 /* before we allocate an interrupt, we must be ready to handle it.
1934 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1935 * as soon as we call pci_request_irq, so we have to setup our
1936 * clean_rx handler before we do so. */
1937 igb_configure(adapter);
1938
e1739522
AD		 1939 igb_set_vmolr(hw, adapter->vfs_allocated_count);
1940
9d5c8243
AK		 1941 err = igb_request_irq(adapter);
1942 if (err)
1943 goto err_req_irq;
1944
1945 /* From here on the code is the same as igb_up() */
1946 clear_bit(__IGB_DOWN, &adapter->state);
1947
047e0030
AD		 1948 for (i = 0; i < adapter->num_q_vectors; i++) {
1949 struct igb_q_vector *q_vector = adapter->q_vector[i];
1950 napi_enable(&q_vector->napi);
1951 }
9d5c8243
AK		 1952
1953 /* Clear any pending interrupts. */
1954 rd32(E1000_ICR);
844290e5
PW		 1955
1956 igb_irq_enable(adapter);
1957
d4960307
AD		 1958 /* notify VFs that reset has been completed */
1959 if (adapter->vfs_allocated_count) {
1960 u32 reg_data = rd32(E1000_CTRL_EXT);
1961 reg_data |= E1000_CTRL_EXT_PFRSTD;
1962 wr32(E1000_CTRL_EXT, reg_data);
1963 }
1964
d55b53ff
JK		 1965 netif_tx_start_all_queues(netdev);
1966
9d5c8243
AK		 1967 /* Fire a link status change interrupt to start the watchdog. */
1968 wr32(E1000_ICS, E1000_ICS_LSC);
1969
1970 return 0;
1971
1972err_req_irq:
1973 igb_release_hw_control(adapter);
1974 /* e1000_power_down_phy(adapter); */
1975 igb_free_all_rx_resources(adapter);
1976err_setup_rx:
1977 igb_free_all_tx_resources(adapter);
1978err_setup_tx:
1979 igb_reset(adapter);
1980
1981 return err;
1982}
1983
1984/**
1985 * igb_close - Disables a network interface
1986 * @netdev: network interface device structure
1987 *
1988 * Returns 0, this is not allowed to fail
1989 *
1990 * The close entry point is called when an interface is de-activated
1991 * by the OS. The hardware is still under the driver's control, but
1992 * needs to be disabled. A global MAC reset is issued to stop the
1993 * hardware, and all transmit and receive resources are freed.
1994 **/
1995static int igb_close(struct net_device *netdev)
1996{
1997 struct igb_adapter *adapter = netdev_priv(netdev);
1998
1999 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2000 igb_down(adapter);
2001
2002 igb_free_irq(adapter);
2003
2004 igb_free_all_tx_resources(adapter);
2005 igb_free_all_rx_resources(adapter);
2006
2007 /* kill manageability vlan ID if supported, but not if a vlan with
2008 * the same ID is registered on the host OS (let 8021q kill it) */
2009 if ((adapter->hw.mng_cookie.status &
2010 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2011 !(adapter->vlgrp &&
2012 vlan_group_get_device(adapter->vlgrp, adapter->mng_vlan_id)))
2013 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2014
2015 return 0;
2016}
2017
2018/**
2019 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2020 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2021 *
2022 * Return 0 on success, negative on failure
2023 **/
80785298 2024int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2025{
80785298 2026 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2027 int size;
2028
2029 size = sizeof(struct igb_buffer) * tx_ring->count;
2030 tx_ring->buffer_info = vmalloc(size);
2031 if (!tx_ring->buffer_info)
2032 goto err;
2033 memset(tx_ring->buffer_info, 0, size);
2034
2035 /* round up to nearest 4K */
85e8d004 2036 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2037 tx_ring->size = ALIGN(tx_ring->size, 4096);
2038
2039 tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
2040 &tx_ring->dma);
2041
2042 if (!tx_ring->desc)
2043 goto err;
2044
9d5c8243
AK		 2045 tx_ring->next_to_use = 0;
2046 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2047 return 0;
2048
2049err:
2050 vfree(tx_ring->buffer_info);
047e0030 2051 dev_err(&pdev->dev,
9d5c8243
AK		 2052 "Unable to allocate memory for the transmit descriptor ring\n");
2053 return -ENOMEM;
2054}
2055
2056/**
2057 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2058 * (Descriptors) for all queues
2059 * @adapter: board private structure
2060 *
2061 * Return 0 on success, negative on failure
2062 **/
2063static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2064{
2065 int i, err = 0;
661086df 2066 int r_idx;
9d5c8243
AK		 2067
2068 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 2069 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2070 if (err) {
2071 dev_err(&adapter->pdev->dev,
2072 "Allocation for Tx Queue %u failed\n", i);
2073 for (i--; i >= 0; i--)
3b644cf6 2074 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2075 break;
2076 }
2077 }
2078
661086df
PWJ		 2079 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
2080 r_idx = i % adapter->num_tx_queues;
2081 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 2082 }
9d5c8243
AK		 2083 return err;
2084}
2085
2086/**
85b430b4
AD		 2087 * igb_setup_tctl - configure the transmit control registers
2088 * @adapter: Board private structure
9d5c8243 2089 **/
85b430b4 2090static void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2091{
9d5c8243
AK		 2092 struct e1000_hw *hw = &adapter->hw;
2093 u32 tctl;
9d5c8243 2094
85b430b4
AD		 2095 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2096 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2097
2098 /* Program the Transmit Control Register */
9d5c8243
AK		 2099 tctl = rd32(E1000_TCTL);
2100 tctl &= ~E1000_TCTL_CT;
2101 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2102 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2103
2104 igb_config_collision_dist(hw);
2105
9d5c8243
AK		 2106 /* Enable transmits */
2107 tctl |= E1000_TCTL_EN;
2108
2109 wr32(E1000_TCTL, tctl);
2110}
2111
85b430b4
AD		 2112/**
2113 * igb_configure_tx_ring - Configure transmit ring after Reset
2114 * @adapter: board private structure
2115 * @ring: tx ring to configure
2116 *
2117 * Configure a transmit ring after a reset.
2118 **/
2119static void igb_configure_tx_ring(struct igb_adapter *adapter,
2120 struct igb_ring *ring)
2121{
2122 struct e1000_hw *hw = &adapter->hw;
2123 u32 txdctl;
2124 u64 tdba = ring->dma;
2125 int reg_idx = ring->reg_idx;
2126
2127 /* disable the queue */
2128 txdctl = rd32(E1000_TXDCTL(reg_idx));
2129 wr32(E1000_TXDCTL(reg_idx),
2130 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2131 wrfl();
2132 mdelay(10);
2133
2134 wr32(E1000_TDLEN(reg_idx),
2135 ring->count * sizeof(union e1000_adv_tx_desc));
2136 wr32(E1000_TDBAL(reg_idx),
2137 tdba & 0x00000000ffffffffULL);
2138 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2139
fce99e34
AD		 2140 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2141 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2142 writel(0, ring->head);
2143 writel(0, ring->tail);
85b430b4
AD		 2144
2145 txdctl |= IGB_TX_PTHRESH;
2146 txdctl |= IGB_TX_HTHRESH << 8;
2147 txdctl |= IGB_TX_WTHRESH << 16;
2148
2149 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2150 wr32(E1000_TXDCTL(reg_idx), txdctl);
2151}
2152
2153/**
2154 * igb_configure_tx - Configure transmit Unit after Reset
2155 * @adapter: board private structure
2156 *
2157 * Configure the Tx unit of the MAC after a reset.
2158 **/
2159static void igb_configure_tx(struct igb_adapter *adapter)
2160{
2161 int i;
2162
2163 for (i = 0; i < adapter->num_tx_queues; i++)
2164 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2165}
2166
9d5c8243
AK		 2167/**
2168 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2169 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2170 *
2171 * Returns 0 on success, negative on failure
2172 **/
80785298 2173int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2174{
80785298 2175 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2176 int size, desc_len;
2177
2178 size = sizeof(struct igb_buffer) * rx_ring->count;
2179 rx_ring->buffer_info = vmalloc(size);
2180 if (!rx_ring->buffer_info)
2181 goto err;
2182 memset(rx_ring->buffer_info, 0, size);
2183
2184 desc_len = sizeof(union e1000_adv_rx_desc);
2185
2186 /* Round up to nearest 4K */
2187 rx_ring->size = rx_ring->count * desc_len;
2188 rx_ring->size = ALIGN(rx_ring->size, 4096);
2189
2190 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2191 &rx_ring->dma);
2192
2193 if (!rx_ring->desc)
2194 goto err;
2195
2196 rx_ring->next_to_clean = 0;
2197 rx_ring->next_to_use = 0;
9d5c8243 2198
9d5c8243
AK		 2199 return 0;
2200
2201err:
2202 vfree(rx_ring->buffer_info);
80785298 2203 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2204 "the receive descriptor ring\n");
2205 return -ENOMEM;
2206}
2207
2208/**
2209 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2210 * (Descriptors) for all queues
2211 * @adapter: board private structure
2212 *
2213 * Return 0 on success, negative on failure
2214 **/
2215static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2216{
2217 int i, err = 0;
2218
2219 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2220 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2221 if (err) {
2222 dev_err(&adapter->pdev->dev,
2223 "Allocation for Rx Queue %u failed\n", i);
2224 for (i--; i >= 0; i--)
3b644cf6 2225 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2226 break;
2227 }
2228 }
2229
2230 return err;
2231}
2232
06cf2666
AD		 2233/**
2234 * igb_setup_mrqc - configure the multiple receive queue control registers
2235 * @adapter: Board private structure
2236 **/
2237static void igb_setup_mrqc(struct igb_adapter *adapter)
2238{
2239 struct e1000_hw *hw = &adapter->hw;
2240 u32 mrqc, rxcsum;
2241 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2242 union e1000_reta {
2243 u32 dword;
2244 u8 bytes[4];
2245 } reta;
2246 static const u8 rsshash[40] = {
2247 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2248 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2249 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2250 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2251
2252 /* Fill out hash function seeds */
2253 for (j = 0; j < 10; j++) {
2254 u32 rsskey = rsshash[(j * 4)];
2255 rsskey |= rsshash[(j * 4) + 1] << 8;
2256 rsskey |= rsshash[(j * 4) + 2] << 16;
2257 rsskey |= rsshash[(j * 4) + 3] << 24;
2258 array_wr32(E1000_RSSRK(0), j, rsskey);
2259 }
2260
2261 num_rx_queues = adapter->num_rx_queues;
2262
2263 if (adapter->vfs_allocated_count) {
2264 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2265 switch (hw->mac.type) {
2266 case e1000_82576:
2267 shift = 3;
2268 num_rx_queues = 2;
2269 break;
2270 case e1000_82575:
2271 shift = 2;
2272 shift2 = 6;
2273 default:
2274 break;
2275 }
2276 } else {
2277 if (hw->mac.type == e1000_82575)
2278 shift = 6;
2279 }
2280
2281 for (j = 0; j < (32 * 4); j++) {
2282 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2283 if (shift2)
2284 reta.bytes[j & 3] |= num_rx_queues << shift2;
2285 if ((j & 3) == 3)
2286 wr32(E1000_RETA(j >> 2), reta.dword);
2287 }
2288
2289 /*
2290 * Disable raw packet checksumming so that RSS hash is placed in
2291 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2292 * offloads as they are enabled by default
2293 */
2294 rxcsum = rd32(E1000_RXCSUM);
2295 rxcsum |= E1000_RXCSUM_PCSD;
2296
2297 if (adapter->hw.mac.type >= e1000_82576)
2298 /* Enable Receive Checksum Offload for SCTP */
2299 rxcsum |= E1000_RXCSUM_CRCOFL;
2300
2301 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2302 wr32(E1000_RXCSUM, rxcsum);
2303
2304 /* If VMDq is enabled then we set the appropriate mode for that, else
2305 * we default to RSS so that an RSS hash is calculated per packet even
2306 * if we are only using one queue */
2307 if (adapter->vfs_allocated_count) {
2308 if (hw->mac.type > e1000_82575) {
2309 /* Set the default pool for the PF's first queue */
2310 u32 vtctl = rd32(E1000_VT_CTL);
2311 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2312 E1000_VT_CTL_DISABLE_DEF_POOL);
2313 vtctl |= adapter->vfs_allocated_count <<
2314 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2315 wr32(E1000_VT_CTL, vtctl);
2316 }
2317 if (adapter->num_rx_queues > 1)
2318 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2319 else
2320 mrqc = E1000_MRQC_ENABLE_VMDQ;
2321 } else {
2322 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2323 }
2324 igb_vmm_control(adapter);
2325
2326 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2327 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2328 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2329 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2330 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2331 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2332 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2333 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2334
2335 wr32(E1000_MRQC, mrqc);
2336}
2337
9d5c8243
AK		 2338/**
2339 * igb_setup_rctl - configure the receive control registers
2340 * @adapter: Board private structure
2341 **/
2342static void igb_setup_rctl(struct igb_adapter *adapter)
2343{
2344 struct e1000_hw *hw = &adapter->hw;
2345 u32 rctl;
9d5c8243
AK		 2346
2347 rctl = rd32(E1000_RCTL);
2348
2349 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2350 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2351
69d728ba 2352 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2353 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2354
87cb7e8c
AK		 2355 /*
2356 * enable stripping of CRC. It's unlikely this will break BMC
2357 * redirection as it did with e1000. Newer features require
2358 * that the HW strips the CRC.
73cd78f1 2359 */
87cb7e8c 2360 rctl |= E1000_RCTL_SECRC;
9d5c8243 2361
9b07f3d3 2362 /*
ec54d7d6 2363 * disable store bad packets and clear size bits.
9b07f3d3 2364 */
ec54d7d6 2365 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2366
6ec43fe6
AD		 2367 /* enable LPE to prevent packets larger than max_frame_size */
2368 rctl |= E1000_RCTL_LPE;
9d5c8243 2369
952f72a8
AD		 2370 /* disable queue 0 to prevent tail write w/o re-config */
2371 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2372
e1739522
AD		 2373 /* Attention!!! For SR-IOV PF driver operations you must enable
2374 * queue drop for all VF and PF queues to prevent head of line blocking
2375 * if an un-trusted VF does not provide descriptors to hardware.
2376 */
2377 if (adapter->vfs_allocated_count) {
2378 u32 vmolr;
2379
e1739522
AD		 2380 /* set all queue drop enable bits */
2381 wr32(E1000_QDE, ALL_QUEUES);
e1739522 2382
77a22941 2383 vmolr = rd32(E1000_VMOLR(adapter->vfs_allocated_count));
e1739522
AD		 2384 if (rctl & E1000_RCTL_LPE)
2385 vmolr |= E1000_VMOLR_LPE;
77a22941 2386 if (adapter->num_rx_queues > 1)
e1739522 2387 vmolr |= E1000_VMOLR_RSSE;
77a22941 2388 wr32(E1000_VMOLR(adapter->vfs_allocated_count), vmolr);
e1739522
AD		 2389 }
2390
9d5c8243
AK		 2391 wr32(E1000_RCTL, rctl);
2392}
2393
e1739522
AD		 2394/**
2395 * igb_rlpml_set - set maximum receive packet size
2396 * @adapter: board private structure
2397 *
2398 * Configure maximum receivable packet size.
2399 **/
2400static void igb_rlpml_set(struct igb_adapter *adapter)
2401{
2402 u32 max_frame_size = adapter->max_frame_size;
2403 struct e1000_hw *hw = &adapter->hw;
2404 u16 pf_id = adapter->vfs_allocated_count;
2405
2406 if (adapter->vlgrp)
2407 max_frame_size += VLAN_TAG_SIZE;
2408
2409 /* if vfs are enabled we set RLPML to the largest possible request
2410 * size and set the VMOLR RLPML to the size we need */
2411 if (pf_id) {
2412 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2413 max_frame_size = MAX_STD_JUMBO_FRAME_SIZE + VLAN_TAG_SIZE;
2414 }
2415
2416 wr32(E1000_RLPML, max_frame_size);
2417}
2418
85b430b4
AD		 2419/**
2420 * igb_configure_rx_ring - Configure a receive ring after Reset
2421 * @adapter: board private structure
2422 * @ring: receive ring to be configured
2423 *
2424 * Configure the Rx unit of the MAC after a reset.
2425 **/
2426static void igb_configure_rx_ring(struct igb_adapter *adapter,
2427 struct igb_ring *ring)
2428{
2429 struct e1000_hw *hw = &adapter->hw;
2430 u64 rdba = ring->dma;
2431 int reg_idx = ring->reg_idx;
952f72a8 2432 u32 srrctl, rxdctl;
85b430b4
AD		 2433
2434 /* disable the queue */
2435 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2436 wr32(E1000_RXDCTL(reg_idx),
2437 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2438
2439 /* Set DMA base address registers */
2440 wr32(E1000_RDBAL(reg_idx),
2441 rdba & 0x00000000ffffffffULL);
2442 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2443 wr32(E1000_RDLEN(reg_idx),
2444 ring->count * sizeof(union e1000_adv_rx_desc));
2445
2446 /* initialize head and tail */
fce99e34
AD		 2447 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2448 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2449 writel(0, ring->head);
2450 writel(0, ring->tail);
85b430b4 2451
952f72a8 2452 /* set descriptor configuration */
4c844851
AD		 2453 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2454 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2455 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2456#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2457 srrctl |= IGB_RXBUFFER_16384 >>
2458 E1000_SRRCTL_BSIZEPKT_SHIFT;
2459#else
2460 srrctl |= (PAGE_SIZE / 2) >>
2461 E1000_SRRCTL_BSIZEPKT_SHIFT;
2462#endif
2463 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2464 } else {
4c844851 2465 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2466 E1000_SRRCTL_BSIZEPKT_SHIFT;
2467 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2468 }
2469
2470 wr32(E1000_SRRCTL(reg_idx), srrctl);
2471
85b430b4
AD		 2472 /* enable receive descriptor fetching */
2473 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2474 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2475 rxdctl &= 0xFFF00000;
2476 rxdctl |= IGB_RX_PTHRESH;
2477 rxdctl |= IGB_RX_HTHRESH << 8;
2478 rxdctl |= IGB_RX_WTHRESH << 16;
2479 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2480}
2481
9d5c8243
AK		 2482/**
2483 * igb_configure_rx - Configure receive Unit after Reset
2484 * @adapter: board private structure
2485 *
2486 * Configure the Rx unit of the MAC after a reset.
2487 **/
2488static void igb_configure_rx(struct igb_adapter *adapter)
2489{
9107584e 2490 int i;
9d5c8243 2491
68d480c4
AD		 2492 /* set UTA to appropriate mode */
2493 igb_set_uta(adapter);
2494
26ad9178
AD		 2495 /* set the correct pool for the PF default MAC address in entry 0 */
2496 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2497 adapter->vfs_allocated_count);
2498
06cf2666
AD		 2499 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2500 * the Base and Length of the Rx Descriptor Ring */
2501 for (i = 0; i < adapter->num_rx_queues; i++)
2502 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2503}
2504
2505/**
2506 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2507 * @tx_ring: Tx descriptor ring for a specific queue
2508 *
2509 * Free all transmit software resources
2510 **/
68fd9910 2511void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2512{
3b644cf6 2513 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2514
2515 vfree(tx_ring->buffer_info);
2516 tx_ring->buffer_info = NULL;
2517
80785298
AD		 2518 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2519 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2520
2521 tx_ring->desc = NULL;
2522}
2523
2524/**
2525 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2526 * @adapter: board private structure
2527 *
2528 * Free all transmit software resources
2529 **/
2530static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2531{
2532 int i;
2533
2534 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2535 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2536}
2537
80785298 2538static void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
9d5c8243
AK		 2539 struct igb_buffer *buffer_info)
2540{
65689fef 2541 buffer_info->dma = 0;
9d5c8243 2542 if (buffer_info->skb) {
80785298
AD		 2543 skb_dma_unmap(&tx_ring->pdev->dev,
2544 buffer_info->skb,
65689fef 2545 DMA_TO_DEVICE);
9d5c8243
AK		 2546 dev_kfree_skb_any(buffer_info->skb);
2547 buffer_info->skb = NULL;
2548 }
2549 buffer_info->time_stamp = 0;
2550 /* buffer_info must be completely set up in the transmit path */
2551}
2552
2553/**
2554 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2555 * @tx_ring: ring to be cleaned
2556 **/
3b644cf6 2557static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2558{
2559 struct igb_buffer *buffer_info;
2560 unsigned long size;
2561 unsigned int i;
2562
2563 if (!tx_ring->buffer_info)
2564 return;
2565 /* Free all the Tx ring sk_buffs */
2566
2567 for (i = 0; i < tx_ring->count; i++) {
2568 buffer_info = &tx_ring->buffer_info[i];
80785298 2569 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2570 }
2571
2572 size = sizeof(struct igb_buffer) * tx_ring->count;
2573 memset(tx_ring->buffer_info, 0, size);
2574
2575 /* Zero out the descriptor ring */
2576
2577 memset(tx_ring->desc, 0, tx_ring->size);
2578
2579 tx_ring->next_to_use = 0;
2580 tx_ring->next_to_clean = 0;
2581
fce99e34
AD		 2582 writel(0, tx_ring->head);
2583 writel(0, tx_ring->tail);
9d5c8243
AK		 2584}
2585
2586/**
2587 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2588 * @adapter: board private structure
2589 **/
2590static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2591{
2592 int i;
2593
2594 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2595 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2596}
2597
2598/**
2599 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2600 * @rx_ring: ring to clean the resources from
2601 *
2602 * Free all receive software resources
2603 **/
68fd9910 2604void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2605{
3b644cf6 2606 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2607
2608 vfree(rx_ring->buffer_info);
2609 rx_ring->buffer_info = NULL;
2610
80785298
AD		 2611 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2612 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2613
2614 rx_ring->desc = NULL;
2615}
2616
2617/**
2618 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2619 * @adapter: board private structure
2620 *
2621 * Free all receive software resources
2622 **/
2623static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2624{
2625 int i;
2626
2627 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2628 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2629}
2630
2631/**
2632 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2633 * @rx_ring: ring to free buffers from
2634 **/
3b644cf6 2635static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2636{
2637 struct igb_buffer *buffer_info;
9d5c8243
AK		 2638 unsigned long size;
2639 unsigned int i;
2640
2641 if (!rx_ring->buffer_info)
2642 return;
2643 /* Free all the Rx ring sk_buffs */
2644 for (i = 0; i < rx_ring->count; i++) {
2645 buffer_info = &rx_ring->buffer_info[i];
2646 if (buffer_info->dma) {
80785298
AD		 2647 pci_unmap_single(rx_ring->pdev,
2648 buffer_info->dma,
4c844851 2649 rx_ring->rx_buffer_len,
6ec43fe6 2650 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2651 buffer_info->dma = 0;
2652 }
2653
2654 if (buffer_info->skb) {
2655 dev_kfree_skb(buffer_info->skb);
2656 buffer_info->skb = NULL;
2657 }
6ec43fe6 2658 if (buffer_info->page_dma) {
80785298
AD		 2659 pci_unmap_page(rx_ring->pdev,
2660 buffer_info->page_dma,
6ec43fe6
AD		 2661 PAGE_SIZE / 2,
2662 PCI_DMA_FROMDEVICE);
2663 buffer_info->page_dma = 0;
2664 }
9d5c8243 2665 if (buffer_info->page) {
9d5c8243
AK		 2666 put_page(buffer_info->page);
2667 buffer_info->page = NULL;
bf36c1a0 2668 buffer_info->page_offset = 0;
9d5c8243
AK		 2669 }
2670 }
2671
9d5c8243
AK		 2672 size = sizeof(struct igb_buffer) * rx_ring->count;
2673 memset(rx_ring->buffer_info, 0, size);
2674
2675 /* Zero out the descriptor ring */
2676 memset(rx_ring->desc, 0, rx_ring->size);
2677
2678 rx_ring->next_to_clean = 0;
2679 rx_ring->next_to_use = 0;
2680
fce99e34
AD		 2681 writel(0, rx_ring->head);
2682 writel(0, rx_ring->tail);
9d5c8243
AK		 2683}
2684
2685/**
2686 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2687 * @adapter: board private structure
2688 **/
2689static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2690{
2691 int i;
2692
2693 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2694 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2695}
2696
2697/**
2698 * igb_set_mac - Change the Ethernet Address of the NIC
2699 * @netdev: network interface device structure
2700 * @p: pointer to an address structure
2701 *
2702 * Returns 0 on success, negative on failure
2703 **/
2704static int igb_set_mac(struct net_device *netdev, void *p)
2705{
2706 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2707 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2708 struct sockaddr *addr = p;
2709
2710 if (!is_valid_ether_addr(addr->sa_data))
2711 return -EADDRNOTAVAIL;
2712
2713 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2714 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2715
26ad9178
AD		 2716 /* set the correct pool for the new PF MAC address in entry 0 */
2717 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2718 adapter->vfs_allocated_count);
e1739522 2719
9d5c8243
AK		 2720 return 0;
2721}
2722
2723/**
68d480c4 2724 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2725 * @netdev: network interface device structure
2726 *
68d480c4
AD		 2727 * Writes multicast address list to the MTA hash table.
2728 * Returns: -ENOMEM on failure
2729 * 0 on no addresses written
2730 * X on writing X addresses to MTA
9d5c8243 2731 **/
68d480c4 2732static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2733{
2734 struct igb_adapter *adapter = netdev_priv(netdev);
2735 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2736 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2737 u8 *mta_list;
2738 u32 vmolr = 0;
9d5c8243
AK		 2739 int i;
2740
68d480c4
AD		 2741 if (!netdev->mc_count) {
2742 /* nothing to program, so clear mc list */
2743 igb_update_mc_addr_list(hw, NULL, 0);
2744 igb_restore_vf_multicasts(adapter);
2745 return 0;
2746 }
9d5c8243 2747
68d480c4
AD		 2748 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2749 if (!mta_list)
2750 return -ENOMEM;
ff41f8dc 2751
68d480c4
AD		 2752 /* set vmolr receive overflow multicast bit */
2753 vmolr |= E1000_VMOLR_ROMPE;
2754
2755 /* The shared function expects a packed array of only addresses. */
2756 mc_ptr = netdev->mc_list;
2757
2758 for (i = 0; i < netdev->mc_count; i++) {
2759 if (!mc_ptr)
2760 break;
2761 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2762 mc_ptr = mc_ptr->next;
746b9f02 2763 }
68d480c4
AD		 2764 igb_update_mc_addr_list(hw, mta_list, i);
2765 kfree(mta_list);
2766
2767 return netdev->mc_count;
2768}
2769
2770/**
2771 * igb_write_uc_addr_list - write unicast addresses to RAR table
2772 * @netdev: network interface device structure
2773 *
2774 * Writes unicast address list to the RAR table.
2775 * Returns: -ENOMEM on failure/insufficient address space
2776 * 0 on no addresses written
2777 * X on writing X addresses to the RAR table
2778 **/
2779static int igb_write_uc_addr_list(struct net_device *netdev)
2780{
2781 struct igb_adapter *adapter = netdev_priv(netdev);
2782 struct e1000_hw *hw = &adapter->hw;
2783 unsigned int vfn = adapter->vfs_allocated_count;
2784 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2785 int count = 0;
2786
2787 /* return ENOMEM indicating insufficient memory for addresses */
2788 if (netdev->uc.count > rar_entries)
2789 return -ENOMEM;
9d5c8243 2790
ff41f8dc
AD		 2791 if (netdev->uc.count && rar_entries) {
2792 struct netdev_hw_addr *ha;
2793 list_for_each_entry(ha, &netdev->uc.list, list) {
2794 if (!rar_entries)
2795 break;
26ad9178
AD		 2796 igb_rar_set_qsel(adapter, ha->addr,
2797 rar_entries--,
68d480c4
AD		 2798 vfn);
2799 count++;
ff41f8dc
AD		 2800 }
2801 }
2802 /* write the addresses in reverse order to avoid write combining */
2803 for (; rar_entries > 0 ; rar_entries--) {
2804 wr32(E1000_RAH(rar_entries), 0);
2805 wr32(E1000_RAL(rar_entries), 0);
2806 }
2807 wrfl();
2808
68d480c4
AD		 2809 return count;
2810}
2811
2812/**
2813 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2814 * @netdev: network interface device structure
2815 *
2816 * The set_rx_mode entry point is called whenever the unicast or multicast
2817 * address lists or the network interface flags are updated. This routine is
2818 * responsible for configuring the hardware for proper unicast, multicast,
2819 * promiscuous mode, and all-multi behavior.
2820 **/
2821static void igb_set_rx_mode(struct net_device *netdev)
2822{
2823 struct igb_adapter *adapter = netdev_priv(netdev);
2824 struct e1000_hw *hw = &adapter->hw;
2825 unsigned int vfn = adapter->vfs_allocated_count;
2826 u32 rctl, vmolr = 0;
2827 int count;
2828
2829 /* Check for Promiscuous and All Multicast modes */
2830 rctl = rd32(E1000_RCTL);
2831
2832 /* clear the effected bits */
2833 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2834
2835 if (netdev->flags & IFF_PROMISC) {
2836 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2837 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2838 } else {
2839 if (netdev->flags & IFF_ALLMULTI) {
2840 rctl |= E1000_RCTL_MPE;
2841 vmolr |= E1000_VMOLR_MPME;
2842 } else {
2843 /*
2844 * Write addresses to the MTA, if the attempt fails
2845 * then we should just turn on promiscous mode so
2846 * that we can at least receive multicast traffic
2847 */
2848 count = igb_write_mc_addr_list(netdev);
2849 if (count < 0) {
2850 rctl |= E1000_RCTL_MPE;
2851 vmolr |= E1000_VMOLR_MPME;
2852 } else if (count) {
2853 vmolr |= E1000_VMOLR_ROMPE;
2854 }
2855 }
2856 /*
2857 * Write addresses to available RAR registers, if there is not
2858 * sufficient space to store all the addresses then enable
2859 * unicast promiscous mode
2860 */
2861 count = igb_write_uc_addr_list(netdev);
2862 if (count < 0) {
2863 rctl |= E1000_RCTL_UPE;
2864 vmolr |= E1000_VMOLR_ROPE;
2865 }
2866 rctl |= E1000_RCTL_VFE;
28fc06f5 2867 }
68d480c4 2868 wr32(E1000_RCTL, rctl);
28fc06f5 2869
68d480c4
AD		 2870 /*
2871 * In order to support SR-IOV and eventually VMDq it is necessary to set
2872 * the VMOLR to enable the appropriate modes. Without this workaround
2873 * we will have issues with VLAN tag stripping not being done for frames
2874 * that are only arriving because we are the default pool
2875 */
2876 if (hw->mac.type < e1000_82576)
28fc06f5 2877 return;
9d5c8243 2878
68d480c4
AD		 2879 vmolr |= rd32(E1000_VMOLR(vfn)) &
2880 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2881 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2882 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2883}
2884
2885/* Need to wait a few seconds after link up to get diagnostic information from
2886 * the phy */
2887static void igb_update_phy_info(unsigned long data)
2888{
2889 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2890 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2891}
2892
4d6b725e
AD		 2893/**
2894 * igb_has_link - check shared code for link and determine up/down
2895 * @adapter: pointer to driver private info
2896 **/
2897static bool igb_has_link(struct igb_adapter *adapter)
2898{
2899 struct e1000_hw *hw = &adapter->hw;
2900 bool link_active = false;
2901 s32 ret_val = 0;
2902
2903 /* get_link_status is set on LSC (link status) interrupt or
2904 * rx sequence error interrupt. get_link_status will stay
2905 * false until the e1000_check_for_link establishes link
2906 * for copper adapters ONLY
2907 */
2908 switch (hw->phy.media_type) {
2909 case e1000_media_type_copper:
2910 if (hw->mac.get_link_status) {
2911 ret_val = hw->mac.ops.check_for_link(hw);
2912 link_active = !hw->mac.get_link_status;
2913 } else {
2914 link_active = true;
2915 }
2916 break;
4d6b725e
AD		 2917 case e1000_media_type_internal_serdes:
2918 ret_val = hw->mac.ops.check_for_link(hw);
2919 link_active = hw->mac.serdes_has_link;
2920 break;
2921 default:
2922 case e1000_media_type_unknown:
2923 break;
2924 }
2925
2926 return link_active;
2927}
2928
9d5c8243
AK		 2929/**
2930 * igb_watchdog - Timer Call-back
2931 * @data: pointer to adapter cast into an unsigned long
2932 **/
2933static void igb_watchdog(unsigned long data)
2934{
2935 struct igb_adapter *adapter = (struct igb_adapter *)data;
2936 /* Do the rest outside of interrupt context */
2937 schedule_work(&adapter->watchdog_task);
2938}
2939
2940static void igb_watchdog_task(struct work_struct *work)
2941{
2942 struct igb_adapter *adapter = container_of(work,
2943 struct igb_adapter, watchdog_task);
2944 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2945 struct net_device *netdev = adapter->netdev;
2946 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2947 u32 link;
7a6ea550 2948 int i;
9d5c8243 2949
4d6b725e
AD		 2950 link = igb_has_link(adapter);
2951 if ((netif_carrier_ok(netdev)) && link)
9d5c8243
AK		 2952 goto link_up;
2953
9d5c8243
AK		 2954 if (link) {
2955 if (!netif_carrier_ok(netdev)) {
2956 u32 ctrl;
2957 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2958 &adapter->link_speed,
2959 &adapter->link_duplex);
2960
2961 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2962 /* Links status message must follow this format */
2963 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2964 "Flow Control: %s\n",
527d47c1 2965 netdev->name,
9d5c8243
AK		 2966 adapter->link_speed,
2967 adapter->link_duplex == FULL_DUPLEX ?
2968 "Full Duplex" : "Half Duplex",
2969 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2970 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2971 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2972 E1000_CTRL_TFCE) ? "TX" : "None")));
2973
2974 /* tweak tx_queue_len according to speed/duplex and
2975 * adjust the timeout factor */
2976 netdev->tx_queue_len = adapter->tx_queue_len;
2977 adapter->tx_timeout_factor = 1;
2978 switch (adapter->link_speed) {
2979 case SPEED_10:
2980 netdev->tx_queue_len = 10;
2981 adapter->tx_timeout_factor = 14;
2982 break;
2983 case SPEED_100:
2984 netdev->tx_queue_len = 100;
2985 /* maybe add some timeout factor ? */
2986 break;
2987 }
2988
2989 netif_carrier_on(netdev);
9d5c8243 2990
4ae196df
AD		 2991 igb_ping_all_vfs(adapter);
2992
4b1a9877 2993 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2994 if (!test_bit(__IGB_DOWN, &adapter->state))
2995 mod_timer(&adapter->phy_info_timer,
2996 round_jiffies(jiffies + 2 * HZ));
2997 }
2998 } else {
2999 if (netif_carrier_ok(netdev)) {
3000 adapter->link_speed = 0;
3001 adapter->link_duplex = 0;
527d47c1
AD		 3002 /* Links status message must follow this format */
3003 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3004 netdev->name);
9d5c8243 3005 netif_carrier_off(netdev);
4b1a9877 3006
4ae196df
AD		 3007 igb_ping_all_vfs(adapter);
3008
4b1a9877 3009 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3010 if (!test_bit(__IGB_DOWN, &adapter->state))
3011 mod_timer(&adapter->phy_info_timer,
3012 round_jiffies(jiffies + 2 * HZ));
3013 }
3014 }
3015
3016link_up:
3017 igb_update_stats(adapter);
3018
4b1a9877 3019 hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
9d5c8243 3020 adapter->tpt_old = adapter->stats.tpt;
4b1a9877 3021 hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
9d5c8243
AK		 3022 adapter->colc_old = adapter->stats.colc;
3023
3024 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
3025 adapter->gorc_old = adapter->stats.gorc;
3026 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
3027 adapter->gotc_old = adapter->stats.gotc;
3028
3029 igb_update_adaptive(&adapter->hw);
3030
3031 if (!netif_carrier_ok(netdev)) {
c493ea45 3032 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 3033 /* We've lost link, so the controller stops DMA,
3034 * but we've got queued Tx work that's never going
3035 * to get done, so reset controller to flush Tx.
3036 * (Do the reset outside of interrupt context). */
3037 adapter->tx_timeout_count++;
3038 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 3039 /* return immediately since reset is imminent */
3040 return;
9d5c8243
AK		 3041 }
3042 }
3043
3044 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3045 if (adapter->msix_entries) {
047e0030
AD		 3046 u32 eics = 0;
3047 for (i = 0; i < adapter->num_q_vectors; i++) {
3048 struct igb_q_vector *q_vector = adapter->q_vector[i];
3049 eics |= q_vector->eims_value;
3050 }
7a6ea550
AD		 3051 wr32(E1000_EICS, eics);
3052 } else {
3053 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3054 }
9d5c8243
AK		 3055
3056 /* Force detection of hung controller every watchdog period */
3057 tx_ring->detect_tx_hung = true;
3058
3059 /* Reset the timer */
3060 if (!test_bit(__IGB_DOWN, &adapter->state))
3061 mod_timer(&adapter->watchdog_timer,
3062 round_jiffies(jiffies + 2 * HZ));
3063}
3064
3065enum latency_range {
3066 lowest_latency = 0,
3067 low_latency = 1,
3068 bulk_latency = 2,
3069 latency_invalid = 255
3070};
3071
3072
6eb5a7f1
AD		 3073/**
3074 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3075 *
3076 * Stores a new ITR value based on strictly on packet size. This
3077 * algorithm is less sophisticated than that used in igb_update_itr,
3078 * due to the difficulty of synchronizing statistics across multiple
3079 * receive rings. The divisors and thresholds used by this fuction
3080 * were determined based on theoretical maximum wire speed and testing
3081 * data, in order to minimize response time while increasing bulk
3082 * throughput.
3083 * This functionality is controlled by the InterruptThrottleRate module
3084 * parameter (see igb_param.c)
3085 * NOTE: This function is called only when operating in a multiqueue
3086 * receive environment.
047e0030 3087 * @q_vector: pointer to q_vector
6eb5a7f1 3088 **/
047e0030 3089static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3090{
047e0030 3091 int new_val = q_vector->itr_val;
6eb5a7f1 3092 int avg_wire_size = 0;
047e0030 3093 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3094
6eb5a7f1
AD		 3095 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3096 * ints/sec - ITR timer value of 120 ticks.
3097 */
3098 if (adapter->link_speed != SPEED_1000) {
047e0030 3099 new_val = 976;
6eb5a7f1 3100 goto set_itr_val;
9d5c8243 3101 }
047e0030
AD		 3102
3103 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3104 struct igb_ring *ring = q_vector->rx_ring;
3105 avg_wire_size = ring->total_bytes / ring->total_packets;
3106 }
3107
3108 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3109 struct igb_ring *ring = q_vector->tx_ring;
3110 avg_wire_size = max_t(u32, avg_wire_size,
3111 (ring->total_bytes /
3112 ring->total_packets));
3113 }
3114
3115 /* if avg_wire_size isn't set no work was done */
3116 if (!avg_wire_size)
3117 goto clear_counts;
9d5c8243 3118
6eb5a7f1
AD		 3119 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3120 avg_wire_size += 24;
3121
3122 /* Don't starve jumbo frames */
3123 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3124
6eb5a7f1
AD		 3125 /* Give a little boost to mid-size frames */
3126 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3127 new_val = avg_wire_size / 3;
3128 else
3129 new_val = avg_wire_size / 2;
9d5c8243 3130
6eb5a7f1 3131set_itr_val:
047e0030
AD		 3132 if (new_val != q_vector->itr_val) {
3133 q_vector->itr_val = new_val;
3134 q_vector->set_itr = 1;
9d5c8243 3135 }
6eb5a7f1 3136clear_counts:
047e0030
AD		 3137 if (q_vector->rx_ring) {
3138 q_vector->rx_ring->total_bytes = 0;
3139 q_vector->rx_ring->total_packets = 0;
3140 }
3141 if (q_vector->tx_ring) {
3142 q_vector->tx_ring->total_bytes = 0;
3143 q_vector->tx_ring->total_packets = 0;
3144 }
9d5c8243
AK		 3145}
3146
3147/**
3148 * igb_update_itr - update the dynamic ITR value based on statistics
3149 * Stores a new ITR value based on packets and byte
3150 * counts during the last interrupt. The advantage of per interrupt
3151 * computation is faster updates and more accurate ITR for the current
3152 * traffic pattern. Constants in this function were computed
3153 * based on theoretical maximum wire speed and thresholds were set based
3154 * on testing data as well as attempting to minimize response time
3155 * while increasing bulk throughput.
3156 * this functionality is controlled by the InterruptThrottleRate module
3157 * parameter (see igb_param.c)
3158 * NOTE: These calculations are only valid when operating in a single-
3159 * queue environment.
3160 * @adapter: pointer to adapter
047e0030 3161 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3162 * @packets: the number of packets during this measurement interval
3163 * @bytes: the number of bytes during this measurement interval
3164 **/
3165static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3166 int packets, int bytes)
3167{
3168 unsigned int retval = itr_setting;
3169
3170 if (packets == 0)
3171 goto update_itr_done;
3172
3173 switch (itr_setting) {
3174 case lowest_latency:
3175 /* handle TSO and jumbo frames */
3176 if (bytes/packets > 8000)
3177 retval = bulk_latency;
3178 else if ((packets < 5) && (bytes > 512))
3179 retval = low_latency;
3180 break;
3181 case low_latency: /* 50 usec aka 20000 ints/s */
3182 if (bytes > 10000) {
3183 /* this if handles the TSO accounting */
3184 if (bytes/packets > 8000) {
3185 retval = bulk_latency;
3186 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3187 retval = bulk_latency;
3188 } else if ((packets > 35)) {
3189 retval = lowest_latency;
3190 }
3191 } else if (bytes/packets > 2000) {
3192 retval = bulk_latency;
3193 } else if (packets <= 2 && bytes < 512) {
3194 retval = lowest_latency;
3195 }
3196 break;
3197 case bulk_latency: /* 250 usec aka 4000 ints/s */
3198 if (bytes > 25000) {
3199 if (packets > 35)
3200 retval = low_latency;
1e5c3d21 3201 } else if (bytes < 1500) {
9d5c8243
AK		 3202 retval = low_latency;
3203 }
3204 break;
3205 }
3206
3207update_itr_done:
3208 return retval;
3209}
3210
6eb5a7f1 3211static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3212{
047e0030 3213 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3214 u16 current_itr;
047e0030 3215 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3216
3217 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3218 if (adapter->link_speed != SPEED_1000) {
3219 current_itr = 0;
3220 new_itr = 4000;
3221 goto set_itr_now;
3222 }
3223
3224 adapter->rx_itr = igb_update_itr(adapter,
3225 adapter->rx_itr,
3226 adapter->rx_ring->total_packets,
3227 adapter->rx_ring->total_bytes);
9d5c8243 3228
047e0030
AD		 3229 adapter->tx_itr = igb_update_itr(adapter,
3230 adapter->tx_itr,
3231 adapter->tx_ring->total_packets,
3232 adapter->tx_ring->total_bytes);
3233 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3234
6eb5a7f1 3235 /* conservative mode (itr 3) eliminates the lowest_latency setting */
73cd78f1 3236 if (adapter->itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3237 current_itr = low_latency;
3238
9d5c8243
AK		 3239 switch (current_itr) {
3240 /* counts and packets in update_itr are dependent on these numbers */
3241 case lowest_latency:
78b1f607 3242 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3243 break;
3244 case low_latency:
78b1f607 3245 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3246 break;
3247 case bulk_latency:
78b1f607 3248 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3249 break;
3250 default:
3251 break;
3252 }
3253
3254set_itr_now:
6eb5a7f1
AD		 3255 adapter->rx_ring->total_bytes = 0;
3256 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3257 adapter->tx_ring->total_bytes = 0;
3258 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3259
047e0030 3260 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3261 /* this attempts to bias the interrupt rate towards Bulk
3262 * by adding intermediate steps when interrupt rate is
3263 * increasing */
047e0030
AD		 3264 new_itr = new_itr > q_vector->itr_val ?
3265 max((new_itr * q_vector->itr_val) /
3266 (new_itr + (q_vector->itr_val >> 2)),
3267 new_itr) :
9d5c8243
AK		 3268 new_itr;
3269 /* Don't write the value here; it resets the adapter's
3270 * internal timer, and causes us to delay far longer than
3271 * we should between interrupts. Instead, we write the ITR
3272 * value at the beginning of the next interrupt so the timing
3273 * ends up being correct.
3274 */
047e0030
AD		 3275 q_vector->itr_val = new_itr;
3276 q_vector->set_itr = 1;
9d5c8243
AK		 3277 }
3278
3279 return;
3280}
3281
9d5c8243
AK		 3282#define IGB_TX_FLAGS_CSUM 0x00000001
3283#define IGB_TX_FLAGS_VLAN 0x00000002
3284#define IGB_TX_FLAGS_TSO 0x00000004
3285#define IGB_TX_FLAGS_IPV4 0x00000008
33af6bcc 3286#define IGB_TX_FLAGS_TSTAMP 0x00000010
9d5c8243
AK		 3287#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3288#define IGB_TX_FLAGS_VLAN_SHIFT 16
3289
85ad76b2 3290static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3291 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3292{
3293 struct e1000_adv_tx_context_desc *context_desc;
3294 unsigned int i;
3295 int err;
3296 struct igb_buffer *buffer_info;
3297 u32 info = 0, tu_cmd = 0;
3298 u32 mss_l4len_idx, l4len;
3299 *hdr_len = 0;
3300
3301 if (skb_header_cloned(skb)) {
3302 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3303 if (err)
3304 return err;
3305 }
3306
3307 l4len = tcp_hdrlen(skb);
3308 *hdr_len += l4len;
3309
3310 if (skb->protocol == htons(ETH_P_IP)) {
3311 struct iphdr *iph = ip_hdr(skb);
3312 iph->tot_len = 0;
3313 iph->check = 0;
3314 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3315 iph->daddr, 0,
3316 IPPROTO_TCP,
3317 0);
3318 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3319 ipv6_hdr(skb)->payload_len = 0;
3320 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3321 &ipv6_hdr(skb)->daddr,
3322 0, IPPROTO_TCP, 0);
3323 }
3324
3325 i = tx_ring->next_to_use;
3326
3327 buffer_info = &tx_ring->buffer_info[i];
3328 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3329 /* VLAN MACLEN IPLEN */
3330 if (tx_flags & IGB_TX_FLAGS_VLAN)
3331 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3332 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3333 *hdr_len += skb_network_offset(skb);
3334 info |= skb_network_header_len(skb);
3335 *hdr_len += skb_network_header_len(skb);
3336 context_desc->vlan_macip_lens = cpu_to_le32(info);
3337
3338 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3339 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3340
3341 if (skb->protocol == htons(ETH_P_IP))
3342 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3343 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3344
3345 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3346
3347 /* MSS L4LEN IDX */
3348 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3349 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3350
73cd78f1 3351 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3352 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3353 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3354
3355 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3356 context_desc->seqnum_seed = 0;
3357
3358 buffer_info->time_stamp = jiffies;
0e014cb1 3359 buffer_info->next_to_watch = i;
9d5c8243
AK		 3360 buffer_info->dma = 0;
3361 i++;
3362 if (i == tx_ring->count)
3363 i = 0;
3364
3365 tx_ring->next_to_use = i;
3366
3367 return true;
3368}
3369
85ad76b2
AD		 3370static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3371 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3372{
3373 struct e1000_adv_tx_context_desc *context_desc;
80785298 3374 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3375 struct igb_buffer *buffer_info;
3376 u32 info = 0, tu_cmd = 0;
80785298 3377 unsigned int i;
9d5c8243
AK		 3378
3379 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3380 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3381 i = tx_ring->next_to_use;
3382 buffer_info = &tx_ring->buffer_info[i];
3383 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3384
3385 if (tx_flags & IGB_TX_FLAGS_VLAN)
3386 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3387 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3388 if (skb->ip_summed == CHECKSUM_PARTIAL)
3389 info |= skb_network_header_len(skb);
3390
3391 context_desc->vlan_macip_lens = cpu_to_le32(info);
3392
3393 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3394
3395 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3396 __be16 protocol;
3397
3398 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3399 const struct vlan_ethhdr *vhdr =
3400 (const struct vlan_ethhdr*)skb->data;
3401
3402 protocol = vhdr->h_vlan_encapsulated_proto;
3403 } else {
3404 protocol = skb->protocol;
3405 }
3406
3407 switch (protocol) {
09640e63 3408 case cpu_to_be16(ETH_P_IP):
9d5c8243 3409 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3410 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3411 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3412 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3413 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3414 break;
09640e63 3415 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3416 /* XXX what about other V6 headers?? */
3417 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3418 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3419 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3420 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3421 break;
3422 default:
3423 if (unlikely(net_ratelimit()))
80785298 3424 dev_warn(&pdev->dev,
44b0cda3
MW		 3425 "partial checksum but proto=%x!\n",
3426 skb->protocol);
3427 break;
3428 }
9d5c8243
AK		 3429 }
3430
3431 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3432 context_desc->seqnum_seed = 0;
85ad76b2 3433 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3434 context_desc->mss_l4len_idx =
85ad76b2 3435 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3436
3437 buffer_info->time_stamp = jiffies;
0e014cb1 3438 buffer_info->next_to_watch = i;
9d5c8243
AK		 3439 buffer_info->dma = 0;
3440
3441 i++;
3442 if (i == tx_ring->count)
3443 i = 0;
3444 tx_ring->next_to_use = i;
3445
3446 return true;
3447 }
9d5c8243
AK		 3448 return false;
3449}
3450
3451#define IGB_MAX_TXD_PWR 16
3452#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3453
80785298 3454static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3455 unsigned int first)
9d5c8243
AK		 3456{
3457 struct igb_buffer *buffer_info;
80785298 3458 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3459 unsigned int len = skb_headlen(skb);
3460 unsigned int count = 0, i;
3461 unsigned int f;
65689fef 3462 dma_addr_t *map;
9d5c8243
AK		 3463
3464 i = tx_ring->next_to_use;
3465
80785298
AD		 3466 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3467 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3468 return 0;
3469 }
3470
3471 map = skb_shinfo(skb)->dma_maps;
3472
9d5c8243
AK		 3473 buffer_info = &tx_ring->buffer_info[i];
3474 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3475 buffer_info->length = len;
3476 /* set time_stamp *before* dma to help avoid a possible race */
3477 buffer_info->time_stamp = jiffies;
0e014cb1 3478 buffer_info->next_to_watch = i;
042a53a9 3479 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3480
3481 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3482 struct skb_frag_struct *frag;
3483
65689fef
AD		 3484 i++;
3485 if (i == tx_ring->count)
3486 i = 0;
3487
9d5c8243
AK		 3488 frag = &skb_shinfo(skb)->frags[f];
3489 len = frag->size;
3490
3491 buffer_info = &tx_ring->buffer_info[i];
3492 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3493 buffer_info->length = len;
3494 buffer_info->time_stamp = jiffies;
0e014cb1 3495 buffer_info->next_to_watch = i;
65689fef 3496 buffer_info->dma = map[count];
9d5c8243 3497 count++;
9d5c8243
AK		 3498 }
3499
9d5c8243 3500 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3501 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3502
042a53a9 3503 return count + 1;
9d5c8243
AK		 3504}
3505
85ad76b2 3506static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3507 int tx_flags, int count, u32 paylen,
3508 u8 hdr_len)
3509{
3510 union e1000_adv_tx_desc *tx_desc = NULL;
3511 struct igb_buffer *buffer_info;
3512 u32 olinfo_status = 0, cmd_type_len;
3513 unsigned int i;
3514
3515 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3516 E1000_ADVTXD_DCMD_DEXT);
3517
3518 if (tx_flags & IGB_TX_FLAGS_VLAN)
3519 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3520
33af6bcc
PO		 3521 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3522 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3523
9d5c8243
AK		 3524 if (tx_flags & IGB_TX_FLAGS_TSO) {
3525 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3526
3527 /* insert tcp checksum */
3528 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3529
3530 /* insert ip checksum */
3531 if (tx_flags & IGB_TX_FLAGS_IPV4)
3532 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3533
3534 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3535 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3536 }
3537
85ad76b2
AD		 3538 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3539 (tx_flags & (IGB_TX_FLAGS_CSUM |
3540 IGB_TX_FLAGS_TSO |
7dfc16fa 3541 IGB_TX_FLAGS_VLAN)))
85ad76b2 3542 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3543
3544 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3545
3546 i = tx_ring->next_to_use;
3547 while (count--) {
3548 buffer_info = &tx_ring->buffer_info[i];
3549 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3550 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3551 tx_desc->read.cmd_type_len =
3552 cpu_to_le32(cmd_type_len | buffer_info->length);
3553 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3554 i++;
3555 if (i == tx_ring->count)
3556 i = 0;
3557 }
3558
85ad76b2 3559 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3560 /* Force memory writes to complete before letting h/w
3561 * know there are new descriptors to fetch. (Only
3562 * applicable for weak-ordered memory model archs,
3563 * such as IA-64). */
3564 wmb();
3565
3566 tx_ring->next_to_use = i;
fce99e34 3567 writel(i, tx_ring->tail);
9d5c8243
AK		 3568 /* we need this if more than one processor can write to our tail
3569 * at a time, it syncronizes IO on IA64/Altix systems */
3570 mmiowb();
3571}
3572
e694e964 3573static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3574{
e694e964
AD		 3575 struct net_device *netdev = tx_ring->netdev;
3576
661086df 3577 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3578
9d5c8243
AK		 3579 /* Herbert's original patch had:
3580 * smp_mb__after_netif_stop_queue();
3581 * but since that doesn't exist yet, just open code it. */
3582 smp_mb();
3583
3584 /* We need to check again in a case another CPU has just
3585 * made room available. */
c493ea45 3586 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3587 return -EBUSY;
3588
3589 /* A reprieve! */
661086df 3590 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3591 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3592 return 0;
3593}
3594
e694e964 3595static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3596{
c493ea45 3597 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3598 return 0;
e694e964 3599 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3600}
3601
3b29a56d 3602static netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3b29a56d 3603 struct igb_ring *tx_ring)
9d5c8243 3604{
e694e964 3605 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3606 unsigned int first;
9d5c8243 3607 unsigned int tx_flags = 0;
9d5c8243 3608 u8 hdr_len = 0;
65689fef 3609 int count = 0;
9d5c8243 3610 int tso = 0;
33af6bcc 3611 union skb_shared_tx *shtx;
9d5c8243 3612
9d5c8243
AK		 3613 if (test_bit(__IGB_DOWN, &adapter->state)) {
3614 dev_kfree_skb_any(skb);
3615 return NETDEV_TX_OK;
3616 }
3617
3618 if (skb->len <= 0) {
3619 dev_kfree_skb_any(skb);
3620 return NETDEV_TX_OK;
3621 }
3622
9d5c8243
AK		 3623 /* need: 1 descriptor per page,
3624 * + 2 desc gap to keep tail from touching head,
3625 * + 1 desc for skb->data,
3626 * + 1 desc for context descriptor,
3627 * otherwise try next time */
e694e964 3628 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3629 /* this is a hard error */
9d5c8243
AK		 3630 return NETDEV_TX_BUSY;
3631 }
33af6bcc
PO		 3632
3633 /*
3634 * TODO: check that there currently is no other packet with
3635 * time stamping in the queue
3636 *
3637 * When doing time stamping, keep the connection to the socket
3638 * a while longer: it is still needed by skb_hwtstamp_tx(),
3639 * called either in igb_tx_hwtstamp() or by our caller when
3640 * doing software time stamping.
3641 */
3642 shtx = skb_tx(skb);
3643 if (unlikely(shtx->hardware)) {
3644 shtx->in_progress = 1;
3645 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3646 }
9d5c8243
AK		 3647
3648 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
3649 tx_flags |= IGB_TX_FLAGS_VLAN;
3650 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3651 }
3652
661086df
PWJ		 3653 if (skb->protocol == htons(ETH_P_IP))
3654 tx_flags |= IGB_TX_FLAGS_IPV4;
3655
0e014cb1 3656 first = tx_ring->next_to_use;
85ad76b2
AD		 3657 if (skb_is_gso(skb)) {
3658 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3659 if (tso < 0) {
3660 dev_kfree_skb_any(skb);
3661 return NETDEV_TX_OK;
3662 }
9d5c8243
AK		 3663 }
3664
3665 if (tso)
3666 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3667 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3668 (skb->ip_summed == CHECKSUM_PARTIAL))
3669 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3670
65689fef
AD		 3671 /*
3672 * count reflects descriptors mapped, if 0 then mapping error
3673 * has occured and we need to rewind the descriptor queue
3674 */
80785298 3675 count = igb_tx_map_adv(tx_ring, skb, first);
65689fef 3676
85ad76b2 3677 if (!count) {
65689fef
AD		 3678 dev_kfree_skb_any(skb);
3679 tx_ring->buffer_info[first].time_stamp = 0;
3680 tx_ring->next_to_use = first;
85ad76b2 3681 return NETDEV_TX_OK;
65689fef 3682 }
9d5c8243 3683
85ad76b2
AD		 3684 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3685
3686 /* Make sure there is space in the ring for the next send. */
e694e964 3687 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3688
9d5c8243
AK		 3689 return NETDEV_TX_OK;
3690}
3691
3b29a56d
SH		 3692static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3693 struct net_device *netdev)
9d5c8243
AK		 3694{
3695 struct igb_adapter *adapter = netdev_priv(netdev);
661086df
PWJ		 3696 struct igb_ring *tx_ring;
3697
661086df 3698 int r_idx = 0;
1bfaf07b 3699 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3700 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3701
3702 /* This goes back to the question of how to logically map a tx queue
3703 * to a flow. Right now, performance is impacted slightly negatively
3704 * if using multiple tx queues. If the stack breaks away from a
3705 * single qdisc implementation, we can look at this again. */
e694e964 3706 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3707}
3708
3709/**
3710 * igb_tx_timeout - Respond to a Tx Hang
3711 * @netdev: network interface device structure
3712 **/
3713static void igb_tx_timeout(struct net_device *netdev)
3714{
3715 struct igb_adapter *adapter = netdev_priv(netdev);
3716 struct e1000_hw *hw = &adapter->hw;
3717
3718 /* Do the reset outside of interrupt context */
3719 adapter->tx_timeout_count++;
3720 schedule_work(&adapter->reset_task);
265de409
AD		 3721 wr32(E1000_EICS,
3722 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3723}
3724
3725static void igb_reset_task(struct work_struct *work)
3726{
3727 struct igb_adapter *adapter;
3728 adapter = container_of(work, struct igb_adapter, reset_task);
3729
3730 igb_reinit_locked(adapter);
3731}
3732
3733/**
3734 * igb_get_stats - Get System Network Statistics
3735 * @netdev: network interface device structure
3736 *
3737 * Returns the address of the device statistics structure.
3738 * The statistics are actually updated from the timer callback.
3739 **/
73cd78f1 3740static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3741{
9d5c8243 3742 /* only return the current stats */
8d24e933 3743 return &netdev->stats;
9d5c8243
AK		 3744}
3745
3746/**
3747 * igb_change_mtu - Change the Maximum Transfer Unit
3748 * @netdev: network interface device structure
3749 * @new_mtu: new value for maximum frame size
3750 *
3751 * Returns 0 on success, negative on failure
3752 **/
3753static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3754{
3755 struct igb_adapter *adapter = netdev_priv(netdev);
3756 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3757 u32 rx_buffer_len, i;
9d5c8243
AK		 3758
3759 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3760 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3761 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3762 return -EINVAL;
3763 }
3764
9d5c8243
AK		 3765 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3766 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3767 return -EINVAL;
3768 }
3769
3770 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3771 msleep(1);
73cd78f1 3772
9d5c8243
AK		 3773 /* igb_down has a dependency on max_frame_size */
3774 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3775 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3776 * means we reserve 2 more, this pushes us to allocate from the next
3777 * larger slab size.
3778 * i.e. RXBUFFER_2048 --> size-4096 slab
3779 */
3780
7d95b717 3781 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3782 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3783 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3784 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3785 else
4c844851
AD		 3786 rx_buffer_len = IGB_RXBUFFER_128;
3787
3788 if (netif_running(netdev))
3789 igb_down(adapter);
9d5c8243
AK		 3790
3791 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3792 netdev->mtu, new_mtu);
3793 netdev->mtu = new_mtu;
3794
4c844851
AD		 3795 for (i = 0; i < adapter->num_rx_queues; i++)
3796 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3797
9d5c8243
AK		 3798 if (netif_running(netdev))
3799 igb_up(adapter);
3800 else
3801 igb_reset(adapter);
3802
3803 clear_bit(__IGB_RESETTING, &adapter->state);
3804
3805 return 0;
3806}
3807
3808/**
3809 * igb_update_stats - Update the board statistics counters
3810 * @adapter: board private structure
3811 **/
3812
3813void igb_update_stats(struct igb_adapter *adapter)
3814{
8d24e933 3815 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3816 struct e1000_hw *hw = &adapter->hw;
3817 struct pci_dev *pdev = adapter->pdev;
3818 u16 phy_tmp;
3819
3820#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3821
3822 /*
3823 * Prevent stats update while adapter is being reset, or if the pci
3824 * connection is down.
3825 */
3826 if (adapter->link_speed == 0)
3827 return;
3828 if (pci_channel_offline(pdev))
3829 return;
3830
3831 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3832 adapter->stats.gprc += rd32(E1000_GPRC);
3833 adapter->stats.gorc += rd32(E1000_GORCL);
3834 rd32(E1000_GORCH); /* clear GORCL */
3835 adapter->stats.bprc += rd32(E1000_BPRC);
3836 adapter->stats.mprc += rd32(E1000_MPRC);
3837 adapter->stats.roc += rd32(E1000_ROC);
3838
3839 adapter->stats.prc64 += rd32(E1000_PRC64);
3840 adapter->stats.prc127 += rd32(E1000_PRC127);
3841 adapter->stats.prc255 += rd32(E1000_PRC255);
3842 adapter->stats.prc511 += rd32(E1000_PRC511);
3843 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3844 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3845 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3846 adapter->stats.sec += rd32(E1000_SEC);
3847
3848 adapter->stats.mpc += rd32(E1000_MPC);
3849 adapter->stats.scc += rd32(E1000_SCC);
3850 adapter->stats.ecol += rd32(E1000_ECOL);
3851 adapter->stats.mcc += rd32(E1000_MCC);
3852 adapter->stats.latecol += rd32(E1000_LATECOL);
3853 adapter->stats.dc += rd32(E1000_DC);
3854 adapter->stats.rlec += rd32(E1000_RLEC);
3855 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3856 adapter->stats.xontxc += rd32(E1000_XONTXC);
3857 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3858 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3859 adapter->stats.fcruc += rd32(E1000_FCRUC);
3860 adapter->stats.gptc += rd32(E1000_GPTC);
3861 adapter->stats.gotc += rd32(E1000_GOTCL);
3862 rd32(E1000_GOTCH); /* clear GOTCL */
3863 adapter->stats.rnbc += rd32(E1000_RNBC);
3864 adapter->stats.ruc += rd32(E1000_RUC);
3865 adapter->stats.rfc += rd32(E1000_RFC);
3866 adapter->stats.rjc += rd32(E1000_RJC);
3867 adapter->stats.tor += rd32(E1000_TORH);
3868 adapter->stats.tot += rd32(E1000_TOTH);
3869 adapter->stats.tpr += rd32(E1000_TPR);
3870
3871 adapter->stats.ptc64 += rd32(E1000_PTC64);
3872 adapter->stats.ptc127 += rd32(E1000_PTC127);
3873 adapter->stats.ptc255 += rd32(E1000_PTC255);
3874 adapter->stats.ptc511 += rd32(E1000_PTC511);
3875 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3876 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3877
3878 adapter->stats.mptc += rd32(E1000_MPTC);
3879 adapter->stats.bptc += rd32(E1000_BPTC);
3880
3881 /* used for adaptive IFS */
3882
3883 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3884 adapter->stats.tpt += hw->mac.tx_packet_delta;
3885 hw->mac.collision_delta = rd32(E1000_COLC);
3886 adapter->stats.colc += hw->mac.collision_delta;
3887
3888 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3889 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3890 adapter->stats.tncrs += rd32(E1000_TNCRS);
3891 adapter->stats.tsctc += rd32(E1000_TSCTC);
3892 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3893
3894 adapter->stats.iac += rd32(E1000_IAC);
3895 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3896 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3897 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3898 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3899 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3900 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3901 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3902 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3903
3904 /* Fill out the OS statistics structure */
8d24e933
AK		 3905 netdev->stats.multicast = adapter->stats.mprc;
3906 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3907
3908 /* Rx Errors */
3909
8c0ab70a
JDB		 3910 if (hw->mac.type != e1000_82575) {
3911 u32 rqdpc_tmp;
3ea73afa 3912 u64 rqdpc_total = 0;
8c0ab70a
JDB		 3913 int i;
3914 /* Read out drops stats per RX queue. Notice RQDPC (Receive
3915 * Queue Drop Packet Count) stats only gets incremented, if
3916 * the DROP_EN but it set (in the SRRCTL register for that
3917 * queue). If DROP_EN bit is NOT set, then the some what
3918 * equivalent count is stored in RNBC (not per queue basis).
3919 * Also note the drop count is due to lack of available
3920 * descriptors.
3921 */
3922 for (i = 0; i < adapter->num_rx_queues; i++) {
3923 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0xFFF;
3924 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3ea73afa 3925 rqdpc_total += adapter->rx_ring[i].rx_stats.drops;
8c0ab70a 3926 }
8d24e933 3927 netdev->stats.rx_fifo_errors = rqdpc_total;
8c0ab70a
JDB		 3928 }
3929
3ea73afa
JDB		 3930 /* Note RNBC (Receive No Buffers Count) is an not an exact
3931 * drop count as the hardware FIFO might save the day. Thats
3932 * one of the reason for saving it in rx_fifo_errors, as its
3933 * potentially not a true drop.
3934 */
8d24e933 3935 netdev->stats.rx_fifo_errors += adapter->stats.rnbc;
3ea73afa 3936
9d5c8243 3937 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3938 * our own version based on RUC and ROC */
8d24e933 3939 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3940 adapter->stats.crcerrs + adapter->stats.algnerrc +
3941 adapter->stats.ruc + adapter->stats.roc +
3942 adapter->stats.cexterr;
8d24e933 3943 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3944 adapter->stats.roc;
8d24e933
AK		 3945 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3946 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3947 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3948
3949 /* Tx Errors */
8d24e933 3950 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3951 adapter->stats.latecol;
8d24e933
AK		 3952 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3953 netdev->stats.tx_window_errors = adapter->stats.latecol;
3954 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3955
3956 /* Tx Dropped needs to be maintained elsewhere */
3957
3958 /* Phy Stats */
3959 if (hw->phy.media_type == e1000_media_type_copper) {
3960 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3961 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3962 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3963 adapter->phy_stats.idle_errors += phy_tmp;
3964 }
3965 }
3966
3967 /* Management Stats */
3968 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3969 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3970 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3971}
3972
9d5c8243
AK		 3973static irqreturn_t igb_msix_other(int irq, void *data)
3974{
047e0030 3975 struct igb_adapter *adapter = data;
9d5c8243 3976 struct e1000_hw *hw = &adapter->hw;
844290e5 3977 u32 icr = rd32(E1000_ICR);
844290e5 3978 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3979
047e0030 3980 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3981 /* HW is reporting DMA is out of sync */
3982 adapter->stats.doosync++;
3983 }
eebbbdba 3984
4ae196df
AD		 3985 /* Check for a mailbox event */
3986 if (icr & E1000_ICR_VMMB)
3987 igb_msg_task(adapter);
3988
3989 if (icr & E1000_ICR_LSC) {
3990 hw->mac.get_link_status = 1;
3991 /* guard against interrupt when we're going down */
3992 if (!test_bit(__IGB_DOWN, &adapter->state))
3993 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3994 }
3995
3996 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_VMMB);
844290e5 3997 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3998
3999 return IRQ_HANDLED;
4000}
4001
047e0030 4002static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4003{
047e0030 4004 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4005
047e0030
AD		 4006 if (!q_vector->set_itr)
4007 return;
73cd78f1 4008
047e0030
AD		 4009 if (!itr_val)
4010 itr_val = 0x4;
661086df 4011
047e0030
AD		 4012 if (q_vector->itr_shift)
4013 itr_val |= itr_val << q_vector->itr_shift;
661086df 4014 else
047e0030 4015 itr_val |= 0x8000000;
661086df 4016
047e0030
AD		 4017 writel(itr_val, q_vector->itr_register);
4018 q_vector->set_itr = 0;
6eb5a7f1
AD		 4019}
4020
047e0030 4021static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4022{
047e0030 4023 struct igb_q_vector *q_vector = data;
9d5c8243 4024
047e0030
AD		 4025 /* Write the ITR value calculated from the previous interrupt. */
4026 igb_write_itr(q_vector);
9d5c8243 4027
047e0030 4028 napi_schedule(&q_vector->napi);
844290e5 4029
047e0030 4030 return IRQ_HANDLED;
fe4506b6
JC		 4031}
4032
421e02f0 4033#ifdef CONFIG_IGB_DCA
047e0030 4034static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4035{
047e0030 4036 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4037 struct e1000_hw *hw = &adapter->hw;
4038 int cpu = get_cpu();
fe4506b6 4039
047e0030
AD		 4040 if (q_vector->cpu == cpu)
4041 goto out_no_update;
4042
4043 if (q_vector->tx_ring) {
4044 int q = q_vector->tx_ring->reg_idx;
4045 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4046 if (hw->mac.type == e1000_82575) {
4047 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4048 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4049 } else {
047e0030
AD		 4050 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4051 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4052 E1000_DCA_TXCTRL_CPUID_SHIFT;
4053 }
4054 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4055 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4056 }
4057 if (q_vector->rx_ring) {
4058 int q = q_vector->rx_ring->reg_idx;
4059 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4060 if (hw->mac.type == e1000_82575) {
2d064c06 4061 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4062 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4063 } else {
4064 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4065 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4066 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4067 }
fe4506b6
JC		 4068 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4069 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4070 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4071 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4072 }
047e0030
AD		 4073 q_vector->cpu = cpu;
4074out_no_update:
fe4506b6
JC		 4075 put_cpu();
4076}
4077
4078static void igb_setup_dca(struct igb_adapter *adapter)
4079{
7e0e99ef 4080 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4081 int i;
4082
7dfc16fa 4083 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4084 return;
4085
7e0e99ef
AD		 4086 /* Always use CB2 mode, difference is masked in the CB driver. */
4087 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4088
047e0030
AD		 4089 for (i = 0; i < adapter->num_q_vectors; i++) {
4090 struct igb_q_vector *q_vector = adapter->q_vector[i];
4091 q_vector->cpu = -1;
4092 igb_update_dca(q_vector);
fe4506b6
JC		 4093 }
4094}
4095
4096static int __igb_notify_dca(struct device *dev, void *data)
4097{
4098 struct net_device *netdev = dev_get_drvdata(dev);
4099 struct igb_adapter *adapter = netdev_priv(netdev);
4100 struct e1000_hw *hw = &adapter->hw;
4101 unsigned long event = *(unsigned long *)data;
4102
4103 switch (event) {
4104 case DCA_PROVIDER_ADD:
4105 /* if already enabled, don't do it again */
7dfc16fa 4106 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4107 break;
fe4506b6
JC		 4108 /* Always use CB2 mode, difference is masked
4109 * in the CB driver. */
cbd347ad 4110 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
fe4506b6 4111 if (dca_add_requester(dev) == 0) {
bbd98fe4 4112 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6
JC		 4113 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4114 igb_setup_dca(adapter);
4115 break;
4116 }
4117 /* Fall Through since DCA is disabled. */
4118 case DCA_PROVIDER_REMOVE:
7dfc16fa 4119 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4120 /* without this a class_device is left
047e0030 4121 * hanging around in the sysfs model */
fe4506b6
JC		 4122 dca_remove_requester(dev);
4123 dev_info(&adapter->pdev->dev, "DCA disabled\n");
7dfc16fa 4124 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4125 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4126 }
4127 break;
4128 }
bbd98fe4 4129
fe4506b6 4130 return 0;
9d5c8243
AK		 4131}
4132
fe4506b6
JC		 4133static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4134 void *p)
4135{
4136 int ret_val;
4137
4138 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4139 __igb_notify_dca);
4140
4141 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4142}
421e02f0 4143#endif /* CONFIG_IGB_DCA */
9d5c8243 4144
4ae196df
AD		 4145static void igb_ping_all_vfs(struct igb_adapter *adapter)
4146{
4147 struct e1000_hw *hw = &adapter->hw;
4148 u32 ping;
4149 int i;
4150
4151 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4152 ping = E1000_PF_CONTROL_MSG;
4153 if (adapter->vf_data[i].clear_to_send)
4154 ping |= E1000_VT_MSGTYPE_CTS;
4155 igb_write_mbx(hw, &ping, 1, i);
4156 }
4157}
4158
4159static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4160 u32 *msgbuf, u32 vf)
4161{
4162 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4163 u16 *hash_list = (u16 *)&msgbuf[1];
4164 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4165 int i;
4166
4167 /* only up to 30 hash values supported */
4168 if (n > 30)
4169 n = 30;
4170
4171 /* salt away the number of multi cast addresses assigned
4172 * to this VF for later use to restore when the PF multi cast
4173 * list changes
4174 */
4175 vf_data->num_vf_mc_hashes = n;
4176
4177 /* VFs are limited to using the MTA hash table for their multicast
4178 * addresses */
4179 for (i = 0; i < n; i++)
a419aef8 4180 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4181
4182 /* Flush and reset the mta with the new values */
ff41f8dc 4183 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4184
4185 return 0;
4186}
4187
4188static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4189{
4190 struct e1000_hw *hw = &adapter->hw;
4191 struct vf_data_storage *vf_data;
4192 int i, j;
4193
4194 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4195 vf_data = &adapter->vf_data[i];
75f4f382 4196 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4ae196df
AD		 4197 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4198 }
4199}
4200
4201static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4202{
4203 struct e1000_hw *hw = &adapter->hw;
4204 u32 pool_mask, reg, vid;
4205 int i;
4206
4207 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4208
4209 /* Find the vlan filter for this id */
4210 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4211 reg = rd32(E1000_VLVF(i));
4212
4213 /* remove the vf from the pool */
4214 reg &= ~pool_mask;
4215
4216 /* if pool is empty then remove entry from vfta */
4217 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4218 (reg & E1000_VLVF_VLANID_ENABLE)) {
4219 reg = 0;
4220 vid = reg & E1000_VLVF_VLANID_MASK;
4221 igb_vfta_set(hw, vid, false);
4222 }
4223
4224 wr32(E1000_VLVF(i), reg);
4225 }
ae641bdc
AD		 4226
4227 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4228}
4229
4230static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4231{
4232 struct e1000_hw *hw = &adapter->hw;
4233 u32 reg, i;
4234
4235 /* It is an error to call this function when VFs are not enabled */
4236 if (!adapter->vfs_allocated_count)
4237 return -1;
4238
4239 /* Find the vlan filter for this id */
4240 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4241 reg = rd32(E1000_VLVF(i));
4242 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4243 vid == (reg & E1000_VLVF_VLANID_MASK))
4244 break;
4245 }
4246
4247 if (add) {
4248 if (i == E1000_VLVF_ARRAY_SIZE) {
4249 /* Did not find a matching VLAN ID entry that was
4250 * enabled. Search for a free filter entry, i.e.
4251 * one without the enable bit set
4252 */
4253 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4254 reg = rd32(E1000_VLVF(i));
4255 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4256 break;
4257 }
4258 }
4259 if (i < E1000_VLVF_ARRAY_SIZE) {
4260 /* Found an enabled/available entry */
4261 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4262
4263 /* if !enabled we need to set this up in vfta */
4264 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
cad6d05f
AD		 4265 /* add VID to filter table, if bit already set
4266 * PF must have added it outside of table */
4267 if (igb_vfta_set(hw, vid, true))
4268 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT +
4269 adapter->vfs_allocated_count);
4ae196df
AD		 4270 reg |= E1000_VLVF_VLANID_ENABLE;
4271 }
cad6d05f
AD		 4272 reg &= ~E1000_VLVF_VLANID_MASK;
4273 reg |= vid;
4ae196df
AD		 4274
4275 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4276
4277 /* do not modify RLPML for PF devices */
4278 if (vf >= adapter->vfs_allocated_count)
4279 return 0;
4280
4281 if (!adapter->vf_data[vf].vlans_enabled) {
4282 u32 size;
4283 reg = rd32(E1000_VMOLR(vf));
4284 size = reg & E1000_VMOLR_RLPML_MASK;
4285 size += 4;
4286 reg &= ~E1000_VMOLR_RLPML_MASK;
4287 reg |= size;
4288 wr32(E1000_VMOLR(vf), reg);
4289 }
4290 adapter->vf_data[vf].vlans_enabled++;
4291
4ae196df
AD		 4292 return 0;
4293 }
4294 } else {
4295 if (i < E1000_VLVF_ARRAY_SIZE) {
4296 /* remove vf from the pool */
4297 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4298 /* if pool is empty then remove entry from vfta */
4299 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4300 reg = 0;
4301 igb_vfta_set(hw, vid, false);
4302 }
4303 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4304
4305 /* do not modify RLPML for PF devices */
4306 if (vf >= adapter->vfs_allocated_count)
4307 return 0;
4308
4309 adapter->vf_data[vf].vlans_enabled--;
4310 if (!adapter->vf_data[vf].vlans_enabled) {
4311 u32 size;
4312 reg = rd32(E1000_VMOLR(vf));
4313 size = reg & E1000_VMOLR_RLPML_MASK;
4314 size -= 4;
4315 reg &= ~E1000_VMOLR_RLPML_MASK;
4316 reg |= size;
4317 wr32(E1000_VMOLR(vf), reg);
4318 }
4ae196df
AD		 4319 return 0;
4320 }
4321 }
4322 return -1;
4323}
4324
4325static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4326{
4327 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4328 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4329
4330 return igb_vlvf_set(adapter, vid, add, vf);
4331}
4332
4333static inline void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4334{
4335 struct e1000_hw *hw = &adapter->hw;
4336
4337 /* disable mailbox functionality for vf */
4338 adapter->vf_data[vf].clear_to_send = false;
4339
4340 /* reset offloads to defaults */
4341 igb_set_vmolr(hw, vf);
4342
4343 /* reset vlans for device */
4344 igb_clear_vf_vfta(adapter, vf);
4345
4346 /* reset multicast table array for vf */
4347 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4348
4349 /* Flush and reset the mta with the new values */
ff41f8dc 4350 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4351}
4352
4353static inline void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4354{
4355 struct e1000_hw *hw = &adapter->hw;
4356 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4357 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4358 u32 reg, msgbuf[3];
4359 u8 *addr = (u8 *)(&msgbuf[1]);
4360
4361 /* process all the same items cleared in a function level reset */
4362 igb_vf_reset_event(adapter, vf);
4363
4364 /* set vf mac address */
26ad9178 4365 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4366
4367 /* enable transmit and receive for vf */
4368 reg = rd32(E1000_VFTE);
4369 wr32(E1000_VFTE, reg | (1 << vf));
4370 reg = rd32(E1000_VFRE);
4371 wr32(E1000_VFRE, reg | (1 << vf));
4372
4373 /* enable mailbox functionality for vf */
4374 adapter->vf_data[vf].clear_to_send = true;
4375
4376 /* reply to reset with ack and vf mac address */
4377 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4378 memcpy(addr, vf_mac, 6);
4379 igb_write_mbx(hw, msgbuf, 3, vf);
4380}
4381
4382static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4383{
4384 unsigned char *addr = (char *)&msg[1];
4385 int err = -1;
4386
4387 if (is_valid_ether_addr(addr))
4388 err = igb_set_vf_mac(adapter, vf, addr);
4389
4390 return err;
4391
4392}
4393
4394static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4395{
4396 struct e1000_hw *hw = &adapter->hw;
4397 u32 msg = E1000_VT_MSGTYPE_NACK;
4398
4399 /* if device isn't clear to send it shouldn't be reading either */
4400 if (!adapter->vf_data[vf].clear_to_send)
4401 igb_write_mbx(hw, &msg, 1, vf);
4402}
4403
4404
4405static void igb_msg_task(struct igb_adapter *adapter)
4406{
4407 struct e1000_hw *hw = &adapter->hw;
4408 u32 vf;
4409
4410 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4411 /* process any reset requests */
4412 if (!igb_check_for_rst(hw, vf)) {
4413 adapter->vf_data[vf].clear_to_send = false;
4414 igb_vf_reset_event(adapter, vf);
4415 }
4416
4417 /* process any messages pending */
4418 if (!igb_check_for_msg(hw, vf))
4419 igb_rcv_msg_from_vf(adapter, vf);
4420
4421 /* process any acks */
4422 if (!igb_check_for_ack(hw, vf))
4423 igb_rcv_ack_from_vf(adapter, vf);
4424
4425 }
4426}
4427
4428static int igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4429{
4430 u32 mbx_size = E1000_VFMAILBOX_SIZE;
4431 u32 msgbuf[mbx_size];
4432 struct e1000_hw *hw = &adapter->hw;
4433 s32 retval;
4434
4435 retval = igb_read_mbx(hw, msgbuf, mbx_size, vf);
4436
4437 if (retval)
4438 dev_err(&adapter->pdev->dev,
4439 "Error receiving message from VF\n");
4440
4441 /* this is a message we already processed, do nothing */
4442 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4443 return retval;
4444
4445 /*
4446 * until the vf completes a reset it should not be
4447 * allowed to start any configuration.
4448 */
4449
4450 if (msgbuf[0] == E1000_VF_RESET) {
4451 igb_vf_reset_msg(adapter, vf);
4452
4453 return retval;
4454 }
4455
4456 if (!adapter->vf_data[vf].clear_to_send) {
4457 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4458 igb_write_mbx(hw, msgbuf, 1, vf);
4459 return retval;
4460 }
4461
4462 switch ((msgbuf[0] & 0xFFFF)) {
4463 case E1000_VF_SET_MAC_ADDR:
4464 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4465 break;
4466 case E1000_VF_SET_MULTICAST:
4467 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4468 break;
4469 case E1000_VF_SET_LPE:
4470 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4471 break;
4472 case E1000_VF_SET_VLAN:
4473 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4474 break;
4475 default:
4476 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4477 retval = -1;
4478 break;
4479 }
4480
4481 /* notify the VF of the results of what it sent us */
4482 if (retval)
4483 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4484 else
4485 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4486
4487 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4488
4489 igb_write_mbx(hw, msgbuf, 1, vf);
4490
4491 return retval;
4492}
4493
68d480c4
AD		 4494/**
4495 * igb_set_uta - Set unicast filter table address
4496 * @adapter: board private structure
4497 *
4498 * The unicast table address is a register array of 32-bit registers.
4499 * The table is meant to be used in a way similar to how the MTA is used
4500 * however due to certain limitations in the hardware it is necessary to
4501 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4502 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4503 **/
4504static void igb_set_uta(struct igb_adapter *adapter)
4505{
4506 struct e1000_hw *hw = &adapter->hw;
4507 int i;
4508
4509 /* The UTA table only exists on 82576 hardware and newer */
4510 if (hw->mac.type < e1000_82576)
4511 return;
4512
4513 /* we only need to do this if VMDq is enabled */
4514 if (!adapter->vfs_allocated_count)
4515 return;
4516
4517 for (i = 0; i < hw->mac.uta_reg_count; i++)
4518 array_wr32(E1000_UTA, i, ~0);
4519}
4520
9d5c8243
AK		 4521/**
4522 * igb_intr_msi - Interrupt Handler
4523 * @irq: interrupt number
4524 * @data: pointer to a network interface device structure
4525 **/
4526static irqreturn_t igb_intr_msi(int irq, void *data)
4527{
047e0030
AD		 4528 struct igb_adapter *adapter = data;
4529 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4530 struct e1000_hw *hw = &adapter->hw;
4531 /* read ICR disables interrupts using IAM */
4532 u32 icr = rd32(E1000_ICR);
4533
047e0030 4534 igb_write_itr(q_vector);
9d5c8243 4535
047e0030 4536 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4537 /* HW is reporting DMA is out of sync */
4538 adapter->stats.doosync++;
4539 }
4540
9d5c8243
AK		 4541 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4542 hw->mac.get_link_status = 1;
4543 if (!test_bit(__IGB_DOWN, &adapter->state))
4544 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4545 }
4546
047e0030 4547 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4548
4549 return IRQ_HANDLED;
4550}
4551
4552/**
4a3c6433 4553 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4554 * @irq: interrupt number
4555 * @data: pointer to a network interface device structure
4556 **/
4557static irqreturn_t igb_intr(int irq, void *data)
4558{
047e0030
AD		 4559 struct igb_adapter *adapter = data;
4560 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4561 struct e1000_hw *hw = &adapter->hw;
4562 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4563 * need for the IMC write */
4564 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4565 if (!icr)
4566 return IRQ_NONE; /* Not our interrupt */
4567
047e0030 4568 igb_write_itr(q_vector);
9d5c8243
AK		 4569
4570 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4571 * not set, then the adapter didn't send an interrupt */
4572 if (!(icr & E1000_ICR_INT_ASSERTED))
4573 return IRQ_NONE;
4574
047e0030 4575 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4576 /* HW is reporting DMA is out of sync */
4577 adapter->stats.doosync++;
4578 }
4579
9d5c8243
AK		 4580 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4581 hw->mac.get_link_status = 1;
4582 /* guard against interrupt when we're going down */
4583 if (!test_bit(__IGB_DOWN, &adapter->state))
4584 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4585 }
4586
047e0030 4587 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4588
4589 return IRQ_HANDLED;
4590}
4591
047e0030 4592static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4593{
047e0030 4594 struct igb_adapter *adapter = q_vector->adapter;
46544258 4595 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4596
46544258 4597 if (adapter->itr_setting & 3) {
047e0030 4598 if (!adapter->msix_entries)
6eb5a7f1 4599 igb_set_itr(adapter);
46544258 4600 else
047e0030 4601 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4602 }
4603
46544258
AD		 4604 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4605 if (adapter->msix_entries)
047e0030 4606 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4607 else
4608 igb_irq_enable(adapter);
4609 }
9d5c8243
AK		 4610}
4611
46544258
AD		 4612/**
4613 * igb_poll - NAPI Rx polling callback
4614 * @napi: napi polling structure
4615 * @budget: count of how many packets we should handle
4616 **/
4617static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4618{
047e0030
AD		 4619 struct igb_q_vector *q_vector = container_of(napi,
4620 struct igb_q_vector,
4621 napi);
4622 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4623
421e02f0 4624#ifdef CONFIG_IGB_DCA
047e0030
AD		 4625 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4626 igb_update_dca(q_vector);
fe4506b6 4627#endif
047e0030
AD		 4628 if (q_vector->tx_ring)
4629 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4630
047e0030
AD		 4631 if (q_vector->rx_ring)
4632 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4633
4634 if (!tx_clean_complete)
4635 work_done = budget;
46544258 4636
9d5c8243 4637 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4638 if (work_done < budget) {
288379f0 4639 napi_complete(napi);
047e0030 4640 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4641 }
4642
46544258 4643 return work_done;
9d5c8243 4644}
6d8126f9 4645
33af6bcc
PO		 4646/**
4647 * igb_hwtstamp - utility function which checks for TX time stamp
4648 * @adapter: board private structure
4649 * @skb: packet that was just sent
4650 *
4651 * If we were asked to do hardware stamping and such a time stamp is
4652 * available, then it must have been for this skb here because we only
4653 * allow only one such packet into the queue.
4654 */
4655static void igb_tx_hwtstamp(struct igb_adapter *adapter, struct sk_buff *skb)
4656{
4657 union skb_shared_tx *shtx = skb_tx(skb);
4658 struct e1000_hw *hw = &adapter->hw;
4659
4660 if (unlikely(shtx->hardware)) {
4661 u32 valid = rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID;
4662 if (valid) {
4663 u64 regval = rd32(E1000_TXSTMPL);
4664 u64 ns;
4665 struct skb_shared_hwtstamps shhwtstamps;
4666
4667 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
4668 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4669 ns = timecounter_cyc2time(&adapter->clock,
4670 regval);
4671 timecompare_update(&adapter->compare, ns);
4672 shhwtstamps.hwtstamp = ns_to_ktime(ns);
4673 shhwtstamps.syststamp =
4674 timecompare_transform(&adapter->compare, ns);
4675 skb_tstamp_tx(skb, &shhwtstamps);
4676 }
33af6bcc
PO		 4677 }
4678}
4679
9d5c8243
AK		 4680/**
4681 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4682 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4683 * returns true if ring is completely cleaned
4684 **/
047e0030 4685static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4686{
047e0030
AD		 4687 struct igb_adapter *adapter = q_vector->adapter;
4688 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4689 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4690 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4691 struct igb_buffer *buffer_info;
4692 struct sk_buff *skb;
0e014cb1 4693 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4694 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4695 unsigned int i, eop, count = 0;
4696 bool cleaned = false;
9d5c8243 4697
9d5c8243 4698 i = tx_ring->next_to_clean;
0e014cb1
AD		 4699 eop = tx_ring->buffer_info[i].next_to_watch;
4700 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4701
4702 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4703 (count < tx_ring->count)) {
4704 for (cleaned = false; !cleaned; count++) {
4705 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4706 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4707 cleaned = (i == eop);
9d5c8243
AK		 4708 skb = buffer_info->skb;
4709
4710 if (skb) {
4711 unsigned int segs, bytecount;
4712 /* gso_segs is currently only valid for tcp */
4713 segs = skb_shinfo(skb)->gso_segs ?: 1;
4714 /* multiply data chunks by size of headers */
4715 bytecount = ((segs - 1) * skb_headlen(skb)) +
4716 skb->len;
4717 total_packets += segs;
4718 total_bytes += bytecount;
33af6bcc
PO		 4719
4720 igb_tx_hwtstamp(adapter, skb);
9d5c8243
AK		 4721 }
4722
80785298 4723 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4724 tx_desc->wb.status = 0;
9d5c8243
AK		 4725
4726 i++;
4727 if (i == tx_ring->count)
4728 i = 0;
9d5c8243 4729 }
0e014cb1
AD		 4730 eop = tx_ring->buffer_info[i].next_to_watch;
4731 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4732 }
4733
9d5c8243
AK		 4734 tx_ring->next_to_clean = i;
4735
fc7d345d 4736 if (unlikely(count &&
9d5c8243 4737 netif_carrier_ok(netdev) &&
c493ea45 4738 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4739 /* Make sure that anybody stopping the queue after this
4740 * sees the new next_to_clean.
4741 */
4742 smp_mb();
661086df
PWJ		 4743 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4744 !(test_bit(__IGB_DOWN, &adapter->state))) {
4745 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4746 tx_ring->tx_stats.restart_queue++;
661086df 4747 }
9d5c8243
AK		 4748 }
4749
4750 if (tx_ring->detect_tx_hung) {
4751 /* Detect a transmit hang in hardware, this serializes the
4752 * check with the clearing of time_stamp and movement of i */
4753 tx_ring->detect_tx_hung = false;
4754 if (tx_ring->buffer_info[i].time_stamp &&
4755 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4756 (adapter->tx_timeout_factor * HZ))
4757 && !(rd32(E1000_STATUS) &
4758 E1000_STATUS_TXOFF)) {
4759
9d5c8243 4760 /* detected Tx unit hang */
80785298 4761 dev_err(&tx_ring->pdev->dev,
9d5c8243 4762 "Detected Tx Unit Hang\n"
2d064c06 4763 " Tx Queue <%d>\n"
9d5c8243
AK		 4764 " TDH <%x>\n"
4765 " TDT <%x>\n"
4766 " next_to_use <%x>\n"
4767 " next_to_clean <%x>\n"
9d5c8243
AK		 4768 "buffer_info[next_to_clean]\n"
4769 " time_stamp <%lx>\n"
0e014cb1 4770 " next_to_watch <%x>\n"
9d5c8243
AK		 4771 " jiffies <%lx>\n"
4772 " desc.status <%x>\n",
2d064c06 4773 tx_ring->queue_index,
fce99e34
AD		 4774 readl(tx_ring->head),
4775 readl(tx_ring->tail),
9d5c8243
AK		 4776 tx_ring->next_to_use,
4777 tx_ring->next_to_clean,
9d5c8243 4778 tx_ring->buffer_info[i].time_stamp,
0e014cb1 4779 eop,
9d5c8243 4780 jiffies,
0e014cb1 4781 eop_desc->wb.status);
661086df 4782 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4783 }
4784 }
4785 tx_ring->total_bytes += total_bytes;
4786 tx_ring->total_packets += total_packets;
e21ed353
AD		 4787 tx_ring->tx_stats.bytes += total_bytes;
4788 tx_ring->tx_stats.packets += total_packets;
8d24e933
AK		 4789 netdev->stats.tx_bytes += total_bytes;
4790 netdev->stats.tx_packets += total_packets;
0e014cb1 4791 return (count < tx_ring->count);
9d5c8243
AK		 4792}
4793
9d5c8243
AK		 4794/**
4795 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4796 * @q_vector: structure containing interrupt and ring information
4797 * @skb: packet to send up
4798 * @vlan_tag: vlan tag for packet
9d5c8243 4799 **/
047e0030
AD		 4800static void igb_receive_skb(struct igb_q_vector *q_vector,
4801 struct sk_buff *skb,
4802 u16 vlan_tag)
4803{
4804 struct igb_adapter *adapter = q_vector->adapter;
4805
4806 if (vlan_tag)
4807 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4808 vlan_tag, skb);
182ff8df 4809 else
047e0030 4810 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4811}
4812
04a5fcaa 4813static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4814 u32 status_err, struct sk_buff *skb)
4815{
4816 skb->ip_summed = CHECKSUM_NONE;
4817
4818 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4819 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4820 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4821 return;
85ad76b2 4822
9d5c8243
AK		 4823 /* TCP/UDP checksum error bit is set */
4824 if (status_err &
4825 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4826 /*
4827 * work around errata with sctp packets where the TCPE aka
4828 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4829 * packets, (aka let the stack check the crc32c)
4830 */
85ad76b2
AD		 4831 if ((skb->len == 60) &&
4832 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4833 ring->rx_stats.csum_err++;
85ad76b2 4834
9d5c8243 4835 /* let the stack verify checksum errors */
9d5c8243
AK		 4836 return;
4837 }
4838 /* It must be a TCP or UDP packet with a valid checksum */
4839 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4840 skb->ip_summed = CHECKSUM_UNNECESSARY;
4841
85ad76b2 4842 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4843}
4844
4c844851 4845static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4846 union e1000_adv_rx_desc *rx_desc)
4847{
4848 /* HW will not DMA in data larger than the given buffer, even if it
4849 * parses the (NFS, of course) header to be larger. In that case, it
4850 * fills the header buffer and spills the rest into the page.
4851 */
4852 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4853 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4854 if (hlen > rx_ring->rx_buffer_len)
4855 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4856 return hlen;
4857}
4858
047e0030
AD		 4859static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4860 int *work_done, int budget)
9d5c8243 4861{
047e0030 4862 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4863 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4864 struct net_device *netdev = rx_ring->netdev;
33af6bcc 4865 struct e1000_hw *hw = &adapter->hw;
80785298 4866 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4867 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4868 struct igb_buffer *buffer_info , *next_buffer;
4869 struct sk_buff *skb;
9d5c8243
AK		 4870 bool cleaned = false;
4871 int cleaned_count = 0;
4872 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4873 unsigned int i;
2d94d8ab
AD		 4874 u32 staterr;
4875 u16 length;
047e0030 4876 u16 vlan_tag;
9d5c8243
AK		 4877
4878 i = rx_ring->next_to_clean;
69d3ca53 4879 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4880 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4881 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4882
4883 while (staterr & E1000_RXD_STAT_DD) {
4884 if (*work_done >= budget)
4885 break;
4886 (*work_done)++;
9d5c8243 4887
69d3ca53
AD		 4888 skb = buffer_info->skb;
4889 prefetch(skb->data - NET_IP_ALIGN);
4890 buffer_info->skb = NULL;
4891
4892 i++;
4893 if (i == rx_ring->count)
4894 i = 0;
4895 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4896 prefetch(next_rxd);
4897 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4898
4899 length = le16_to_cpu(rx_desc->wb.upper.length);
4900 cleaned = true;
4901 cleaned_count++;
4902
2d94d8ab 4903 if (buffer_info->dma) {
bf36c1a0 4904 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4905 rx_ring->rx_buffer_len,
bf36c1a0 4906 PCI_DMA_FROMDEVICE);
91615f76 4907 buffer_info->dma = 0;
4c844851 4908 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4909 skb_put(skb, length);
4910 goto send_up;
4911 }
4c844851 4912 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4913 }
4914
4915 if (length) {
9d5c8243 4916 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4917 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4918 buffer_info->page_dma = 0;
bf36c1a0
AD		 4919
4920 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4921 buffer_info->page,
4922 buffer_info->page_offset,
4923 length);
4924
6ec43fe6 4925 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4926 buffer_info->page = NULL;
4927 else
4928 get_page(buffer_info->page);
9d5c8243
AK		 4929
4930 skb->len += length;
4931 skb->data_len += length;
9d5c8243 4932
bf36c1a0 4933 skb->truesize += length;
9d5c8243 4934 }
9d5c8243 4935
bf36c1a0 4936 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4937 buffer_info->skb = next_buffer->skb;
4938 buffer_info->dma = next_buffer->dma;
4939 next_buffer->skb = skb;
4940 next_buffer->dma = 0;
bf36c1a0
AD		 4941 goto next_desc;
4942 }
69d3ca53 4943send_up:
33af6bcc
PO		 4944 /*
4945 * If this bit is set, then the RX registers contain
4946 * the time stamp. No other packet will be time
4947 * stamped until we read these registers, so read the
4948 * registers to make them available again. Because
4949 * only one packet can be time stamped at a time, we
4950 * know that the register values must belong to this
4951 * one here and therefore we don't need to compare
4952 * any of the additional attributes stored for it.
4953 *
4954 * If nothing went wrong, then it should have a
4955 * skb_shared_tx that we can turn into a
4956 * skb_shared_hwtstamps.
4957 *
4958 * TODO: can time stamping be triggered (thus locking
4959 * the registers) without the packet reaching this point
4960 * here? In that case RX time stamping would get stuck.
4961 *
4962 * TODO: in "time stamp all packets" mode this bit is
4963 * not set. Need a global flag for this mode and then
4964 * always read the registers. Cannot be done without
4965 * a race condition.
4966 */
4967 if (unlikely(staterr & E1000_RXD_STAT_TS)) {
4968 u64 regval;
4969 u64 ns;
4970 struct skb_shared_hwtstamps *shhwtstamps =
4971 skb_hwtstamps(skb);
4972
4973 WARN(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID),
4974 "igb: no RX time stamp available for time stamped packet");
4975 regval = rd32(E1000_RXSTMPL);
4976 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4977 ns = timecounter_cyc2time(&adapter->clock, regval);
4978 timecompare_update(&adapter->compare, ns);
4979 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
4980 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4981 shhwtstamps->syststamp =
4982 timecompare_transform(&adapter->compare, ns);
4983 }
4984
9d5c8243
AK		 4985 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
4986 dev_kfree_skb_irq(skb);
4987 goto next_desc;
4988 }
9d5c8243
AK		 4989
4990 total_bytes += skb->len;
4991 total_packets++;
4992
85ad76b2 4993 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 4994
4995 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 4996 skb_record_rx_queue(skb, rx_ring->queue_index);
4997
4998 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
4999 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5000
047e0030 5001 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5002
9d5c8243
AK		 5003next_desc:
5004 rx_desc->wb.upper.status_error = 0;
5005
5006 /* return some buffers to hardware, one at a time is too slow */
5007 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5008 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5009 cleaned_count = 0;
5010 }
5011
5012 /* use prefetched values */
5013 rx_desc = next_rxd;
5014 buffer_info = next_buffer;
9d5c8243
AK		 5015 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5016 }
bf36c1a0 5017
9d5c8243 5018 rx_ring->next_to_clean = i;
c493ea45 5019 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5020
5021 if (cleaned_count)
3b644cf6 5022 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5023
5024 rx_ring->total_packets += total_packets;
5025 rx_ring->total_bytes += total_bytes;
5026 rx_ring->rx_stats.packets += total_packets;
5027 rx_ring->rx_stats.bytes += total_bytes;
8d24e933
AK		 5028 netdev->stats.rx_bytes += total_bytes;
5029 netdev->stats.rx_packets += total_packets;
9d5c8243
AK		 5030 return cleaned;
5031}
5032
9d5c8243
AK		 5033/**
5034 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5035 * @adapter: address of board private structure
5036 **/
3b644cf6 5037static void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring,
9d5c8243
AK		 5038 int cleaned_count)
5039{
e694e964 5040 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5041 union e1000_adv_rx_desc *rx_desc;
5042 struct igb_buffer *buffer_info;
5043 struct sk_buff *skb;
5044 unsigned int i;
db761762 5045 int bufsz;
9d5c8243
AK		 5046
5047 i = rx_ring->next_to_use;
5048 buffer_info = &rx_ring->buffer_info[i];
5049
4c844851 5050 bufsz = rx_ring->rx_buffer_len;
db761762 5051
9d5c8243
AK		 5052 while (cleaned_count--) {
5053 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5054
6ec43fe6 5055 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5056 if (!buffer_info->page) {
bf36c1a0
AD		 5057 buffer_info->page = alloc_page(GFP_ATOMIC);
5058 if (!buffer_info->page) {
04a5fcaa 5059 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5060 goto no_buffers;
5061 }
5062 buffer_info->page_offset = 0;
5063 } else {
5064 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5065 }
5066 buffer_info->page_dma =
80785298 5067 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5068 buffer_info->page_offset,
5069 PAGE_SIZE / 2,
9d5c8243
AK		 5070 PCI_DMA_FROMDEVICE);
5071 }
5072
5073 if (!buffer_info->skb) {
89d71a66 5074 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5075 if (!skb) {
04a5fcaa 5076 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5077 goto no_buffers;
5078 }
5079
9d5c8243 5080 buffer_info->skb = skb;
80785298
AD		 5081 buffer_info->dma = pci_map_single(rx_ring->pdev,
5082 skb->data,
9d5c8243
AK		 5083 bufsz,
5084 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 5085 }
5086 /* Refresh the desc even if buffer_addrs didn't change because
5087 * each write-back erases this info. */
6ec43fe6 5088 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5089 rx_desc->read.pkt_addr =
5090 cpu_to_le64(buffer_info->page_dma);
5091 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5092 } else {
5093 rx_desc->read.pkt_addr =
5094 cpu_to_le64(buffer_info->dma);
5095 rx_desc->read.hdr_addr = 0;
5096 }
5097
5098 i++;
5099 if (i == rx_ring->count)
5100 i = 0;
5101 buffer_info = &rx_ring->buffer_info[i];
5102 }
5103
5104no_buffers:
5105 if (rx_ring->next_to_use != i) {
5106 rx_ring->next_to_use = i;
5107 if (i == 0)
5108 i = (rx_ring->count - 1);
5109 else
5110 i--;
5111
5112 /* Force memory writes to complete before letting h/w
5113 * know there are new descriptors to fetch. (Only
5114 * applicable for weak-ordered memory model archs,
5115 * such as IA-64). */
5116 wmb();
fce99e34 5117 writel(i, rx_ring->tail);
9d5c8243
AK		 5118 }
5119}
5120
5121/**
5122 * igb_mii_ioctl -
5123 * @netdev:
5124 * @ifreq:
5125 * @cmd:
5126 **/
5127static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5128{
5129 struct igb_adapter *adapter = netdev_priv(netdev);
5130 struct mii_ioctl_data *data = if_mii(ifr);
5131
5132 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5133 return -EOPNOTSUPP;
5134
5135 switch (cmd) {
5136 case SIOCGMIIPHY:
5137 data->phy_id = adapter->hw.phy.addr;
5138 break;
5139 case SIOCGMIIREG:
f5f4cf08
AD		 5140 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5141 &data->val_out))
9d5c8243
AK		 5142 return -EIO;
5143 break;
5144 case SIOCSMIIREG:
5145 default:
5146 return -EOPNOTSUPP;
5147 }
5148 return 0;
5149}
5150
c6cb090b
PO		 5151/**
5152 * igb_hwtstamp_ioctl - control hardware time stamping
5153 * @netdev:
5154 * @ifreq:
5155 * @cmd:
5156 *
33af6bcc
PO		 5157 * Outgoing time stamping can be enabled and disabled. Play nice and
5158 * disable it when requested, although it shouldn't case any overhead
5159 * when no packet needs it. At most one packet in the queue may be
5160 * marked for time stamping, otherwise it would be impossible to tell
5161 * for sure to which packet the hardware time stamp belongs.
5162 *
5163 * Incoming time stamping has to be configured via the hardware
5164 * filters. Not all combinations are supported, in particular event
5165 * type has to be specified. Matching the kind of event packet is
5166 * not supported, with the exception of "all V2 events regardless of
5167 * level 2 or 4".
5168 *
c6cb090b
PO		 5169 **/
5170static int igb_hwtstamp_ioctl(struct net_device *netdev,
5171 struct ifreq *ifr, int cmd)
5172{
33af6bcc
PO		 5173 struct igb_adapter *adapter = netdev_priv(netdev);
5174 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5175 struct hwtstamp_config config;
33af6bcc
PO		 5176 u32 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
5177 u32 tsync_rx_ctl_bit = E1000_TSYNCRXCTL_ENABLED;
5178 u32 tsync_rx_ctl_type = 0;
5179 u32 tsync_rx_cfg = 0;
5180 int is_l4 = 0;
5181 int is_l2 = 0;
5182 short port = 319; /* PTP */
5183 u32 regval;
c6cb090b
PO		 5184
5185 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5186 return -EFAULT;
5187
5188 /* reserved for future extensions */
5189 if (config.flags)
5190 return -EINVAL;
5191
33af6bcc
PO		 5192 switch (config.tx_type) {
5193 case HWTSTAMP_TX_OFF:
5194 tsync_tx_ctl_bit = 0;
5195 break;
5196 case HWTSTAMP_TX_ON:
5197 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
5198 break;
5199 default:
5200 return -ERANGE;
5201 }
5202
5203 switch (config.rx_filter) {
5204 case HWTSTAMP_FILTER_NONE:
5205 tsync_rx_ctl_bit = 0;
5206 break;
5207 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5208 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5209 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5210 case HWTSTAMP_FILTER_ALL:
5211 /*
5212 * register TSYNCRXCFG must be set, therefore it is not
5213 * possible to time stamp both Sync and Delay_Req messages
5214 * => fall back to time stamping all packets
5215 */
5216 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_ALL;
5217 config.rx_filter = HWTSTAMP_FILTER_ALL;
5218 break;
5219 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5220 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
5221 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5222 is_l4 = 1;
5223 break;
5224 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5225 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
5226 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5227 is_l4 = 1;
5228 break;
5229 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5230 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5231 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5232 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5233 is_l2 = 1;
5234 is_l4 = 1;
5235 config.rx_filter = HWTSTAMP_FILTER_SOME;
5236 break;
5237 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5238 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5239 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5240 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5241 is_l2 = 1;
5242 is_l4 = 1;
5243 config.rx_filter = HWTSTAMP_FILTER_SOME;
5244 break;
5245 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5246 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5247 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5248 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5249 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5250 is_l2 = 1;
5251 break;
5252 default:
5253 return -ERANGE;
5254 }
5255
5256 /* enable/disable TX */
5257 regval = rd32(E1000_TSYNCTXCTL);
5258 regval = (regval & ~E1000_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit;
5259 wr32(E1000_TSYNCTXCTL, regval);
5260
5261 /* enable/disable RX, define which PTP packets are time stamped */
5262 regval = rd32(E1000_TSYNCRXCTL);
5263 regval = (regval & ~E1000_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit;
5264 regval = (regval & ~0xE) | tsync_rx_ctl_type;
5265 wr32(E1000_TSYNCRXCTL, regval);
5266 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5267
5268 /*
5269 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7
5270 * (Ethertype to filter on)
5271 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
5272 * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping)
5273 */
5274 wr32(E1000_ETQF0, is_l2 ? 0x440088f7 : 0);
5275
5276 /* L4 Queue Filter[0]: only filter by source and destination port */
5277 wr32(E1000_SPQF0, htons(port));
5278 wr32(E1000_IMIREXT(0), is_l4 ?
5279 ((1<<12) | (1<<19) /* bypass size and control flags */) : 0);
5280 wr32(E1000_IMIR(0), is_l4 ?
5281 (htons(port)
5282 | (0<<16) /* immediate interrupt disabled */
5283 | 0 /* (1<<17) bit cleared: do not bypass
5284 destination port check */)
5285 : 0);
5286 wr32(E1000_FTQF0, is_l4 ?
5287 (0x11 /* UDP */
5288 | (1<<15) /* VF not compared */
5289 | (1<<27) /* Enable Timestamping */
5290 | (7<<28) /* only source port filter enabled,
5291 source/target address and protocol
5292 masked */)
5293 : ((1<<15) | (15<<28) /* all mask bits set = filter not
5294 enabled */));
5295
5296 wrfl();
5297
5298 adapter->hwtstamp_config = config;
5299
5300 /* clear TX/RX time stamp registers, just to be sure */
5301 regval = rd32(E1000_TXSTMPH);
5302 regval = rd32(E1000_RXSTMPH);
c6cb090b 5303
33af6bcc
PO		 5304 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5305 -EFAULT : 0;
c6cb090b
PO		 5306}
5307
9d5c8243
AK		 5308/**
5309 * igb_ioctl -
5310 * @netdev:
5311 * @ifreq:
5312 * @cmd:
5313 **/
5314static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5315{
5316 switch (cmd) {
5317 case SIOCGMIIPHY:
5318 case SIOCGMIIREG:
5319 case SIOCSMIIREG:
5320 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5321 case SIOCSHWTSTAMP:
5322 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5323 default:
5324 return -EOPNOTSUPP;
5325 }
5326}
5327
009bc06e
AD		 5328s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5329{
5330 struct igb_adapter *adapter = hw->back;
5331 u16 cap_offset;
5332
5333 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5334 if (!cap_offset)
5335 return -E1000_ERR_CONFIG;
5336
5337 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5338
5339 return 0;
5340}
5341
5342s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5343{
5344 struct igb_adapter *adapter = hw->back;
5345 u16 cap_offset;
5346
5347 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5348 if (!cap_offset)
5349 return -E1000_ERR_CONFIG;
5350
5351 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5352
5353 return 0;
5354}
5355
9d5c8243
AK		 5356static void igb_vlan_rx_register(struct net_device *netdev,
5357 struct vlan_group *grp)
5358{
5359 struct igb_adapter *adapter = netdev_priv(netdev);
5360 struct e1000_hw *hw = &adapter->hw;
5361 u32 ctrl, rctl;
5362
5363 igb_irq_disable(adapter);
5364 adapter->vlgrp = grp;
5365
5366 if (grp) {
5367 /* enable VLAN tag insert/strip */
5368 ctrl = rd32(E1000_CTRL);
5369 ctrl |= E1000_CTRL_VME;
5370 wr32(E1000_CTRL, ctrl);
5371
5372 /* enable VLAN receive filtering */
5373 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5374 rctl &= ~E1000_RCTL_CFIEN;
5375 wr32(E1000_RCTL, rctl);
5376 igb_update_mng_vlan(adapter);
9d5c8243
AK		 5377 } else {
5378 /* disable VLAN tag insert/strip */
5379 ctrl = rd32(E1000_CTRL);
5380 ctrl &= ~E1000_CTRL_VME;
5381 wr32(E1000_CTRL, ctrl);
5382
9d5c8243
AK		 5383 if (adapter->mng_vlan_id != (u16)IGB_MNG_VLAN_NONE) {
5384 igb_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
5385 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
5386 }
9d5c8243
AK		 5387 }
5388
e1739522
AD		 5389 igb_rlpml_set(adapter);
5390
9d5c8243
AK		 5391 if (!test_bit(__IGB_DOWN, &adapter->state))
5392 igb_irq_enable(adapter);
5393}
5394
5395static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5396{
5397 struct igb_adapter *adapter = netdev_priv(netdev);
5398 struct e1000_hw *hw = &adapter->hw;
4ae196df 5399 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5400
28b0759c 5401 if ((hw->mng_cookie.status &
9d5c8243
AK		 5402 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
5403 (vid == adapter->mng_vlan_id))
5404 return;
4ae196df
AD		 5405
5406 /* add vid to vlvf if sr-iov is enabled,
5407 * if that fails add directly to filter table */
5408 if (igb_vlvf_set(adapter, vid, true, pf_id))
5409 igb_vfta_set(hw, vid, true);
5410
9d5c8243
AK		 5411}
5412
5413static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5414{
5415 struct igb_adapter *adapter = netdev_priv(netdev);
5416 struct e1000_hw *hw = &adapter->hw;
4ae196df 5417 int pf_id = adapter->vfs_allocated_count;
9d5c8243
AK		 5418
5419 igb_irq_disable(adapter);
5420 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5421
5422 if (!test_bit(__IGB_DOWN, &adapter->state))
5423 igb_irq_enable(adapter);
5424
5425 if ((adapter->hw.mng_cookie.status &
5426 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
5427 (vid == adapter->mng_vlan_id)) {
5428 /* release control to f/w */
5429 igb_release_hw_control(adapter);
5430 return;
5431 }
5432
4ae196df
AD		 5433 /* remove vid from vlvf if sr-iov is enabled,
5434 * if not in vlvf remove from vfta */
5435 if (igb_vlvf_set(adapter, vid, false, pf_id))
5436 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5437}
5438
5439static void igb_restore_vlan(struct igb_adapter *adapter)
5440{
5441 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5442
5443 if (adapter->vlgrp) {
5444 u16 vid;
5445 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5446 if (!vlan_group_get_device(adapter->vlgrp, vid))
5447 continue;
5448 igb_vlan_rx_add_vid(adapter->netdev, vid);
5449 }
5450 }
5451}
5452
5453int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5454{
5455 struct e1000_mac_info *mac = &adapter->hw.mac;
5456
5457 mac->autoneg = 0;
5458
9d5c8243
AK		 5459 switch (spddplx) {
5460 case SPEED_10 + DUPLEX_HALF:
5461 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5462 break;
5463 case SPEED_10 + DUPLEX_FULL:
5464 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5465 break;
5466 case SPEED_100 + DUPLEX_HALF:
5467 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5468 break;
5469 case SPEED_100 + DUPLEX_FULL:
5470 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5471 break;
5472 case SPEED_1000 + DUPLEX_FULL:
5473 mac->autoneg = 1;
5474 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5475 break;
5476 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5477 default:
5478 dev_err(&adapter->pdev->dev,
5479 "Unsupported Speed/Duplex configuration\n");
5480 return -EINVAL;
5481 }
5482 return 0;
5483}
5484
3fe7c4c9 5485static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5486{
5487 struct net_device *netdev = pci_get_drvdata(pdev);
5488 struct igb_adapter *adapter = netdev_priv(netdev);
5489 struct e1000_hw *hw = &adapter->hw;
2d064c06 5490 u32 ctrl, rctl, status;
9d5c8243
AK		 5491 u32 wufc = adapter->wol;
5492#ifdef CONFIG_PM
5493 int retval = 0;
5494#endif
5495
5496 netif_device_detach(netdev);
5497
a88f10ec
AD		 5498 if (netif_running(netdev))
5499 igb_close(netdev);
5500
047e0030 5501 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5502
5503#ifdef CONFIG_PM
5504 retval = pci_save_state(pdev);
5505 if (retval)
5506 return retval;
5507#endif
5508
5509 status = rd32(E1000_STATUS);
5510 if (status & E1000_STATUS_LU)
5511 wufc &= ~E1000_WUFC_LNKC;
5512
5513 if (wufc) {
5514 igb_setup_rctl(adapter);
ff41f8dc 5515 igb_set_rx_mode(netdev);
9d5c8243
AK		 5516
5517 /* turn on all-multi mode if wake on multicast is enabled */
5518 if (wufc & E1000_WUFC_MC) {
5519 rctl = rd32(E1000_RCTL);
5520 rctl |= E1000_RCTL_MPE;
5521 wr32(E1000_RCTL, rctl);
5522 }
5523
5524 ctrl = rd32(E1000_CTRL);
5525 /* advertise wake from D3Cold */
5526 #define E1000_CTRL_ADVD3WUC 0x00100000
5527 /* phy power management enable */
5528 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5529 ctrl |= E1000_CTRL_ADVD3WUC;
5530 wr32(E1000_CTRL, ctrl);
5531
9d5c8243
AK		 5532 /* Allow time for pending master requests to run */
5533 igb_disable_pcie_master(&adapter->hw);
5534
5535 wr32(E1000_WUC, E1000_WUC_PME_EN);
5536 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5537 } else {
5538 wr32(E1000_WUC, 0);
5539 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5540 }
5541
3fe7c4c9
RW		 5542 *enable_wake = wufc || adapter->en_mng_pt;
5543 if (!*enable_wake)
2fb02a26 5544 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5545
5546 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5547 * would have already happened in close and is redundant. */
5548 igb_release_hw_control(adapter);
5549
5550 pci_disable_device(pdev);
5551
9d5c8243
AK		 5552 return 0;
5553}
5554
5555#ifdef CONFIG_PM
3fe7c4c9
RW		 5556static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5557{
5558 int retval;
5559 bool wake;
5560
5561 retval = __igb_shutdown(pdev, &wake);
5562 if (retval)
5563 return retval;
5564
5565 if (wake) {
5566 pci_prepare_to_sleep(pdev);
5567 } else {
5568 pci_wake_from_d3(pdev, false);
5569 pci_set_power_state(pdev, PCI_D3hot);
5570 }
5571
5572 return 0;
5573}
5574
9d5c8243
AK		 5575static int igb_resume(struct pci_dev *pdev)
5576{
5577 struct net_device *netdev = pci_get_drvdata(pdev);
5578 struct igb_adapter *adapter = netdev_priv(netdev);
5579 struct e1000_hw *hw = &adapter->hw;
5580 u32 err;
5581
5582 pci_set_power_state(pdev, PCI_D0);
5583 pci_restore_state(pdev);
42bfd33a 5584
aed5dec3 5585 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5586 if (err) {
5587 dev_err(&pdev->dev,
5588 "igb: Cannot enable PCI device from suspend\n");
5589 return err;
5590 }
5591 pci_set_master(pdev);
5592
5593 pci_enable_wake(pdev, PCI_D3hot, 0);
5594 pci_enable_wake(pdev, PCI_D3cold, 0);
5595
047e0030 5596 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5597 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5598 return -ENOMEM;
9d5c8243
AK		 5599 }
5600
5601 /* e1000_power_up_phy(adapter); */
5602
5603 igb_reset(adapter);
a8564f03
AD		 5604
5605 /* let the f/w know that the h/w is now under the control of the
5606 * driver. */
5607 igb_get_hw_control(adapter);
5608
9d5c8243
AK		 5609 wr32(E1000_WUS, ~0);
5610
a88f10ec
AD		 5611 if (netif_running(netdev)) {
5612 err = igb_open(netdev);
5613 if (err)
5614 return err;
5615 }
9d5c8243
AK		 5616
5617 netif_device_attach(netdev);
5618
9d5c8243
AK		 5619 return 0;
5620}
5621#endif
5622
5623static void igb_shutdown(struct pci_dev *pdev)
5624{
3fe7c4c9
RW		 5625 bool wake;
5626
5627 __igb_shutdown(pdev, &wake);
5628
5629 if (system_state == SYSTEM_POWER_OFF) {
5630 pci_wake_from_d3(pdev, wake);
5631 pci_set_power_state(pdev, PCI_D3hot);
5632 }
9d5c8243
AK		 5633}
5634
5635#ifdef CONFIG_NET_POLL_CONTROLLER
5636/*
5637 * Polling 'interrupt' - used by things like netconsole to send skbs
5638 * without having to re-enable interrupts. It's not called while
5639 * the interrupt routine is executing.
5640 */
5641static void igb_netpoll(struct net_device *netdev)
5642{
5643 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5644 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5645 int i;
9d5c8243 5646
eebbbdba 5647 if (!adapter->msix_entries) {
047e0030 5648 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5649 igb_irq_disable(adapter);
047e0030 5650 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5651 return;
5652 }
9d5c8243 5653
047e0030
AD		 5654 for (i = 0; i < adapter->num_q_vectors; i++) {
5655 struct igb_q_vector *q_vector = adapter->q_vector[i];
5656 wr32(E1000_EIMC, q_vector->eims_value);
5657 napi_schedule(&q_vector->napi);
eebbbdba 5658 }
9d5c8243
AK		 5659}
5660#endif /* CONFIG_NET_POLL_CONTROLLER */
5661
5662/**
5663 * igb_io_error_detected - called when PCI error is detected
5664 * @pdev: Pointer to PCI device
5665 * @state: The current pci connection state
5666 *
5667 * This function is called after a PCI bus error affecting
5668 * this device has been detected.
5669 */
5670static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5671 pci_channel_state_t state)
5672{
5673 struct net_device *netdev = pci_get_drvdata(pdev);
5674 struct igb_adapter *adapter = netdev_priv(netdev);
5675
5676 netif_device_detach(netdev);
5677
59ed6eec
AD		 5678 if (state == pci_channel_io_perm_failure)
5679 return PCI_ERS_RESULT_DISCONNECT;
5680
9d5c8243
AK		 5681 if (netif_running(netdev))
5682 igb_down(adapter);
5683 pci_disable_device(pdev);
5684
5685 /* Request a slot slot reset. */
5686 return PCI_ERS_RESULT_NEED_RESET;
5687}
5688
5689/**
5690 * igb_io_slot_reset - called after the pci bus has been reset.
5691 * @pdev: Pointer to PCI device
5692 *
5693 * Restart the card from scratch, as if from a cold-boot. Implementation
5694 * resembles the first-half of the igb_resume routine.
5695 */
5696static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5697{
5698 struct net_device *netdev = pci_get_drvdata(pdev);
5699 struct igb_adapter *adapter = netdev_priv(netdev);
5700 struct e1000_hw *hw = &adapter->hw;
40a914fa 5701 pci_ers_result_t result;
42bfd33a 5702 int err;
9d5c8243 5703
aed5dec3 5704 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5705 dev_err(&pdev->dev,
5706 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5707 result = PCI_ERS_RESULT_DISCONNECT;
5708 } else {
5709 pci_set_master(pdev);
5710 pci_restore_state(pdev);
9d5c8243 5711
40a914fa
AD		 5712 pci_enable_wake(pdev, PCI_D3hot, 0);
5713 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5714
40a914fa
AD		 5715 igb_reset(adapter);
5716 wr32(E1000_WUS, ~0);
5717 result = PCI_ERS_RESULT_RECOVERED;
5718 }
9d5c8243 5719
ea943d41
JK		 5720 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5721 if (err) {
5722 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5723 "failed 0x%0x\n", err);
5724 /* non-fatal, continue */
5725 }
40a914fa
AD		 5726
5727 return result;
9d5c8243
AK		 5728}
5729
5730/**
5731 * igb_io_resume - called when traffic can start flowing again.
5732 * @pdev: Pointer to PCI device
5733 *
5734 * This callback is called when the error recovery driver tells us that
5735 * its OK to resume normal operation. Implementation resembles the
5736 * second-half of the igb_resume routine.
5737 */
5738static void igb_io_resume(struct pci_dev *pdev)
5739{
5740 struct net_device *netdev = pci_get_drvdata(pdev);
5741 struct igb_adapter *adapter = netdev_priv(netdev);
5742
9d5c8243
AK		 5743 if (netif_running(netdev)) {
5744 if (igb_up(adapter)) {
5745 dev_err(&pdev->dev, "igb_up failed after reset\n");
5746 return;
5747 }
5748 }
5749
5750 netif_device_attach(netdev);
5751
5752 /* let the f/w know that the h/w is now under the control of the
5753 * driver. */
5754 igb_get_hw_control(adapter);
9d5c8243
AK		 5755}
5756
26ad9178
AD		 5757static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5758 u8 qsel)
5759{
5760 u32 rar_low, rar_high;
5761 struct e1000_hw *hw = &adapter->hw;
5762
5763 /* HW expects these in little endian so we reverse the byte order
5764 * from network order (big endian) to little endian
5765 */
5766 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5767 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5768 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5769
5770 /* Indicate to hardware the Address is Valid. */
5771 rar_high |= E1000_RAH_AV;
5772
5773 if (hw->mac.type == e1000_82575)
5774 rar_high |= E1000_RAH_POOL_1 * qsel;
5775 else
5776 rar_high |= E1000_RAH_POOL_1 << qsel;
5777
5778 wr32(E1000_RAL(index), rar_low);
5779 wrfl();
5780 wr32(E1000_RAH(index), rar_high);
5781 wrfl();
5782}
5783
4ae196df
AD		 5784static int igb_set_vf_mac(struct igb_adapter *adapter,
5785 int vf, unsigned char *mac_addr)
5786{
5787 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5788 /* VF MAC addresses start at end of receive addresses and moves
5789 * torwards the first, as a result a collision should not be possible */
5790 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5791
37680117 5792 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5793
26ad9178 5794 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5795
5796 return 0;
5797}
5798
5799static void igb_vmm_control(struct igb_adapter *adapter)
5800{
5801 struct e1000_hw *hw = &adapter->hw;
10d8e907 5802 u32 reg;
4ae196df 5803
d4960307
AD		 5804 /* replication is not supported for 82575 */
5805 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5806 return;
5807
10d8e907
AD		 5808 /* enable replication vlan tag stripping */
5809 reg = rd32(E1000_RPLOLR);
5810 reg |= E1000_RPLOLR_STRVLAN;
5811 wr32(E1000_RPLOLR, reg);
5812
5813 /* notify HW that the MAC is adding vlan tags */
5814 reg = rd32(E1000_DTXCTL);
5815 reg |= E1000_DTXCTL_VLAN_ADDED;
5816 wr32(E1000_DTXCTL, reg);
5817
d4960307
AD		 5818 if (adapter->vfs_allocated_count) {
5819 igb_vmdq_set_loopback_pf(hw, true);
5820 igb_vmdq_set_replication_pf(hw, true);
5821 } else {
5822 igb_vmdq_set_loopback_pf(hw, false);
5823 igb_vmdq_set_replication_pf(hw, false);
5824 }
4ae196df
AD		 5825}
5826
9d5c8243 5827/* igb_main.c */
Clone of davem's net-next-2.6 tree