]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: count Rx FIFO errors correctly
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
55cac248 52#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
a3aa1884 63static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
55cac248
AD		 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
80 /* required last entry */
81 {0, }
82};
83
84MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
85
86void igb_reset(struct igb_adapter *);
87static int igb_setup_all_tx_resources(struct igb_adapter *);
88static int igb_setup_all_rx_resources(struct igb_adapter *);
89static void igb_free_all_tx_resources(struct igb_adapter *);
90static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 91static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 92void igb_update_stats(struct igb_adapter *);
93static int igb_probe(struct pci_dev *, const struct pci_device_id *);
94static void __devexit igb_remove(struct pci_dev *pdev);
95static int igb_sw_init(struct igb_adapter *);
96static int igb_open(struct net_device *);
97static int igb_close(struct net_device *);
98static void igb_configure_tx(struct igb_adapter *);
99static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 100static void igb_clean_all_tx_rings(struct igb_adapter *);
101static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 102static void igb_clean_tx_ring(struct igb_ring *);
103static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 104static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 105static void igb_update_phy_info(unsigned long);
106static void igb_watchdog(unsigned long);
107static void igb_watchdog_task(struct work_struct *);
b1a436c3 108static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 109static struct net_device_stats *igb_get_stats(struct net_device *);
110static int igb_change_mtu(struct net_device *, int);
111static int igb_set_mac(struct net_device *, void *);
68d480c4 112static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 113static irqreturn_t igb_intr(int irq, void *);
114static irqreturn_t igb_intr_msi(int irq, void *);
115static irqreturn_t igb_msix_other(int irq, void *);
047e0030 116static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 117#ifdef CONFIG_IGB_DCA
047e0030 118static void igb_update_dca(struct igb_q_vector *);
fe4506b6 119static void igb_setup_dca(struct igb_adapter *);
421e02f0 120#endif /* CONFIG_IGB_DCA */
047e0030 121static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 122static int igb_poll(struct napi_struct *, int);
047e0030 123static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 124static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
125static void igb_tx_timeout(struct net_device *);
126static void igb_reset_task(struct work_struct *);
127static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
128static void igb_vlan_rx_add_vid(struct net_device *, u16);
129static void igb_vlan_rx_kill_vid(struct net_device *, u16);
130static void igb_restore_vlan(struct igb_adapter *);
26ad9178 131static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 132static void igb_ping_all_vfs(struct igb_adapter *);
133static void igb_msg_task(struct igb_adapter *);
4ae196df 134static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 135static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 136static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 137static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
138static int igb_ndo_set_vf_vlan(struct net_device *netdev,
139 int vf, u16 vlan, u8 qos);
140static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
141static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
142 struct ifla_vf_info *ivi);
9d5c8243 143
9d5c8243 144#ifdef CONFIG_PM
3fe7c4c9 145static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 146static int igb_resume(struct pci_dev *);
147#endif
148static void igb_shutdown(struct pci_dev *);
421e02f0 149#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 150static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
151static struct notifier_block dca_notifier = {
152 .notifier_call = igb_notify_dca,
153 .next = NULL,
154 .priority = 0
155};
156#endif
9d5c8243
AK		 157#ifdef CONFIG_NET_POLL_CONTROLLER
158/* for netdump / net console */
159static void igb_netpoll(struct net_device *);
160#endif
37680117 161#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 162static unsigned int max_vfs = 0;
163module_param(max_vfs, uint, 0);
164MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
165 "per physical function");
166#endif /* CONFIG_PCI_IOV */
167
9d5c8243
AK		 168static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
169 pci_channel_state_t);
170static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
171static void igb_io_resume(struct pci_dev *);
172
173static struct pci_error_handlers igb_err_handler = {
174 .error_detected = igb_io_error_detected,
175 .slot_reset = igb_io_slot_reset,
176 .resume = igb_io_resume,
177};
178
179
180static struct pci_driver igb_driver = {
181 .name = igb_driver_name,
182 .id_table = igb_pci_tbl,
183 .probe = igb_probe,
184 .remove = __devexit_p(igb_remove),
185#ifdef CONFIG_PM
186 /* Power Managment Hooks */
187 .suspend = igb_suspend,
188 .resume = igb_resume,
189#endif
190 .shutdown = igb_shutdown,
191 .err_handler = &igb_err_handler
192};
193
194MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
195MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
196MODULE_LICENSE("GPL");
197MODULE_VERSION(DRV_VERSION);
198
38c845c7
PO		 199/**
200 * igb_read_clock - read raw cycle counter (to be used by time counter)
201 */
202static cycle_t igb_read_clock(const struct cyclecounter *tc)
203{
204 struct igb_adapter *adapter =
205 container_of(tc, struct igb_adapter, cycles);
206 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 207 u64 stamp = 0;
208 int shift = 0;
38c845c7 209
55cac248
AD		 210 /*
211 * The timestamp latches on lowest register read. For the 82580
212 * the lowest register is SYSTIMR instead of SYSTIML. However we never
213 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
214 */
215 if (hw->mac.type == e1000_82580) {
216 stamp = rd32(E1000_SYSTIMR) >> 8;
217 shift = IGB_82580_TSYNC_SHIFT;
218 }
219
c5b9bd5e
AD		 220 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
221 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 222 return stamp;
223}
224
9d5c8243
AK		 225#ifdef DEBUG
226/**
227 * igb_get_hw_dev_name - return device name string
228 * used by hardware layer to print debugging information
229 **/
230char *igb_get_hw_dev_name(struct e1000_hw *hw)
231{
232 struct igb_adapter *adapter = hw->back;
233 return adapter->netdev->name;
234}
38c845c7
PO		 235
236/**
237 * igb_get_time_str - format current NIC and system time as string
238 */
239static char *igb_get_time_str(struct igb_adapter *adapter,
240 char buffer[160])
241{
242 cycle_t hw = adapter->cycles.read(&adapter->cycles);
243 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
244 struct timespec sys;
245 struct timespec delta;
246 getnstimeofday(&sys);
247
248 delta = timespec_sub(nic, sys);
249
250 sprintf(buffer,
33af6bcc
PO		 251 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
252 hw,
38c845c7
PO		 253 (long)nic.tv_sec, nic.tv_nsec,
254 (long)sys.tv_sec, sys.tv_nsec,
255 (long)delta.tv_sec, delta.tv_nsec);
256
257 return buffer;
258}
9d5c8243
AK		 259#endif
260
261/**
262 * igb_init_module - Driver Registration Routine
263 *
264 * igb_init_module is the first routine called when the driver is
265 * loaded. All it does is register with the PCI subsystem.
266 **/
267static int __init igb_init_module(void)
268{
269 int ret;
270 printk(KERN_INFO "%s - version %s\n",
271 igb_driver_string, igb_driver_version);
272
273 printk(KERN_INFO "%s\n", igb_copyright);
274
421e02f0 275#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 276 dca_register_notify(&dca_notifier);
277#endif
bbd98fe4 278 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 279 return ret;
280}
281
282module_init(igb_init_module);
283
284/**
285 * igb_exit_module - Driver Exit Cleanup Routine
286 *
287 * igb_exit_module is called just before the driver is removed
288 * from memory.
289 **/
290static void __exit igb_exit_module(void)
291{
421e02f0 292#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 293 dca_unregister_notify(&dca_notifier);
294#endif
9d5c8243
AK		 295 pci_unregister_driver(&igb_driver);
296}
297
298module_exit(igb_exit_module);
299
26bc19ec
AD		 300#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
301/**
302 * igb_cache_ring_register - Descriptor ring to register mapping
303 * @adapter: board private structure to initialize
304 *
305 * Once we know the feature-set enabled for the device, we'll cache
306 * the register offset the descriptor ring is assigned to.
307 **/
308static void igb_cache_ring_register(struct igb_adapter *adapter)
309{
ee1b9f06 310 int i = 0, j = 0;
047e0030 311 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 312
313 switch (adapter->hw.mac.type) {
314 case e1000_82576:
315 /* The queues are allocated for virtualization such that VF 0
316 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
317 * In order to avoid collision we start at the first free queue
318 * and continue consuming queues in the same sequence
319 */
ee1b9f06 320 if (adapter->vfs_allocated_count) {
a99955fc 321 for (; i < adapter->rss_queues; i++)
3025a446
AD		 322 adapter->rx_ring[i]->reg_idx = rbase_offset +
323 Q_IDX_82576(i);
a99955fc 324 for (; j < adapter->rss_queues; j++)
3025a446
AD		 325 adapter->tx_ring[j]->reg_idx = rbase_offset +
326 Q_IDX_82576(j);
ee1b9f06 327 }
26bc19ec 328 case e1000_82575:
55cac248 329 case e1000_82580:
26bc19ec 330 default:
ee1b9f06 331 for (; i < adapter->num_rx_queues; i++)
3025a446 332 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 333 for (; j < adapter->num_tx_queues; j++)
3025a446 334 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 335 break;
336 }
337}
338
047e0030
AD		 339static void igb_free_queues(struct igb_adapter *adapter)
340{
3025a446 341 int i;
047e0030 342
3025a446
AD		 343 for (i = 0; i < adapter->num_tx_queues; i++) {
344 kfree(adapter->tx_ring[i]);
345 adapter->tx_ring[i] = NULL;
346 }
347 for (i = 0; i < adapter->num_rx_queues; i++) {
348 kfree(adapter->rx_ring[i]);
349 adapter->rx_ring[i] = NULL;
350 }
047e0030
AD		 351 adapter->num_rx_queues = 0;
352 adapter->num_tx_queues = 0;
353}
354
9d5c8243
AK		 355/**
356 * igb_alloc_queues - Allocate memory for all rings
357 * @adapter: board private structure to initialize
358 *
359 * We allocate one ring per queue at run-time since we don't know the
360 * number of queues at compile-time.
361 **/
362static int igb_alloc_queues(struct igb_adapter *adapter)
363{
3025a446 364 struct igb_ring *ring;
9d5c8243
AK		 365 int i;
366
661086df 367 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 368 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
369 if (!ring)
370 goto err;
68fd9910 371 ring->count = adapter->tx_ring_count;
661086df 372 ring->queue_index = i;
80785298 373 ring->pdev = adapter->pdev;
e694e964 374 ring->netdev = adapter->netdev;
85ad76b2
AD		 375 /* For 82575, context index must be unique per ring. */
376 if (adapter->hw.mac.type == e1000_82575)
377 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 378 adapter->tx_ring[i] = ring;
661086df 379 }
85ad76b2 380
9d5c8243 381 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 382 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
383 if (!ring)
384 goto err;
68fd9910 385 ring->count = adapter->rx_ring_count;
844290e5 386 ring->queue_index = i;
80785298 387 ring->pdev = adapter->pdev;
e694e964 388 ring->netdev = adapter->netdev;
4c844851 389 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 390 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
391 /* set flag indicating ring supports SCTP checksum offload */
392 if (adapter->hw.mac.type >= e1000_82576)
393 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 394 adapter->rx_ring[i] = ring;
9d5c8243 395 }
26bc19ec
AD		 396
397 igb_cache_ring_register(adapter);
9d5c8243 398
047e0030 399 return 0;
a88f10ec 400
047e0030
AD		 401err:
402 igb_free_queues(adapter);
d1a8c9e1 403
047e0030 404 return -ENOMEM;
a88f10ec
AD		 405}
406
9d5c8243 407#define IGB_N0_QUEUE -1
047e0030 408static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 409{
410 u32 msixbm = 0;
047e0030 411 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 412 struct e1000_hw *hw = &adapter->hw;
2d064c06 413 u32 ivar, index;
047e0030
AD		 414 int rx_queue = IGB_N0_QUEUE;
415 int tx_queue = IGB_N0_QUEUE;
416
417 if (q_vector->rx_ring)
418 rx_queue = q_vector->rx_ring->reg_idx;
419 if (q_vector->tx_ring)
420 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 421
422 switch (hw->mac.type) {
423 case e1000_82575:
9d5c8243
AK		 424 /* The 82575 assigns vectors using a bitmask, which matches the
425 bitmask for the EICR/EIMS/EIMC registers. To assign one
426 or more queues to a vector, we write the appropriate bits
427 into the MSIXBM register for that vector. */
047e0030 428 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 429 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 430 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 431 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 432 if (!adapter->msix_entries && msix_vector == 0)
433 msixbm |= E1000_EIMS_OTHER;
9d5c8243 434 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 435 q_vector->eims_value = msixbm;
2d064c06
AD		 436 break;
437 case e1000_82576:
26bc19ec 438 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 439 Each queue has a single entry in the table to which we write
440 a vector number along with a "valid" bit. Sadly, the layout
441 of the table is somewhat counterintuitive. */
442 if (rx_queue > IGB_N0_QUEUE) {
047e0030 443 index = (rx_queue & 0x7);
2d064c06 444 ivar = array_rd32(E1000_IVAR0, index);
047e0030 445 if (rx_queue < 8) {
26bc19ec
AD		 446 /* vector goes into low byte of register */
447 ivar = ivar & 0xFFFFFF00;
448 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 449 } else {
450 /* vector goes into third byte of register */
451 ivar = ivar & 0xFF00FFFF;
452 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 453 }
2d064c06
AD		 454 array_wr32(E1000_IVAR0, index, ivar);
455 }
456 if (tx_queue > IGB_N0_QUEUE) {
047e0030 457 index = (tx_queue & 0x7);
2d064c06 458 ivar = array_rd32(E1000_IVAR0, index);
047e0030 459 if (tx_queue < 8) {
26bc19ec
AD		 460 /* vector goes into second byte of register */
461 ivar = ivar & 0xFFFF00FF;
462 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 463 } else {
464 /* vector goes into high byte of register */
465 ivar = ivar & 0x00FFFFFF;
466 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 467 }
2d064c06
AD		 468 array_wr32(E1000_IVAR0, index, ivar);
469 }
047e0030 470 q_vector->eims_value = 1 << msix_vector;
2d064c06 471 break;
55cac248
AD		 472 case e1000_82580:
473 /* 82580 uses the same table-based approach as 82576 but has fewer
474 entries as a result we carry over for queues greater than 4. */
475 if (rx_queue > IGB_N0_QUEUE) {
476 index = (rx_queue >> 1);
477 ivar = array_rd32(E1000_IVAR0, index);
478 if (rx_queue & 0x1) {
479 /* vector goes into third byte of register */
480 ivar = ivar & 0xFF00FFFF;
481 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
482 } else {
483 /* vector goes into low byte of register */
484 ivar = ivar & 0xFFFFFF00;
485 ivar |= msix_vector | E1000_IVAR_VALID;
486 }
487 array_wr32(E1000_IVAR0, index, ivar);
488 }
489 if (tx_queue > IGB_N0_QUEUE) {
490 index = (tx_queue >> 1);
491 ivar = array_rd32(E1000_IVAR0, index);
492 if (tx_queue & 0x1) {
493 /* vector goes into high byte of register */
494 ivar = ivar & 0x00FFFFFF;
495 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
496 } else {
497 /* vector goes into second byte of register */
498 ivar = ivar & 0xFFFF00FF;
499 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
500 }
501 array_wr32(E1000_IVAR0, index, ivar);
502 }
503 q_vector->eims_value = 1 << msix_vector;
504 break;
2d064c06
AD		 505 default:
506 BUG();
507 break;
508 }
26b39276
AD		 509
510 /* add q_vector eims value to global eims_enable_mask */
511 adapter->eims_enable_mask |= q_vector->eims_value;
512
513 /* configure q_vector to set itr on first interrupt */
514 q_vector->set_itr = 1;
9d5c8243
AK		 515}
516
517/**
518 * igb_configure_msix - Configure MSI-X hardware
519 *
520 * igb_configure_msix sets up the hardware to properly
521 * generate MSI-X interrupts.
522 **/
523static void igb_configure_msix(struct igb_adapter *adapter)
524{
525 u32 tmp;
526 int i, vector = 0;
527 struct e1000_hw *hw = &adapter->hw;
528
529 adapter->eims_enable_mask = 0;
9d5c8243
AK		 530
531 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 532 switch (hw->mac.type) {
533 case e1000_82575:
9d5c8243
AK		 534 tmp = rd32(E1000_CTRL_EXT);
535 /* enable MSI-X PBA support*/
536 tmp |= E1000_CTRL_EXT_PBA_CLR;
537
538 /* Auto-Mask interrupts upon ICR read. */
539 tmp |= E1000_CTRL_EXT_EIAME;
540 tmp |= E1000_CTRL_EXT_IRCA;
541
542 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 543
544 /* enable msix_other interrupt */
545 array_wr32(E1000_MSIXBM(0), vector++,
546 E1000_EIMS_OTHER);
844290e5 547 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 548
2d064c06
AD		 549 break;
550
551 case e1000_82576:
55cac248 552 case e1000_82580:
047e0030
AD		 553 /* Turn on MSI-X capability first, or our settings
554 * won't stick. And it will take days to debug. */
555 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
556 E1000_GPIE_PBA | E1000_GPIE_EIAME |
557 E1000_GPIE_NSICR);
558
559 /* enable msix_other interrupt */
560 adapter->eims_other = 1 << vector;
2d064c06 561 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 562
047e0030 563 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 564 break;
565 default:
566 /* do nothing, since nothing else supports MSI-X */
567 break;
568 } /* switch (hw->mac.type) */
047e0030
AD		 569
570 adapter->eims_enable_mask |= adapter->eims_other;
571
26b39276
AD		 572 for (i = 0; i < adapter->num_q_vectors; i++)
573 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 574
9d5c8243
AK		 575 wrfl();
576}
577
578/**
579 * igb_request_msix - Initialize MSI-X interrupts
580 *
581 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
582 * kernel.
583 **/
584static int igb_request_msix(struct igb_adapter *adapter)
585{
586 struct net_device *netdev = adapter->netdev;
047e0030 587 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 588 int i, err = 0, vector = 0;
589
047e0030 590 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 591 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 592 if (err)
593 goto out;
594 vector++;
595
596 for (i = 0; i < adapter->num_q_vectors; i++) {
597 struct igb_q_vector *q_vector = adapter->q_vector[i];
598
599 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
600
601 if (q_vector->rx_ring && q_vector->tx_ring)
602 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
603 q_vector->rx_ring->queue_index);
604 else if (q_vector->tx_ring)
605 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
606 q_vector->tx_ring->queue_index);
607 else if (q_vector->rx_ring)
608 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
609 q_vector->rx_ring->queue_index);
9d5c8243 610 else
047e0030
AD		 611 sprintf(q_vector->name, "%s-unused", netdev->name);
612
9d5c8243 613 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 614 igb_msix_ring, 0, q_vector->name,
047e0030 615 q_vector);
9d5c8243
AK		 616 if (err)
617 goto out;
9d5c8243
AK		 618 vector++;
619 }
620
9d5c8243
AK		 621 igb_configure_msix(adapter);
622 return 0;
623out:
624 return err;
625}
626
627static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
628{
629 if (adapter->msix_entries) {
630 pci_disable_msix(adapter->pdev);
631 kfree(adapter->msix_entries);
632 adapter->msix_entries = NULL;
047e0030 633 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 634 pci_disable_msi(adapter->pdev);
047e0030 635 }
9d5c8243
AK		 636}
637
047e0030
AD		 638/**
639 * igb_free_q_vectors - Free memory allocated for interrupt vectors
640 * @adapter: board private structure to initialize
641 *
642 * This function frees the memory allocated to the q_vectors. In addition if
643 * NAPI is enabled it will delete any references to the NAPI struct prior
644 * to freeing the q_vector.
645 **/
646static void igb_free_q_vectors(struct igb_adapter *adapter)
647{
648 int v_idx;
649
650 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
651 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
652 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 653 if (!q_vector)
654 continue;
047e0030
AD		 655 netif_napi_del(&q_vector->napi);
656 kfree(q_vector);
657 }
658 adapter->num_q_vectors = 0;
659}
660
661/**
662 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
663 *
664 * This function resets the device so that it has 0 rx queues, tx queues, and
665 * MSI-X interrupts allocated.
666 */
667static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
668{
669 igb_free_queues(adapter);
670 igb_free_q_vectors(adapter);
671 igb_reset_interrupt_capability(adapter);
672}
9d5c8243
AK		 673
674/**
675 * igb_set_interrupt_capability - set MSI or MSI-X if supported
676 *
677 * Attempt to configure interrupts using the best available
678 * capabilities of the hardware and kernel.
679 **/
680static void igb_set_interrupt_capability(struct igb_adapter *adapter)
681{
682 int err;
683 int numvecs, i;
684
83b7180d 685 /* Number of supported queues. */
a99955fc
AD		 686 adapter->num_rx_queues = adapter->rss_queues;
687 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 688
047e0030
AD		 689 /* start with one vector for every rx queue */
690 numvecs = adapter->num_rx_queues;
691
3ad2f3fb 692 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 693 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
694 numvecs += adapter->num_tx_queues;
047e0030
AD		 695
696 /* store the number of vectors reserved for queues */
697 adapter->num_q_vectors = numvecs;
698
699 /* add 1 vector for link status interrupts */
700 numvecs++;
9d5c8243
AK		 701 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
702 GFP_KERNEL);
703 if (!adapter->msix_entries)
704 goto msi_only;
705
706 for (i = 0; i < numvecs; i++)
707 adapter->msix_entries[i].entry = i;
708
709 err = pci_enable_msix(adapter->pdev,
710 adapter->msix_entries,
711 numvecs);
712 if (err == 0)
34a20e89 713 goto out;
9d5c8243
AK		 714
715 igb_reset_interrupt_capability(adapter);
716
717 /* If we can't do MSI-X, try MSI */
718msi_only:
2a3abf6d
AD		 719#ifdef CONFIG_PCI_IOV
720 /* disable SR-IOV for non MSI-X configurations */
721 if (adapter->vf_data) {
722 struct e1000_hw *hw = &adapter->hw;
723 /* disable iov and allow time for transactions to clear */
724 pci_disable_sriov(adapter->pdev);
725 msleep(500);
726
727 kfree(adapter->vf_data);
728 adapter->vf_data = NULL;
729 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
730 msleep(100);
731 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
732 }
733#endif
4fc82adf 734 adapter->vfs_allocated_count = 0;
a99955fc 735 adapter->rss_queues = 1;
4fc82adf 736 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 737 adapter->num_rx_queues = 1;
661086df 738 adapter->num_tx_queues = 1;
047e0030 739 adapter->num_q_vectors = 1;
9d5c8243 740 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 741 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 742out:
661086df 743 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 744 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 745 return;
746}
747
047e0030
AD		 748/**
749 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
750 * @adapter: board private structure to initialize
751 *
752 * We allocate one q_vector per queue interrupt. If allocation fails we
753 * return -ENOMEM.
754 **/
755static int igb_alloc_q_vectors(struct igb_adapter *adapter)
756{
757 struct igb_q_vector *q_vector;
758 struct e1000_hw *hw = &adapter->hw;
759 int v_idx;
760
761 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
762 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
763 if (!q_vector)
764 goto err_out;
765 q_vector->adapter = adapter;
047e0030
AD		 766 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
767 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 768 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
769 adapter->q_vector[v_idx] = q_vector;
770 }
771 return 0;
772
773err_out:
fe0592b4 774 igb_free_q_vectors(adapter);
047e0030
AD		 775 return -ENOMEM;
776}
777
778static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
779 int ring_idx, int v_idx)
780{
3025a446 781 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 782
3025a446 783 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 784 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 785 q_vector->itr_val = adapter->rx_itr_setting;
786 if (q_vector->itr_val && q_vector->itr_val <= 3)
787 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 788}
789
790static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
791 int ring_idx, int v_idx)
792{
3025a446 793 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 794
3025a446 795 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 796 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 797 q_vector->itr_val = adapter->tx_itr_setting;
798 if (q_vector->itr_val && q_vector->itr_val <= 3)
799 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 800}
801
802/**
803 * igb_map_ring_to_vector - maps allocated queues to vectors
804 *
805 * This function maps the recently allocated queues to vectors.
806 **/
807static int igb_map_ring_to_vector(struct igb_adapter *adapter)
808{
809 int i;
810 int v_idx = 0;
811
812 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
813 (adapter->num_q_vectors < adapter->num_tx_queues))
814 return -ENOMEM;
815
816 if (adapter->num_q_vectors >=
817 (adapter->num_rx_queues + adapter->num_tx_queues)) {
818 for (i = 0; i < adapter->num_rx_queues; i++)
819 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
820 for (i = 0; i < adapter->num_tx_queues; i++)
821 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
822 } else {
823 for (i = 0; i < adapter->num_rx_queues; i++) {
824 if (i < adapter->num_tx_queues)
825 igb_map_tx_ring_to_vector(adapter, i, v_idx);
826 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
827 }
828 for (; i < adapter->num_tx_queues; i++)
829 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
830 }
831 return 0;
832}
833
834/**
835 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
836 *
837 * This function initializes the interrupts and allocates all of the queues.
838 **/
839static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
840{
841 struct pci_dev *pdev = adapter->pdev;
842 int err;
843
844 igb_set_interrupt_capability(adapter);
845
846 err = igb_alloc_q_vectors(adapter);
847 if (err) {
848 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
849 goto err_alloc_q_vectors;
850 }
851
852 err = igb_alloc_queues(adapter);
853 if (err) {
854 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
855 goto err_alloc_queues;
856 }
857
858 err = igb_map_ring_to_vector(adapter);
859 if (err) {
860 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
861 goto err_map_queues;
862 }
863
864
865 return 0;
866err_map_queues:
867 igb_free_queues(adapter);
868err_alloc_queues:
869 igb_free_q_vectors(adapter);
870err_alloc_q_vectors:
871 igb_reset_interrupt_capability(adapter);
872 return err;
873}
874
9d5c8243
AK		 875/**
876 * igb_request_irq - initialize interrupts
877 *
878 * Attempts to configure interrupts using the best available
879 * capabilities of the hardware and kernel.
880 **/
881static int igb_request_irq(struct igb_adapter *adapter)
882{
883 struct net_device *netdev = adapter->netdev;
047e0030 884 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 885 int err = 0;
886
887 if (adapter->msix_entries) {
888 err = igb_request_msix(adapter);
844290e5 889 if (!err)
9d5c8243 890 goto request_done;
9d5c8243 891 /* fall back to MSI */
047e0030 892 igb_clear_interrupt_scheme(adapter);
9d5c8243 893 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 894 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 895 igb_free_all_tx_resources(adapter);
896 igb_free_all_rx_resources(adapter);
047e0030 897 adapter->num_tx_queues = 1;
9d5c8243 898 adapter->num_rx_queues = 1;
047e0030
AD		 899 adapter->num_q_vectors = 1;
900 err = igb_alloc_q_vectors(adapter);
901 if (err) {
902 dev_err(&pdev->dev,
903 "Unable to allocate memory for vectors\n");
904 goto request_done;
905 }
906 err = igb_alloc_queues(adapter);
907 if (err) {
908 dev_err(&pdev->dev,
909 "Unable to allocate memory for queues\n");
910 igb_free_q_vectors(adapter);
911 goto request_done;
912 }
913 igb_setup_all_tx_resources(adapter);
914 igb_setup_all_rx_resources(adapter);
844290e5 915 } else {
feeb2721 916 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 917 }
844290e5 918
7dfc16fa 919 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 920 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 921 netdev->name, adapter);
9d5c8243
AK		 922 if (!err)
923 goto request_done;
047e0030 924
9d5c8243
AK		 925 /* fall back to legacy interrupts */
926 igb_reset_interrupt_capability(adapter);
7dfc16fa 927 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 928 }
929
a0607fd3 930 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 931 netdev->name, adapter);
9d5c8243 932
6cb5e577 933 if (err)
9d5c8243
AK		 934 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
935 err);
9d5c8243
AK		 936
937request_done:
938 return err;
939}
940
941static void igb_free_irq(struct igb_adapter *adapter)
942{
9d5c8243
AK		 943 if (adapter->msix_entries) {
944 int vector = 0, i;
945
047e0030 946 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 947
047e0030
AD		 948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950 free_irq(adapter->msix_entries[vector++].vector,
951 q_vector);
952 }
953 } else {
954 free_irq(adapter->pdev->irq, adapter);
9d5c8243 955 }
9d5c8243
AK		 956}
957
958/**
959 * igb_irq_disable - Mask off interrupt generation on the NIC
960 * @adapter: board private structure
961 **/
962static void igb_irq_disable(struct igb_adapter *adapter)
963{
964 struct e1000_hw *hw = &adapter->hw;
965
25568a53
AD		 966 /*
967 * we need to be careful when disabling interrupts. The VFs are also
968 * mapped into these registers and so clearing the bits can cause
969 * issues on the VF drivers so we only need to clear what we set
970 */
9d5c8243 971 if (adapter->msix_entries) {
2dfd1212
AD		 972 u32 regval = rd32(E1000_EIAM);
973 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
974 wr32(E1000_EIMC, adapter->eims_enable_mask);
975 regval = rd32(E1000_EIAC);
976 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 977 }
844290e5
PW		 978
979 wr32(E1000_IAM, 0);
9d5c8243
AK		 980 wr32(E1000_IMC, ~0);
981 wrfl();
982 synchronize_irq(adapter->pdev->irq);
983}
984
985/**
986 * igb_irq_enable - Enable default interrupt generation settings
987 * @adapter: board private structure
988 **/
989static void igb_irq_enable(struct igb_adapter *adapter)
990{
991 struct e1000_hw *hw = &adapter->hw;
992
993 if (adapter->msix_entries) {
25568a53 994 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 995 u32 regval = rd32(E1000_EIAC);
996 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
997 regval = rd32(E1000_EIAM);
998 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 999 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1000 if (adapter->vfs_allocated_count) {
4ae196df 1001 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1002 ims |= E1000_IMS_VMMB;
1003 }
55cac248
AD		 1004 if (adapter->hw.mac.type == e1000_82580)
1005 ims |= E1000_IMS_DRSTA;
1006
25568a53 1007 wr32(E1000_IMS, ims);
844290e5 1008 } else {
55cac248
AD		 1009 wr32(E1000_IMS, IMS_ENABLE_MASK |
1010 E1000_IMS_DRSTA);
1011 wr32(E1000_IAM, IMS_ENABLE_MASK |
1012 E1000_IMS_DRSTA);
844290e5 1013 }
9d5c8243
AK		 1014}
1015
1016static void igb_update_mng_vlan(struct igb_adapter *adapter)
1017{
51466239 1018 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1019 u16 vid = adapter->hw.mng_cookie.vlan_id;
1020 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1021
1022 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1023 /* add VID to filter table */
1024 igb_vfta_set(hw, vid, true);
1025 adapter->mng_vlan_id = vid;
1026 } else {
1027 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1028 }
1029
1030 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1031 (vid != old_vid) &&
1032 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1033 /* remove VID from filter table */
1034 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1035 }
1036}
1037
1038/**
1039 * igb_release_hw_control - release control of the h/w to f/w
1040 * @adapter: address of board private structure
1041 *
1042 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1043 * For ASF and Pass Through versions of f/w this means that the
1044 * driver is no longer loaded.
1045 *
1046 **/
1047static void igb_release_hw_control(struct igb_adapter *adapter)
1048{
1049 struct e1000_hw *hw = &adapter->hw;
1050 u32 ctrl_ext;
1051
1052 /* Let firmware take over control of h/w */
1053 ctrl_ext = rd32(E1000_CTRL_EXT);
1054 wr32(E1000_CTRL_EXT,
1055 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1056}
1057
9d5c8243
AK		 1058/**
1059 * igb_get_hw_control - get control of the h/w from f/w
1060 * @adapter: address of board private structure
1061 *
1062 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1063 * For ASF and Pass Through versions of f/w this means that
1064 * the driver is loaded.
1065 *
1066 **/
1067static void igb_get_hw_control(struct igb_adapter *adapter)
1068{
1069 struct e1000_hw *hw = &adapter->hw;
1070 u32 ctrl_ext;
1071
1072 /* Let firmware know the driver has taken over */
1073 ctrl_ext = rd32(E1000_CTRL_EXT);
1074 wr32(E1000_CTRL_EXT,
1075 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1076}
1077
9d5c8243
AK		 1078/**
1079 * igb_configure - configure the hardware for RX and TX
1080 * @adapter: private board structure
1081 **/
1082static void igb_configure(struct igb_adapter *adapter)
1083{
1084 struct net_device *netdev = adapter->netdev;
1085 int i;
1086
1087 igb_get_hw_control(adapter);
ff41f8dc 1088 igb_set_rx_mode(netdev);
9d5c8243
AK		 1089
1090 igb_restore_vlan(adapter);
9d5c8243 1091
85b430b4 1092 igb_setup_tctl(adapter);
06cf2666 1093 igb_setup_mrqc(adapter);
9d5c8243 1094 igb_setup_rctl(adapter);
85b430b4
AD		 1095
1096 igb_configure_tx(adapter);
9d5c8243 1097 igb_configure_rx(adapter);
662d7205
AD		 1098
1099 igb_rx_fifo_flush_82575(&adapter->hw);
1100
c493ea45 1101 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1102 * at least 1 descriptor unused to make sure
1103 * next_to_use != next_to_clean */
1104 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1105 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1106 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1107 }
1108
1109
1110 adapter->tx_queue_len = netdev->tx_queue_len;
1111}
1112
88a268c1
NN		 1113/**
1114 * igb_power_up_link - Power up the phy/serdes link
1115 * @adapter: address of board private structure
1116 **/
1117void igb_power_up_link(struct igb_adapter *adapter)
1118{
1119 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1120 igb_power_up_phy_copper(&adapter->hw);
1121 else
1122 igb_power_up_serdes_link_82575(&adapter->hw);
1123}
1124
1125/**
1126 * igb_power_down_link - Power down the phy/serdes link
1127 * @adapter: address of board private structure
1128 */
1129static void igb_power_down_link(struct igb_adapter *adapter)
1130{
1131 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1132 igb_power_down_phy_copper_82575(&adapter->hw);
1133 else
1134 igb_shutdown_serdes_link_82575(&adapter->hw);
1135}
9d5c8243
AK		 1136
1137/**
1138 * igb_up - Open the interface and prepare it to handle traffic
1139 * @adapter: board private structure
1140 **/
9d5c8243
AK		 1141int igb_up(struct igb_adapter *adapter)
1142{
1143 struct e1000_hw *hw = &adapter->hw;
1144 int i;
1145
1146 /* hardware has been reset, we need to reload some things */
1147 igb_configure(adapter);
1148
1149 clear_bit(__IGB_DOWN, &adapter->state);
1150
047e0030
AD		 1151 for (i = 0; i < adapter->num_q_vectors; i++) {
1152 struct igb_q_vector *q_vector = adapter->q_vector[i];
1153 napi_enable(&q_vector->napi);
1154 }
844290e5 1155 if (adapter->msix_entries)
9d5c8243 1156 igb_configure_msix(adapter);
feeb2721
AD		 1157 else
1158 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1159
1160 /* Clear any pending interrupts. */
1161 rd32(E1000_ICR);
1162 igb_irq_enable(adapter);
1163
d4960307
AD		 1164 /* notify VFs that reset has been completed */
1165 if (adapter->vfs_allocated_count) {
1166 u32 reg_data = rd32(E1000_CTRL_EXT);
1167 reg_data |= E1000_CTRL_EXT_PFRSTD;
1168 wr32(E1000_CTRL_EXT, reg_data);
1169 }
1170
4cb9be7a
JB		 1171 netif_tx_start_all_queues(adapter->netdev);
1172
25568a53
AD		 1173 /* start the watchdog. */
1174 hw->mac.get_link_status = 1;
1175 schedule_work(&adapter->watchdog_task);
1176
9d5c8243
AK		 1177 return 0;
1178}
1179
1180void igb_down(struct igb_adapter *adapter)
1181{
9d5c8243 1182 struct net_device *netdev = adapter->netdev;
330a6d6a 1183 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1184 u32 tctl, rctl;
1185 int i;
1186
1187 /* signal that we're down so the interrupt handler does not
1188 * reschedule our watchdog timer */
1189 set_bit(__IGB_DOWN, &adapter->state);
1190
1191 /* disable receives in the hardware */
1192 rctl = rd32(E1000_RCTL);
1193 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1194 /* flush and sleep below */
1195
fd2ea0a7 1196 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1197
1198 /* disable transmits in the hardware */
1199 tctl = rd32(E1000_TCTL);
1200 tctl &= ~E1000_TCTL_EN;
1201 wr32(E1000_TCTL, tctl);
1202 /* flush both disables and wait for them to finish */
1203 wrfl();
1204 msleep(10);
1205
047e0030
AD		 1206 for (i = 0; i < adapter->num_q_vectors; i++) {
1207 struct igb_q_vector *q_vector = adapter->q_vector[i];
1208 napi_disable(&q_vector->napi);
1209 }
9d5c8243 1210
9d5c8243
AK		 1211 igb_irq_disable(adapter);
1212
1213 del_timer_sync(&adapter->watchdog_timer);
1214 del_timer_sync(&adapter->phy_info_timer);
1215
1216 netdev->tx_queue_len = adapter->tx_queue_len;
1217 netif_carrier_off(netdev);
04fe6358
AD		 1218
1219 /* record the stats before reset*/
1220 igb_update_stats(adapter);
1221
9d5c8243
AK		 1222 adapter->link_speed = 0;
1223 adapter->link_duplex = 0;
1224
3023682e
JK		 1225 if (!pci_channel_offline(adapter->pdev))
1226 igb_reset(adapter);
9d5c8243
AK		 1227 igb_clean_all_tx_rings(adapter);
1228 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1229#ifdef CONFIG_IGB_DCA
1230
1231 /* since we reset the hardware DCA settings were cleared */
1232 igb_setup_dca(adapter);
1233#endif
9d5c8243
AK		 1234}
1235
1236void igb_reinit_locked(struct igb_adapter *adapter)
1237{
1238 WARN_ON(in_interrupt());
1239 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1240 msleep(1);
1241 igb_down(adapter);
1242 igb_up(adapter);
1243 clear_bit(__IGB_RESETTING, &adapter->state);
1244}
1245
1246void igb_reset(struct igb_adapter *adapter)
1247{
090b1795 1248 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1249 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1250 struct e1000_mac_info *mac = &hw->mac;
1251 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1252 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1253 u16 hwm;
1254
1255 /* Repartition Pba for greater than 9k mtu
1256 * To take effect CTRL.RST is required.
1257 */
fa4dfae0 1258 switch (mac->type) {
55cac248
AD		 1259 case e1000_82580:
1260 pba = rd32(E1000_RXPBS);
1261 pba = igb_rxpbs_adjust_82580(pba);
1262 break;
fa4dfae0 1263 case e1000_82576:
d249be54
AD		 1264 pba = rd32(E1000_RXPBS);
1265 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1266 break;
1267 case e1000_82575:
1268 default:
1269 pba = E1000_PBA_34K;
1270 break;
2d064c06 1271 }
9d5c8243 1272
2d064c06
AD		 1273 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1274 (mac->type < e1000_82576)) {
9d5c8243
AK		 1275 /* adjust PBA for jumbo frames */
1276 wr32(E1000_PBA, pba);
1277
1278 /* To maintain wire speed transmits, the Tx FIFO should be
1279 * large enough to accommodate two full transmit packets,
1280 * rounded up to the next 1KB and expressed in KB. Likewise,
1281 * the Rx FIFO should be large enough to accommodate at least
1282 * one full receive packet and is similarly rounded up and
1283 * expressed in KB. */
1284 pba = rd32(E1000_PBA);
1285 /* upper 16 bits has Tx packet buffer allocation size in KB */
1286 tx_space = pba >> 16;
1287 /* lower 16 bits has Rx packet buffer allocation size in KB */
1288 pba &= 0xffff;
1289 /* the tx fifo also stores 16 bytes of information about the tx
1290 * but don't include ethernet FCS because hardware appends it */
1291 min_tx_space = (adapter->max_frame_size +
85e8d004 1292 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1293 ETH_FCS_LEN) * 2;
1294 min_tx_space = ALIGN(min_tx_space, 1024);
1295 min_tx_space >>= 10;
1296 /* software strips receive CRC, so leave room for it */
1297 min_rx_space = adapter->max_frame_size;
1298 min_rx_space = ALIGN(min_rx_space, 1024);
1299 min_rx_space >>= 10;
1300
1301 /* If current Tx allocation is less than the min Tx FIFO size,
1302 * and the min Tx FIFO size is less than the current Rx FIFO
1303 * allocation, take space away from current Rx allocation */
1304 if (tx_space < min_tx_space &&
1305 ((min_tx_space - tx_space) < pba)) {
1306 pba = pba - (min_tx_space - tx_space);
1307
1308 /* if short on rx space, rx wins and must trump tx
1309 * adjustment */
1310 if (pba < min_rx_space)
1311 pba = min_rx_space;
1312 }
2d064c06 1313 wr32(E1000_PBA, pba);
9d5c8243 1314 }
9d5c8243
AK		 1315
1316 /* flow control settings */
1317 /* The high water mark must be low enough to fit one full frame
1318 * (or the size used for early receive) above it in the Rx FIFO.
1319 * Set it to the lower of:
1320 * - 90% of the Rx FIFO size, or
1321 * - the full Rx FIFO size minus one full frame */
1322 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1323 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1324
d405ea3e
AD		 1325 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1326 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1327 fc->pause_time = 0xFFFF;
1328 fc->send_xon = 1;
0cce119a 1329 fc->current_mode = fc->requested_mode;
9d5c8243 1330
4ae196df
AD		 1331 /* disable receive for all VFs and wait one second */
1332 if (adapter->vfs_allocated_count) {
1333 int i;
1334 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1335 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1336
1337 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1338 igb_ping_all_vfs(adapter);
4ae196df
AD		 1339
1340 /* disable transmits and receives */
1341 wr32(E1000_VFRE, 0);
1342 wr32(E1000_VFTE, 0);
1343 }
1344
9d5c8243 1345 /* Allow time for pending master requests to run */
330a6d6a 1346 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1347 wr32(E1000_WUC, 0);
1348
330a6d6a 1349 if (hw->mac.ops.init_hw(hw))
090b1795 1350 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1351
55cac248
AD		 1352 if (hw->mac.type == e1000_82580) {
1353 u32 reg = rd32(E1000_PCIEMISC);
1354 wr32(E1000_PCIEMISC,
1355 reg & ~E1000_PCIEMISC_LX_DECISION);
1356 }
88a268c1
NN		 1357 if (!netif_running(adapter->netdev))
1358 igb_power_down_link(adapter);
1359
9d5c8243
AK		 1360 igb_update_mng_vlan(adapter);
1361
1362 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1363 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1364
330a6d6a 1365 igb_get_phy_info(hw);
9d5c8243
AK		 1366}
1367
2e5c6922 1368static const struct net_device_ops igb_netdev_ops = {
559e9c49 1369 .ndo_open = igb_open,
2e5c6922 1370 .ndo_stop = igb_close,
00829823 1371 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1372 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1373 .ndo_set_rx_mode = igb_set_rx_mode,
1374 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1375 .ndo_set_mac_address = igb_set_mac,
1376 .ndo_change_mtu = igb_change_mtu,
1377 .ndo_do_ioctl = igb_ioctl,
1378 .ndo_tx_timeout = igb_tx_timeout,
1379 .ndo_validate_addr = eth_validate_addr,
1380 .ndo_vlan_rx_register = igb_vlan_rx_register,
1381 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1382 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1383 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1384 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1385 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1386 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1387#ifdef CONFIG_NET_POLL_CONTROLLER
1388 .ndo_poll_controller = igb_netpoll,
1389#endif
1390};
1391
9d5c8243
AK		 1392/**
1393 * igb_probe - Device Initialization Routine
1394 * @pdev: PCI device information struct
1395 * @ent: entry in igb_pci_tbl
1396 *
1397 * Returns 0 on success, negative on failure
1398 *
1399 * igb_probe initializes an adapter identified by a pci_dev structure.
1400 * The OS initialization, configuring of the adapter private structure,
1401 * and a hardware reset occur.
1402 **/
1403static int __devinit igb_probe(struct pci_dev *pdev,
1404 const struct pci_device_id *ent)
1405{
1406 struct net_device *netdev;
1407 struct igb_adapter *adapter;
1408 struct e1000_hw *hw;
4337e993
AD		 1409 u16 eeprom_data = 0;
1410 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1411 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1412 unsigned long mmio_start, mmio_len;
2d6a5e95 1413 int err, pci_using_dac;
9d5c8243
AK		 1414 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1415 u32 part_num;
1416
aed5dec3 1417 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1418 if (err)
1419 return err;
1420
1421 pci_using_dac = 0;
6a35528a 1422 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1423 if (!err) {
6a35528a 1424 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1425 if (!err)
1426 pci_using_dac = 1;
1427 } else {
284901a9 1428 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1429 if (err) {
284901a9 1430 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1431 if (err) {
1432 dev_err(&pdev->dev, "No usable DMA "
1433 "configuration, aborting\n");
1434 goto err_dma;
1435 }
1436 }
1437 }
1438
aed5dec3
AD		 1439 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1440 IORESOURCE_MEM),
1441 igb_driver_name);
9d5c8243
AK		 1442 if (err)
1443 goto err_pci_reg;
1444
19d5afd4 1445 pci_enable_pcie_error_reporting(pdev);
40a914fa 1446
9d5c8243 1447 pci_set_master(pdev);
c682fc23 1448 pci_save_state(pdev);
9d5c8243
AK		 1449
1450 err = -ENOMEM;
1bfaf07b
AD		 1451 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1452 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1453 if (!netdev)
1454 goto err_alloc_etherdev;
1455
1456 SET_NETDEV_DEV(netdev, &pdev->dev);
1457
1458 pci_set_drvdata(pdev, netdev);
1459 adapter = netdev_priv(netdev);
1460 adapter->netdev = netdev;
1461 adapter->pdev = pdev;
1462 hw = &adapter->hw;
1463 hw->back = adapter;
1464 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1465
1466 mmio_start = pci_resource_start(pdev, 0);
1467 mmio_len = pci_resource_len(pdev, 0);
1468
1469 err = -EIO;
28b0759c
AD		 1470 hw->hw_addr = ioremap(mmio_start, mmio_len);
1471 if (!hw->hw_addr)
9d5c8243
AK		 1472 goto err_ioremap;
1473
2e5c6922 1474 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1475 igb_set_ethtool_ops(netdev);
9d5c8243 1476 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1477
1478 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1479
1480 netdev->mem_start = mmio_start;
1481 netdev->mem_end = mmio_start + mmio_len;
1482
9d5c8243
AK		 1483 /* PCI config space info */
1484 hw->vendor_id = pdev->vendor;
1485 hw->device_id = pdev->device;
1486 hw->revision_id = pdev->revision;
1487 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1488 hw->subsystem_device_id = pdev->subsystem_device;
1489
9d5c8243
AK		 1490 /* Copy the default MAC, PHY and NVM function pointers */
1491 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1492 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1493 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1494 /* Initialize skew-specific constants */
1495 err = ei->get_invariants(hw);
1496 if (err)
450c87c8 1497 goto err_sw_init;
9d5c8243 1498
450c87c8 1499 /* setup the private structure */
9d5c8243
AK		 1500 err = igb_sw_init(adapter);
1501 if (err)
1502 goto err_sw_init;
1503
1504 igb_get_bus_info_pcie(hw);
1505
1506 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1507
1508 /* Copper options */
1509 if (hw->phy.media_type == e1000_media_type_copper) {
1510 hw->phy.mdix = AUTO_ALL_MODES;
1511 hw->phy.disable_polarity_correction = false;
1512 hw->phy.ms_type = e1000_ms_hw_default;
1513 }
1514
1515 if (igb_check_reset_block(hw))
1516 dev_info(&pdev->dev,
1517 "PHY reset is blocked due to SOL/IDER session.\n");
1518
1519 netdev->features = NETIF_F_SG |
7d8eb29e 1520 NETIF_F_IP_CSUM |
9d5c8243
AK		 1521 NETIF_F_HW_VLAN_TX |
1522 NETIF_F_HW_VLAN_RX |
1523 NETIF_F_HW_VLAN_FILTER;
1524
7d8eb29e 1525 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1526 netdev->features |= NETIF_F_TSO;
9d5c8243 1527 netdev->features |= NETIF_F_TSO6;
5c0999b7 1528 netdev->features |= NETIF_F_GRO;
d3352520 1529
48f29ffc
JK		 1530 netdev->vlan_features |= NETIF_F_TSO;
1531 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1532 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1533 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1534 netdev->vlan_features |= NETIF_F_SG;
1535
9d5c8243
AK		 1536 if (pci_using_dac)
1537 netdev->features |= NETIF_F_HIGHDMA;
1538
5b043fb0 1539 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1540 netdev->features |= NETIF_F_SCTP_CSUM;
1541
330a6d6a 1542 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1543
1544 /* before reading the NVM, reset the controller to put the device in a
1545 * known good starting state */
1546 hw->mac.ops.reset_hw(hw);
1547
1548 /* make sure the NVM is good */
1549 if (igb_validate_nvm_checksum(hw) < 0) {
1550 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1551 err = -EIO;
1552 goto err_eeprom;
1553 }
1554
1555 /* copy the MAC address out of the NVM */
1556 if (hw->mac.ops.read_mac_addr(hw))
1557 dev_err(&pdev->dev, "NVM Read Error\n");
1558
1559 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1560 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1561
1562 if (!is_valid_ether_addr(netdev->perm_addr)) {
1563 dev_err(&pdev->dev, "Invalid MAC Address\n");
1564 err = -EIO;
1565 goto err_eeprom;
1566 }
1567
0e340485
AD		 1568 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1569 (unsigned long) adapter);
1570 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1571 (unsigned long) adapter);
9d5c8243
AK		 1572
1573 INIT_WORK(&adapter->reset_task, igb_reset_task);
1574 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1575
450c87c8 1576 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1577 adapter->fc_autoneg = true;
1578 hw->mac.autoneg = true;
1579 hw->phy.autoneg_advertised = 0x2f;
1580
0cce119a
AD		 1581 hw->fc.requested_mode = e1000_fc_default;
1582 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1583
9d5c8243
AK		 1584 igb_validate_mdi_setting(hw);
1585
9d5c8243
AK		 1586 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1587 * enable the ACPI Magic Packet filter
1588 */
1589
a2cf8b6c 1590 if (hw->bus.func == 0)
312c75ae 1591 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1592 else if (hw->mac.type == e1000_82580)
1593 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1594 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1595 &eeprom_data);
a2cf8b6c
AD		 1596 else if (hw->bus.func == 1)
1597 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1598
1599 if (eeprom_data & eeprom_apme_mask)
1600 adapter->eeprom_wol |= E1000_WUFC_MAG;
1601
1602 /* now that we have the eeprom settings, apply the special cases where
1603 * the eeprom may be wrong or the board simply won't support wake on
1604 * lan on a particular port */
1605 switch (pdev->device) {
1606 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1607 adapter->eeprom_wol = 0;
1608 break;
1609 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1610 case E1000_DEV_ID_82576_FIBER:
1611 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1612 /* Wake events only supported on port A for dual fiber
1613 * regardless of eeprom setting */
1614 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1615 adapter->eeprom_wol = 0;
1616 break;
c8ea5ea9
AD		 1617 case E1000_DEV_ID_82576_QUAD_COPPER:
1618 /* if quad port adapter, disable WoL on all but port A */
1619 if (global_quad_port_a != 0)
1620 adapter->eeprom_wol = 0;
1621 else
1622 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1623 /* Reset for multiple quad port adapters */
1624 if (++global_quad_port_a == 4)
1625 global_quad_port_a = 0;
1626 break;
9d5c8243
AK		 1627 }
1628
1629 /* initialize the wol settings based on the eeprom settings */
1630 adapter->wol = adapter->eeprom_wol;
e1b86d84 1631 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1632
1633 /* reset the hardware with the new settings */
1634 igb_reset(adapter);
1635
1636 /* let the f/w know that the h/w is now under the control of the
1637 * driver. */
1638 igb_get_hw_control(adapter);
1639
9d5c8243
AK		 1640 strcpy(netdev->name, "eth%d");
1641 err = register_netdev(netdev);
1642 if (err)
1643 goto err_register;
1644
b168dfc5
JB		 1645 /* carrier off reporting is important to ethtool even BEFORE open */
1646 netif_carrier_off(netdev);
1647
421e02f0 1648#ifdef CONFIG_IGB_DCA
bbd98fe4 1649 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1650 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1651 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1652 igb_setup_dca(adapter);
1653 }
fe4506b6 1654
38c845c7 1655#endif
9d5c8243
AK		 1656 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1657 /* print bus type/speed/width info */
7c510e4b 1658 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1659 netdev->name,
559e9c49
AD		 1660 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1661 "unknown"),
59c3de89
AD		 1662 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1663 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1664 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1665 "unknown"),
7c510e4b 1666 netdev->dev_addr);
9d5c8243
AK		 1667
1668 igb_read_part_num(hw, &part_num);
1669 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1670 (part_num >> 8), (part_num & 0xff));
1671
1672 dev_info(&pdev->dev,
1673 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1674 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1675 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1676 adapter->num_rx_queues, adapter->num_tx_queues);
1677
9d5c8243
AK		 1678 return 0;
1679
1680err_register:
1681 igb_release_hw_control(adapter);
1682err_eeprom:
1683 if (!igb_check_reset_block(hw))
f5f4cf08 1684 igb_reset_phy(hw);
9d5c8243
AK		 1685
1686 if (hw->flash_address)
1687 iounmap(hw->flash_address);
9d5c8243 1688err_sw_init:
047e0030 1689 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1690 iounmap(hw->hw_addr);
1691err_ioremap:
1692 free_netdev(netdev);
1693err_alloc_etherdev:
559e9c49
AD		 1694 pci_release_selected_regions(pdev,
1695 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1696err_pci_reg:
1697err_dma:
1698 pci_disable_device(pdev);
1699 return err;
1700}
1701
1702/**
1703 * igb_remove - Device Removal Routine
1704 * @pdev: PCI device information struct
1705 *
1706 * igb_remove is called by the PCI subsystem to alert the driver
1707 * that it should release a PCI device. The could be caused by a
1708 * Hot-Plug event, or because the driver is going to be removed from
1709 * memory.
1710 **/
1711static void __devexit igb_remove(struct pci_dev *pdev)
1712{
1713 struct net_device *netdev = pci_get_drvdata(pdev);
1714 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1715 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1716
1717 /* flush_scheduled work may reschedule our watchdog task, so
1718 * explicitly disable watchdog tasks from being rescheduled */
1719 set_bit(__IGB_DOWN, &adapter->state);
1720 del_timer_sync(&adapter->watchdog_timer);
1721 del_timer_sync(&adapter->phy_info_timer);
1722
1723 flush_scheduled_work();
1724
421e02f0 1725#ifdef CONFIG_IGB_DCA
7dfc16fa 1726 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1727 dev_info(&pdev->dev, "DCA disabled\n");
1728 dca_remove_requester(&pdev->dev);
7dfc16fa 1729 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1730 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1731 }
1732#endif
1733
9d5c8243
AK		 1734 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1735 * would have already happened in close and is redundant. */
1736 igb_release_hw_control(adapter);
1737
1738 unregister_netdev(netdev);
1739
047e0030 1740 igb_clear_interrupt_scheme(adapter);
9d5c8243 1741
37680117
AD		 1742#ifdef CONFIG_PCI_IOV
1743 /* reclaim resources allocated to VFs */
1744 if (adapter->vf_data) {
1745 /* disable iov and allow time for transactions to clear */
1746 pci_disable_sriov(pdev);
1747 msleep(500);
1748
1749 kfree(adapter->vf_data);
1750 adapter->vf_data = NULL;
1751 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1752 msleep(100);
1753 dev_info(&pdev->dev, "IOV Disabled\n");
1754 }
1755#endif
559e9c49 1756
28b0759c
AD		 1757 iounmap(hw->hw_addr);
1758 if (hw->flash_address)
1759 iounmap(hw->flash_address);
559e9c49
AD		 1760 pci_release_selected_regions(pdev,
1761 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1762
1763 free_netdev(netdev);
1764
19d5afd4 1765 pci_disable_pcie_error_reporting(pdev);
40a914fa 1766
9d5c8243
AK		 1767 pci_disable_device(pdev);
1768}
1769
a6b623e0
AD		 1770/**
1771 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1772 * @adapter: board private structure to initialize
1773 *
1774 * This function initializes the vf specific data storage and then attempts to
1775 * allocate the VFs. The reason for ordering it this way is because it is much
1776 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1777 * the memory for the VFs.
1778 **/
1779static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1780{
1781#ifdef CONFIG_PCI_IOV
1782 struct pci_dev *pdev = adapter->pdev;
1783
1784 if (adapter->vfs_allocated_count > 7)
1785 adapter->vfs_allocated_count = 7;
1786
1787 if (adapter->vfs_allocated_count) {
1788 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1789 sizeof(struct vf_data_storage),
1790 GFP_KERNEL);
1791 /* if allocation failed then we do not support SR-IOV */
1792 if (!adapter->vf_data) {
1793 adapter->vfs_allocated_count = 0;
1794 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1795 "Data Storage\n");
1796 }
1797 }
1798
1799 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1800 kfree(adapter->vf_data);
1801 adapter->vf_data = NULL;
1802#endif /* CONFIG_PCI_IOV */
1803 adapter->vfs_allocated_count = 0;
1804#ifdef CONFIG_PCI_IOV
1805 } else {
1806 unsigned char mac_addr[ETH_ALEN];
1807 int i;
1808 dev_info(&pdev->dev, "%d vfs allocated\n",
1809 adapter->vfs_allocated_count);
1810 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1811 random_ether_addr(mac_addr);
1812 igb_set_vf_mac(adapter, i, mac_addr);
1813 }
1814 }
1815#endif /* CONFIG_PCI_IOV */
1816}
1817
115f459a
AD		 1818
1819/**
1820 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1821 * @adapter: board private structure to initialize
1822 *
1823 * igb_init_hw_timer initializes the function pointer and values for the hw
1824 * timer found in hardware.
1825 **/
1826static void igb_init_hw_timer(struct igb_adapter *adapter)
1827{
1828 struct e1000_hw *hw = &adapter->hw;
1829
1830 switch (hw->mac.type) {
55cac248
AD		 1831 case e1000_82580:
1832 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1833 adapter->cycles.read = igb_read_clock;
1834 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1835 adapter->cycles.mult = 1;
1836 /*
1837 * The 82580 timesync updates the system timer every 8ns by 8ns
1838 * and the value cannot be shifted. Instead we need to shift
1839 * the registers to generate a 64bit timer value. As a result
1840 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1841 * 24 in order to generate a larger value for synchronization.
1842 */
1843 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1844 /* disable system timer temporarily by setting bit 31 */
1845 wr32(E1000_TSAUXC, 0x80000000);
1846 wrfl();
1847
1848 /* Set registers so that rollover occurs soon to test this. */
1849 wr32(E1000_SYSTIMR, 0x00000000);
1850 wr32(E1000_SYSTIML, 0x80000000);
1851 wr32(E1000_SYSTIMH, 0x000000FF);
1852 wrfl();
1853
1854 /* enable system timer by clearing bit 31 */
1855 wr32(E1000_TSAUXC, 0x0);
1856 wrfl();
1857
1858 timecounter_init(&adapter->clock,
1859 &adapter->cycles,
1860 ktime_to_ns(ktime_get_real()));
1861 /*
1862 * Synchronize our NIC clock against system wall clock. NIC
1863 * time stamp reading requires ~3us per sample, each sample
1864 * was pretty stable even under load => only require 10
1865 * samples for each offset comparison.
1866 */
1867 memset(&adapter->compare, 0, sizeof(adapter->compare));
1868 adapter->compare.source = &adapter->clock;
1869 adapter->compare.target = ktime_get_real;
1870 adapter->compare.num_samples = 10;
1871 timecompare_update(&adapter->compare, 0);
1872 break;
115f459a
AD		 1873 case e1000_82576:
1874 /*
1875 * Initialize hardware timer: we keep it running just in case
1876 * that some program needs it later on.
1877 */
1878 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1879 adapter->cycles.read = igb_read_clock;
1880 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1881 adapter->cycles.mult = 1;
1882 /**
1883 * Scale the NIC clock cycle by a large factor so that
1884 * relatively small clock corrections can be added or
1885 * substracted at each clock tick. The drawbacks of a large
1886 * factor are a) that the clock register overflows more quickly
1887 * (not such a big deal) and b) that the increment per tick has
1888 * to fit into 24 bits. As a result we need to use a shift of
1889 * 19 so we can fit a value of 16 into the TIMINCA register.
1890 */
1891 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1892 wr32(E1000_TIMINCA,
1893 (1 << E1000_TIMINCA_16NS_SHIFT) |
1894 (16 << IGB_82576_TSYNC_SHIFT));
1895
1896 /* Set registers so that rollover occurs soon to test this. */
1897 wr32(E1000_SYSTIML, 0x00000000);
1898 wr32(E1000_SYSTIMH, 0xFF800000);
1899 wrfl();
1900
1901 timecounter_init(&adapter->clock,
1902 &adapter->cycles,
1903 ktime_to_ns(ktime_get_real()));
1904 /*
1905 * Synchronize our NIC clock against system wall clock. NIC
1906 * time stamp reading requires ~3us per sample, each sample
1907 * was pretty stable even under load => only require 10
1908 * samples for each offset comparison.
1909 */
1910 memset(&adapter->compare, 0, sizeof(adapter->compare));
1911 adapter->compare.source = &adapter->clock;
1912 adapter->compare.target = ktime_get_real;
1913 adapter->compare.num_samples = 10;
1914 timecompare_update(&adapter->compare, 0);
1915 break;
1916 case e1000_82575:
1917 /* 82575 does not support timesync */
1918 default:
1919 break;
1920 }
1921
1922}
1923
9d5c8243
AK		 1924/**
1925 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1926 * @adapter: board private structure to initialize
1927 *
1928 * igb_sw_init initializes the Adapter private data structure.
1929 * Fields are initialized based on PCI device information and
1930 * OS network device settings (MTU size).
1931 **/
1932static int __devinit igb_sw_init(struct igb_adapter *adapter)
1933{
1934 struct e1000_hw *hw = &adapter->hw;
1935 struct net_device *netdev = adapter->netdev;
1936 struct pci_dev *pdev = adapter->pdev;
1937
1938 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1939
68fd9910
AD		 1940 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1941 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1942 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1943 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1944
9d5c8243
AK		 1945 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1946 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1947
a6b623e0
AD		 1948#ifdef CONFIG_PCI_IOV
1949 if (hw->mac.type == e1000_82576)
1950 adapter->vfs_allocated_count = max_vfs;
1951
1952#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 1953 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1954
1955 /*
1956 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1957 * then we should combine the queues into a queue pair in order to
1958 * conserve interrupts due to limited supply
1959 */
1960 if ((adapter->rss_queues > 4) ||
1961 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1962 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1963
a6b623e0 1964 /* This call may decrease the number of queues */
047e0030 1965 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1966 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1967 return -ENOMEM;
1968 }
1969
115f459a 1970 igb_init_hw_timer(adapter);
a6b623e0
AD		 1971 igb_probe_vfs(adapter);
1972
9d5c8243
AK		 1973 /* Explicitly disable IRQ since the NIC can be in any state. */
1974 igb_irq_disable(adapter);
1975
1976 set_bit(__IGB_DOWN, &adapter->state);
1977 return 0;
1978}
1979
1980/**
1981 * igb_open - Called when a network interface is made active
1982 * @netdev: network interface device structure
1983 *
1984 * Returns 0 on success, negative value on failure
1985 *
1986 * The open entry point is called when a network interface is made
1987 * active by the system (IFF_UP). At this point all resources needed
1988 * for transmit and receive operations are allocated, the interrupt
1989 * handler is registered with the OS, the watchdog timer is started,
1990 * and the stack is notified that the interface is ready.
1991 **/
1992static int igb_open(struct net_device *netdev)
1993{
1994 struct igb_adapter *adapter = netdev_priv(netdev);
1995 struct e1000_hw *hw = &adapter->hw;
1996 int err;
1997 int i;
1998
1999 /* disallow open during test */
2000 if (test_bit(__IGB_TESTING, &adapter->state))
2001 return -EBUSY;
2002
b168dfc5
JB		 2003 netif_carrier_off(netdev);
2004
9d5c8243
AK		 2005 /* allocate transmit descriptors */
2006 err = igb_setup_all_tx_resources(adapter);
2007 if (err)
2008 goto err_setup_tx;
2009
2010 /* allocate receive descriptors */
2011 err = igb_setup_all_rx_resources(adapter);
2012 if (err)
2013 goto err_setup_rx;
2014
88a268c1 2015 igb_power_up_link(adapter);
9d5c8243 2016
9d5c8243
AK		 2017 /* before we allocate an interrupt, we must be ready to handle it.
2018 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2019 * as soon as we call pci_request_irq, so we have to setup our
2020 * clean_rx handler before we do so. */
2021 igb_configure(adapter);
2022
2023 err = igb_request_irq(adapter);
2024 if (err)
2025 goto err_req_irq;
2026
2027 /* From here on the code is the same as igb_up() */
2028 clear_bit(__IGB_DOWN, &adapter->state);
2029
047e0030
AD		 2030 for (i = 0; i < adapter->num_q_vectors; i++) {
2031 struct igb_q_vector *q_vector = adapter->q_vector[i];
2032 napi_enable(&q_vector->napi);
2033 }
9d5c8243
AK		 2034
2035 /* Clear any pending interrupts. */
2036 rd32(E1000_ICR);
844290e5
PW		 2037
2038 igb_irq_enable(adapter);
2039
d4960307
AD		 2040 /* notify VFs that reset has been completed */
2041 if (adapter->vfs_allocated_count) {
2042 u32 reg_data = rd32(E1000_CTRL_EXT);
2043 reg_data |= E1000_CTRL_EXT_PFRSTD;
2044 wr32(E1000_CTRL_EXT, reg_data);
2045 }
2046
d55b53ff
JK		 2047 netif_tx_start_all_queues(netdev);
2048
25568a53
AD		 2049 /* start the watchdog. */
2050 hw->mac.get_link_status = 1;
2051 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2052
2053 return 0;
2054
2055err_req_irq:
2056 igb_release_hw_control(adapter);
88a268c1 2057 igb_power_down_link(adapter);
9d5c8243
AK		 2058 igb_free_all_rx_resources(adapter);
2059err_setup_rx:
2060 igb_free_all_tx_resources(adapter);
2061err_setup_tx:
2062 igb_reset(adapter);
2063
2064 return err;
2065}
2066
2067/**
2068 * igb_close - Disables a network interface
2069 * @netdev: network interface device structure
2070 *
2071 * Returns 0, this is not allowed to fail
2072 *
2073 * The close entry point is called when an interface is de-activated
2074 * by the OS. The hardware is still under the driver's control, but
2075 * needs to be disabled. A global MAC reset is issued to stop the
2076 * hardware, and all transmit and receive resources are freed.
2077 **/
2078static int igb_close(struct net_device *netdev)
2079{
2080 struct igb_adapter *adapter = netdev_priv(netdev);
2081
2082 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2083 igb_down(adapter);
2084
2085 igb_free_irq(adapter);
2086
2087 igb_free_all_tx_resources(adapter);
2088 igb_free_all_rx_resources(adapter);
2089
9d5c8243
AK		 2090 return 0;
2091}
2092
2093/**
2094 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2095 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2096 *
2097 * Return 0 on success, negative on failure
2098 **/
80785298 2099int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2100{
80785298 2101 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2102 int size;
2103
2104 size = sizeof(struct igb_buffer) * tx_ring->count;
2105 tx_ring->buffer_info = vmalloc(size);
2106 if (!tx_ring->buffer_info)
2107 goto err;
2108 memset(tx_ring->buffer_info, 0, size);
2109
2110 /* round up to nearest 4K */
85e8d004 2111 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2112 tx_ring->size = ALIGN(tx_ring->size, 4096);
2113
439705e1
AD		 2114 tx_ring->desc = pci_alloc_consistent(pdev,
2115 tx_ring->size,
9d5c8243
AK		 2116 &tx_ring->dma);
2117
2118 if (!tx_ring->desc)
2119 goto err;
2120
9d5c8243
AK		 2121 tx_ring->next_to_use = 0;
2122 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2123 return 0;
2124
2125err:
2126 vfree(tx_ring->buffer_info);
047e0030 2127 dev_err(&pdev->dev,
9d5c8243
AK		 2128 "Unable to allocate memory for the transmit descriptor ring\n");
2129 return -ENOMEM;
2130}
2131
2132/**
2133 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2134 * (Descriptors) for all queues
2135 * @adapter: board private structure
2136 *
2137 * Return 0 on success, negative on failure
2138 **/
2139static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2140{
439705e1 2141 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2142 int i, err = 0;
2143
2144 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2145 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2146 if (err) {
439705e1 2147 dev_err(&pdev->dev,
9d5c8243
AK		 2148 "Allocation for Tx Queue %u failed\n", i);
2149 for (i--; i >= 0; i--)
3025a446 2150 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2151 break;
2152 }
2153 }
2154
a99955fc 2155 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2156 int r_idx = i % adapter->num_tx_queues;
3025a446 2157 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2158 }
9d5c8243
AK		 2159 return err;
2160}
2161
2162/**
85b430b4
AD		 2163 * igb_setup_tctl - configure the transmit control registers
2164 * @adapter: Board private structure
9d5c8243 2165 **/
d7ee5b3a 2166void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2167{
9d5c8243
AK		 2168 struct e1000_hw *hw = &adapter->hw;
2169 u32 tctl;
9d5c8243 2170
85b430b4
AD		 2171 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2172 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2173
2174 /* Program the Transmit Control Register */
9d5c8243
AK		 2175 tctl = rd32(E1000_TCTL);
2176 tctl &= ~E1000_TCTL_CT;
2177 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2178 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2179
2180 igb_config_collision_dist(hw);
2181
9d5c8243
AK		 2182 /* Enable transmits */
2183 tctl |= E1000_TCTL_EN;
2184
2185 wr32(E1000_TCTL, tctl);
2186}
2187
85b430b4
AD		 2188/**
2189 * igb_configure_tx_ring - Configure transmit ring after Reset
2190 * @adapter: board private structure
2191 * @ring: tx ring to configure
2192 *
2193 * Configure a transmit ring after a reset.
2194 **/
d7ee5b3a
AD		 2195void igb_configure_tx_ring(struct igb_adapter *adapter,
2196 struct igb_ring *ring)
85b430b4
AD		 2197{
2198 struct e1000_hw *hw = &adapter->hw;
2199 u32 txdctl;
2200 u64 tdba = ring->dma;
2201 int reg_idx = ring->reg_idx;
2202
2203 /* disable the queue */
2204 txdctl = rd32(E1000_TXDCTL(reg_idx));
2205 wr32(E1000_TXDCTL(reg_idx),
2206 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2207 wrfl();
2208 mdelay(10);
2209
2210 wr32(E1000_TDLEN(reg_idx),
2211 ring->count * sizeof(union e1000_adv_tx_desc));
2212 wr32(E1000_TDBAL(reg_idx),
2213 tdba & 0x00000000ffffffffULL);
2214 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2215
fce99e34
AD		 2216 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2217 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2218 writel(0, ring->head);
2219 writel(0, ring->tail);
85b430b4
AD		 2220
2221 txdctl |= IGB_TX_PTHRESH;
2222 txdctl |= IGB_TX_HTHRESH << 8;
2223 txdctl |= IGB_TX_WTHRESH << 16;
2224
2225 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2226 wr32(E1000_TXDCTL(reg_idx), txdctl);
2227}
2228
2229/**
2230 * igb_configure_tx - Configure transmit Unit after Reset
2231 * @adapter: board private structure
2232 *
2233 * Configure the Tx unit of the MAC after a reset.
2234 **/
2235static void igb_configure_tx(struct igb_adapter *adapter)
2236{
2237 int i;
2238
2239 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2240 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2241}
2242
9d5c8243
AK		 2243/**
2244 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2245 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2246 *
2247 * Returns 0 on success, negative on failure
2248 **/
80785298 2249int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2250{
80785298 2251 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2252 int size, desc_len;
2253
2254 size = sizeof(struct igb_buffer) * rx_ring->count;
2255 rx_ring->buffer_info = vmalloc(size);
2256 if (!rx_ring->buffer_info)
2257 goto err;
2258 memset(rx_ring->buffer_info, 0, size);
2259
2260 desc_len = sizeof(union e1000_adv_rx_desc);
2261
2262 /* Round up to nearest 4K */
2263 rx_ring->size = rx_ring->count * desc_len;
2264 rx_ring->size = ALIGN(rx_ring->size, 4096);
2265
2266 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2267 &rx_ring->dma);
2268
2269 if (!rx_ring->desc)
2270 goto err;
2271
2272 rx_ring->next_to_clean = 0;
2273 rx_ring->next_to_use = 0;
9d5c8243 2274
9d5c8243
AK		 2275 return 0;
2276
2277err:
2278 vfree(rx_ring->buffer_info);
439705e1 2279 rx_ring->buffer_info = NULL;
80785298 2280 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2281 "the receive descriptor ring\n");
2282 return -ENOMEM;
2283}
2284
2285/**
2286 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2287 * (Descriptors) for all queues
2288 * @adapter: board private structure
2289 *
2290 * Return 0 on success, negative on failure
2291 **/
2292static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2293{
439705e1 2294 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2295 int i, err = 0;
2296
2297 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2298 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2299 if (err) {
439705e1 2300 dev_err(&pdev->dev,
9d5c8243
AK		 2301 "Allocation for Rx Queue %u failed\n", i);
2302 for (i--; i >= 0; i--)
3025a446 2303 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2304 break;
2305 }
2306 }
2307
2308 return err;
2309}
2310
06cf2666
AD		 2311/**
2312 * igb_setup_mrqc - configure the multiple receive queue control registers
2313 * @adapter: Board private structure
2314 **/
2315static void igb_setup_mrqc(struct igb_adapter *adapter)
2316{
2317 struct e1000_hw *hw = &adapter->hw;
2318 u32 mrqc, rxcsum;
2319 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2320 union e1000_reta {
2321 u32 dword;
2322 u8 bytes[4];
2323 } reta;
2324 static const u8 rsshash[40] = {
2325 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2326 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2327 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2328 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2329
2330 /* Fill out hash function seeds */
2331 for (j = 0; j < 10; j++) {
2332 u32 rsskey = rsshash[(j * 4)];
2333 rsskey |= rsshash[(j * 4) + 1] << 8;
2334 rsskey |= rsshash[(j * 4) + 2] << 16;
2335 rsskey |= rsshash[(j * 4) + 3] << 24;
2336 array_wr32(E1000_RSSRK(0), j, rsskey);
2337 }
2338
a99955fc 2339 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2340
2341 if (adapter->vfs_allocated_count) {
2342 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2343 switch (hw->mac.type) {
55cac248
AD		 2344 case e1000_82580:
2345 num_rx_queues = 1;
2346 shift = 0;
2347 break;
06cf2666
AD		 2348 case e1000_82576:
2349 shift = 3;
2350 num_rx_queues = 2;
2351 break;
2352 case e1000_82575:
2353 shift = 2;
2354 shift2 = 6;
2355 default:
2356 break;
2357 }
2358 } else {
2359 if (hw->mac.type == e1000_82575)
2360 shift = 6;
2361 }
2362
2363 for (j = 0; j < (32 * 4); j++) {
2364 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2365 if (shift2)
2366 reta.bytes[j & 3] |= num_rx_queues << shift2;
2367 if ((j & 3) == 3)
2368 wr32(E1000_RETA(j >> 2), reta.dword);
2369 }
2370
2371 /*
2372 * Disable raw packet checksumming so that RSS hash is placed in
2373 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2374 * offloads as they are enabled by default
2375 */
2376 rxcsum = rd32(E1000_RXCSUM);
2377 rxcsum |= E1000_RXCSUM_PCSD;
2378
2379 if (adapter->hw.mac.type >= e1000_82576)
2380 /* Enable Receive Checksum Offload for SCTP */
2381 rxcsum |= E1000_RXCSUM_CRCOFL;
2382
2383 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2384 wr32(E1000_RXCSUM, rxcsum);
2385
2386 /* If VMDq is enabled then we set the appropriate mode for that, else
2387 * we default to RSS so that an RSS hash is calculated per packet even
2388 * if we are only using one queue */
2389 if (adapter->vfs_allocated_count) {
2390 if (hw->mac.type > e1000_82575) {
2391 /* Set the default pool for the PF's first queue */
2392 u32 vtctl = rd32(E1000_VT_CTL);
2393 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2394 E1000_VT_CTL_DISABLE_DEF_POOL);
2395 vtctl |= adapter->vfs_allocated_count <<
2396 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2397 wr32(E1000_VT_CTL, vtctl);
2398 }
a99955fc 2399 if (adapter->rss_queues > 1)
06cf2666
AD		 2400 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2401 else
2402 mrqc = E1000_MRQC_ENABLE_VMDQ;
2403 } else {
2404 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2405 }
2406 igb_vmm_control(adapter);
2407
2408 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2409 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2410 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2411 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2412 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2413 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2414 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2415 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2416
2417 wr32(E1000_MRQC, mrqc);
2418}
2419
9d5c8243
AK		 2420/**
2421 * igb_setup_rctl - configure the receive control registers
2422 * @adapter: Board private structure
2423 **/
d7ee5b3a 2424void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2425{
2426 struct e1000_hw *hw = &adapter->hw;
2427 u32 rctl;
9d5c8243
AK		 2428
2429 rctl = rd32(E1000_RCTL);
2430
2431 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2432 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2433
69d728ba 2434 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2435 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2436
87cb7e8c
AK		 2437 /*
2438 * enable stripping of CRC. It's unlikely this will break BMC
2439 * redirection as it did with e1000. Newer features require
2440 * that the HW strips the CRC.
73cd78f1 2441 */
87cb7e8c 2442 rctl |= E1000_RCTL_SECRC;
9d5c8243 2443
559e9c49 2444 /* disable store bad packets and clear size bits. */
ec54d7d6 2445 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2446
6ec43fe6
AD		 2447 /* enable LPE to prevent packets larger than max_frame_size */
2448 rctl |= E1000_RCTL_LPE;
9d5c8243 2449
952f72a8
AD		 2450 /* disable queue 0 to prevent tail write w/o re-config */
2451 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2452
e1739522
AD		 2453 /* Attention!!! For SR-IOV PF driver operations you must enable
2454 * queue drop for all VF and PF queues to prevent head of line blocking
2455 * if an un-trusted VF does not provide descriptors to hardware.
2456 */
2457 if (adapter->vfs_allocated_count) {
e1739522
AD		 2458 /* set all queue drop enable bits */
2459 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2460 }
2461
9d5c8243
AK		 2462 wr32(E1000_RCTL, rctl);
2463}
2464
7d5753f0
AD		 2465static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2466 int vfn)
2467{
2468 struct e1000_hw *hw = &adapter->hw;
2469 u32 vmolr;
2470
2471 /* if it isn't the PF check to see if VFs are enabled and
2472 * increase the size to support vlan tags */
2473 if (vfn < adapter->vfs_allocated_count &&
2474 adapter->vf_data[vfn].vlans_enabled)
2475 size += VLAN_TAG_SIZE;
2476
2477 vmolr = rd32(E1000_VMOLR(vfn));
2478 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2479 vmolr |= size | E1000_VMOLR_LPE;
2480 wr32(E1000_VMOLR(vfn), vmolr);
2481
2482 return 0;
2483}
2484
e1739522
AD		 2485/**
2486 * igb_rlpml_set - set maximum receive packet size
2487 * @adapter: board private structure
2488 *
2489 * Configure maximum receivable packet size.
2490 **/
2491static void igb_rlpml_set(struct igb_adapter *adapter)
2492{
2493 u32 max_frame_size = adapter->max_frame_size;
2494 struct e1000_hw *hw = &adapter->hw;
2495 u16 pf_id = adapter->vfs_allocated_count;
2496
2497 if (adapter->vlgrp)
2498 max_frame_size += VLAN_TAG_SIZE;
2499
2500 /* if vfs are enabled we set RLPML to the largest possible request
2501 * size and set the VMOLR RLPML to the size we need */
2502 if (pf_id) {
2503 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2504 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2505 }
2506
2507 wr32(E1000_RLPML, max_frame_size);
2508}
2509
8151d294
WM		 2510static inline void igb_set_vmolr(struct igb_adapter *adapter,
2511 int vfn, bool aupe)
7d5753f0
AD		 2512{
2513 struct e1000_hw *hw = &adapter->hw;
2514 u32 vmolr;
2515
2516 /*
2517 * This register exists only on 82576 and newer so if we are older then
2518 * we should exit and do nothing
2519 */
2520 if (hw->mac.type < e1000_82576)
2521 return;
2522
2523 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2524 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2525 if (aupe)
2526 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2527 else
2528 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2529
2530 /* clear all bits that might not be set */
2531 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2532
a99955fc 2533 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2534 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2535 /*
2536 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2537 * multicast packets
2538 */
2539 if (vfn <= adapter->vfs_allocated_count)
2540 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2541
2542 wr32(E1000_VMOLR(vfn), vmolr);
2543}
2544
85b430b4
AD		 2545/**
2546 * igb_configure_rx_ring - Configure a receive ring after Reset
2547 * @adapter: board private structure
2548 * @ring: receive ring to be configured
2549 *
2550 * Configure the Rx unit of the MAC after a reset.
2551 **/
d7ee5b3a
AD		 2552void igb_configure_rx_ring(struct igb_adapter *adapter,
2553 struct igb_ring *ring)
85b430b4
AD		 2554{
2555 struct e1000_hw *hw = &adapter->hw;
2556 u64 rdba = ring->dma;
2557 int reg_idx = ring->reg_idx;
952f72a8 2558 u32 srrctl, rxdctl;
85b430b4
AD		 2559
2560 /* disable the queue */
2561 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2562 wr32(E1000_RXDCTL(reg_idx),
2563 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2564
2565 /* Set DMA base address registers */
2566 wr32(E1000_RDBAL(reg_idx),
2567 rdba & 0x00000000ffffffffULL);
2568 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2569 wr32(E1000_RDLEN(reg_idx),
2570 ring->count * sizeof(union e1000_adv_rx_desc));
2571
2572 /* initialize head and tail */
fce99e34
AD		 2573 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2574 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2575 writel(0, ring->head);
2576 writel(0, ring->tail);
85b430b4 2577
952f72a8 2578 /* set descriptor configuration */
4c844851
AD		 2579 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2580 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2581 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2582#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2583 srrctl |= IGB_RXBUFFER_16384 >>
2584 E1000_SRRCTL_BSIZEPKT_SHIFT;
2585#else
2586 srrctl |= (PAGE_SIZE / 2) >>
2587 E1000_SRRCTL_BSIZEPKT_SHIFT;
2588#endif
2589 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2590 } else {
4c844851 2591 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2592 E1000_SRRCTL_BSIZEPKT_SHIFT;
2593 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2594 }
e6bdb6fe
NN		 2595 /* Only set Drop Enable if we are supporting multiple queues */
2596 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2597 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2598
2599 wr32(E1000_SRRCTL(reg_idx), srrctl);
2600
7d5753f0 2601 /* set filtering for VMDQ pools */
8151d294 2602 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2603
85b430b4
AD		 2604 /* enable receive descriptor fetching */
2605 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2606 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2607 rxdctl &= 0xFFF00000;
2608 rxdctl |= IGB_RX_PTHRESH;
2609 rxdctl |= IGB_RX_HTHRESH << 8;
2610 rxdctl |= IGB_RX_WTHRESH << 16;
2611 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2612}
2613
9d5c8243
AK		 2614/**
2615 * igb_configure_rx - Configure receive Unit after Reset
2616 * @adapter: board private structure
2617 *
2618 * Configure the Rx unit of the MAC after a reset.
2619 **/
2620static void igb_configure_rx(struct igb_adapter *adapter)
2621{
9107584e 2622 int i;
9d5c8243 2623
68d480c4
AD		 2624 /* set UTA to appropriate mode */
2625 igb_set_uta(adapter);
2626
26ad9178
AD		 2627 /* set the correct pool for the PF default MAC address in entry 0 */
2628 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2629 adapter->vfs_allocated_count);
2630
06cf2666
AD		 2631 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2632 * the Base and Length of the Rx Descriptor Ring */
2633 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2634 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2635}
2636
2637/**
2638 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2639 * @tx_ring: Tx descriptor ring for a specific queue
2640 *
2641 * Free all transmit software resources
2642 **/
68fd9910 2643void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2644{
3b644cf6 2645 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2646
2647 vfree(tx_ring->buffer_info);
2648 tx_ring->buffer_info = NULL;
2649
439705e1
AD		 2650 /* if not set, then don't free */
2651 if (!tx_ring->desc)
2652 return;
2653
80785298
AD		 2654 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2655 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2656
2657 tx_ring->desc = NULL;
2658}
2659
2660/**
2661 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2662 * @adapter: board private structure
2663 *
2664 * Free all transmit software resources
2665 **/
2666static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2667{
2668 int i;
2669
2670 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2671 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2672}
2673
b1a436c3
AD		 2674void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2675 struct igb_buffer *buffer_info)
9d5c8243 2676{
6366ad33
AD		 2677 if (buffer_info->dma) {
2678 if (buffer_info->mapped_as_page)
2679 pci_unmap_page(tx_ring->pdev,
2680 buffer_info->dma,
2681 buffer_info->length,
2682 PCI_DMA_TODEVICE);
2683 else
2684 pci_unmap_single(tx_ring->pdev,
2685 buffer_info->dma,
2686 buffer_info->length,
2687 PCI_DMA_TODEVICE);
2688 buffer_info->dma = 0;
2689 }
9d5c8243
AK		 2690 if (buffer_info->skb) {
2691 dev_kfree_skb_any(buffer_info->skb);
2692 buffer_info->skb = NULL;
2693 }
2694 buffer_info->time_stamp = 0;
6366ad33
AD		 2695 buffer_info->length = 0;
2696 buffer_info->next_to_watch = 0;
2697 buffer_info->mapped_as_page = false;
9d5c8243
AK		 2698}
2699
2700/**
2701 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2702 * @tx_ring: ring to be cleaned
2703 **/
3b644cf6 2704static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2705{
2706 struct igb_buffer *buffer_info;
2707 unsigned long size;
2708 unsigned int i;
2709
2710 if (!tx_ring->buffer_info)
2711 return;
2712 /* Free all the Tx ring sk_buffs */
2713
2714 for (i = 0; i < tx_ring->count; i++) {
2715 buffer_info = &tx_ring->buffer_info[i];
80785298 2716 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2717 }
2718
2719 size = sizeof(struct igb_buffer) * tx_ring->count;
2720 memset(tx_ring->buffer_info, 0, size);
2721
2722 /* Zero out the descriptor ring */
9d5c8243
AK		 2723 memset(tx_ring->desc, 0, tx_ring->size);
2724
2725 tx_ring->next_to_use = 0;
2726 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2727}
2728
2729/**
2730 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2731 * @adapter: board private structure
2732 **/
2733static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2734{
2735 int i;
2736
2737 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2738 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 2739}
2740
2741/**
2742 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2743 * @rx_ring: ring to clean the resources from
2744 *
2745 * Free all receive software resources
2746 **/
68fd9910 2747void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2748{
3b644cf6 2749 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2750
2751 vfree(rx_ring->buffer_info);
2752 rx_ring->buffer_info = NULL;
2753
439705e1
AD		 2754 /* if not set, then don't free */
2755 if (!rx_ring->desc)
2756 return;
2757
80785298
AD		 2758 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2759 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2760
2761 rx_ring->desc = NULL;
2762}
2763
2764/**
2765 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2766 * @adapter: board private structure
2767 *
2768 * Free all receive software resources
2769 **/
2770static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2771{
2772 int i;
2773
2774 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2775 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2776}
2777
2778/**
2779 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2780 * @rx_ring: ring to free buffers from
2781 **/
3b644cf6 2782static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2783{
2784 struct igb_buffer *buffer_info;
9d5c8243
AK		 2785 unsigned long size;
2786 unsigned int i;
2787
2788 if (!rx_ring->buffer_info)
2789 return;
439705e1 2790
9d5c8243
AK		 2791 /* Free all the Rx ring sk_buffs */
2792 for (i = 0; i < rx_ring->count; i++) {
2793 buffer_info = &rx_ring->buffer_info[i];
2794 if (buffer_info->dma) {
80785298
AD		 2795 pci_unmap_single(rx_ring->pdev,
2796 buffer_info->dma,
4c844851 2797 rx_ring->rx_buffer_len,
6ec43fe6 2798 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2799 buffer_info->dma = 0;
2800 }
2801
2802 if (buffer_info->skb) {
2803 dev_kfree_skb(buffer_info->skb);
2804 buffer_info->skb = NULL;
2805 }
6ec43fe6 2806 if (buffer_info->page_dma) {
80785298
AD		 2807 pci_unmap_page(rx_ring->pdev,
2808 buffer_info->page_dma,
6ec43fe6
AD		 2809 PAGE_SIZE / 2,
2810 PCI_DMA_FROMDEVICE);
2811 buffer_info->page_dma = 0;
2812 }
9d5c8243 2813 if (buffer_info->page) {
9d5c8243
AK		 2814 put_page(buffer_info->page);
2815 buffer_info->page = NULL;
bf36c1a0 2816 buffer_info->page_offset = 0;
9d5c8243
AK		 2817 }
2818 }
2819
9d5c8243
AK		 2820 size = sizeof(struct igb_buffer) * rx_ring->count;
2821 memset(rx_ring->buffer_info, 0, size);
2822
2823 /* Zero out the descriptor ring */
2824 memset(rx_ring->desc, 0, rx_ring->size);
2825
2826 rx_ring->next_to_clean = 0;
2827 rx_ring->next_to_use = 0;
9d5c8243
AK		 2828}
2829
2830/**
2831 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2832 * @adapter: board private structure
2833 **/
2834static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2835{
2836 int i;
2837
2838 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2839 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 2840}
2841
2842/**
2843 * igb_set_mac - Change the Ethernet Address of the NIC
2844 * @netdev: network interface device structure
2845 * @p: pointer to an address structure
2846 *
2847 * Returns 0 on success, negative on failure
2848 **/
2849static int igb_set_mac(struct net_device *netdev, void *p)
2850{
2851 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2852 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2853 struct sockaddr *addr = p;
2854
2855 if (!is_valid_ether_addr(addr->sa_data))
2856 return -EADDRNOTAVAIL;
2857
2858 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2859 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2860
26ad9178
AD		 2861 /* set the correct pool for the new PF MAC address in entry 0 */
2862 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2863 adapter->vfs_allocated_count);
e1739522 2864
9d5c8243
AK		 2865 return 0;
2866}
2867
2868/**
68d480c4 2869 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2870 * @netdev: network interface device structure
2871 *
68d480c4
AD		 2872 * Writes multicast address list to the MTA hash table.
2873 * Returns: -ENOMEM on failure
2874 * 0 on no addresses written
2875 * X on writing X addresses to MTA
9d5c8243 2876 **/
68d480c4 2877static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2878{
2879 struct igb_adapter *adapter = netdev_priv(netdev);
2880 struct e1000_hw *hw = &adapter->hw;
48e2f183 2881 struct dev_mc_list *mc_ptr;
68d480c4 2882 u8 *mta_list;
9d5c8243
AK		 2883 int i;
2884
4cd24eaf 2885 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 2886 /* nothing to program, so clear mc list */
2887 igb_update_mc_addr_list(hw, NULL, 0);
2888 igb_restore_vf_multicasts(adapter);
2889 return 0;
2890 }
9d5c8243 2891
4cd24eaf 2892 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 2893 if (!mta_list)
2894 return -ENOMEM;
ff41f8dc 2895
68d480c4 2896 /* The shared function expects a packed array of only addresses. */
48e2f183
JP		 2897 i = 0;
2898 netdev_for_each_mc_addr(mc_ptr, netdev)
2899 memcpy(mta_list + (i++ * ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
68d480c4 2900
68d480c4
AD		 2901 igb_update_mc_addr_list(hw, mta_list, i);
2902 kfree(mta_list);
2903
4cd24eaf 2904 return netdev_mc_count(netdev);
68d480c4
AD		 2905}
2906
2907/**
2908 * igb_write_uc_addr_list - write unicast addresses to RAR table
2909 * @netdev: network interface device structure
2910 *
2911 * Writes unicast address list to the RAR table.
2912 * Returns: -ENOMEM on failure/insufficient address space
2913 * 0 on no addresses written
2914 * X on writing X addresses to the RAR table
2915 **/
2916static int igb_write_uc_addr_list(struct net_device *netdev)
2917{
2918 struct igb_adapter *adapter = netdev_priv(netdev);
2919 struct e1000_hw *hw = &adapter->hw;
2920 unsigned int vfn = adapter->vfs_allocated_count;
2921 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2922 int count = 0;
2923
2924 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 2925 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 2926 return -ENOMEM;
9d5c8243 2927
32e7bfc4 2928 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 2929 struct netdev_hw_addr *ha;
32e7bfc4
JP		 2930
2931 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 2932 if (!rar_entries)
2933 break;
26ad9178
AD		 2934 igb_rar_set_qsel(adapter, ha->addr,
2935 rar_entries--,
68d480c4
AD		 2936 vfn);
2937 count++;
ff41f8dc
AD		 2938 }
2939 }
2940 /* write the addresses in reverse order to avoid write combining */
2941 for (; rar_entries > 0 ; rar_entries--) {
2942 wr32(E1000_RAH(rar_entries), 0);
2943 wr32(E1000_RAL(rar_entries), 0);
2944 }
2945 wrfl();
2946
68d480c4
AD		 2947 return count;
2948}
2949
2950/**
2951 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2952 * @netdev: network interface device structure
2953 *
2954 * The set_rx_mode entry point is called whenever the unicast or multicast
2955 * address lists or the network interface flags are updated. This routine is
2956 * responsible for configuring the hardware for proper unicast, multicast,
2957 * promiscuous mode, and all-multi behavior.
2958 **/
2959static void igb_set_rx_mode(struct net_device *netdev)
2960{
2961 struct igb_adapter *adapter = netdev_priv(netdev);
2962 struct e1000_hw *hw = &adapter->hw;
2963 unsigned int vfn = adapter->vfs_allocated_count;
2964 u32 rctl, vmolr = 0;
2965 int count;
2966
2967 /* Check for Promiscuous and All Multicast modes */
2968 rctl = rd32(E1000_RCTL);
2969
2970 /* clear the effected bits */
2971 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2972
2973 if (netdev->flags & IFF_PROMISC) {
2974 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2975 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2976 } else {
2977 if (netdev->flags & IFF_ALLMULTI) {
2978 rctl |= E1000_RCTL_MPE;
2979 vmolr |= E1000_VMOLR_MPME;
2980 } else {
2981 /*
2982 * Write addresses to the MTA, if the attempt fails
2983 * then we should just turn on promiscous mode so
2984 * that we can at least receive multicast traffic
2985 */
2986 count = igb_write_mc_addr_list(netdev);
2987 if (count < 0) {
2988 rctl |= E1000_RCTL_MPE;
2989 vmolr |= E1000_VMOLR_MPME;
2990 } else if (count) {
2991 vmolr |= E1000_VMOLR_ROMPE;
2992 }
2993 }
2994 /*
2995 * Write addresses to available RAR registers, if there is not
2996 * sufficient space to store all the addresses then enable
2997 * unicast promiscous mode
2998 */
2999 count = igb_write_uc_addr_list(netdev);
3000 if (count < 0) {
3001 rctl |= E1000_RCTL_UPE;
3002 vmolr |= E1000_VMOLR_ROPE;
3003 }
3004 rctl |= E1000_RCTL_VFE;
28fc06f5 3005 }
68d480c4 3006 wr32(E1000_RCTL, rctl);
28fc06f5 3007
68d480c4
AD		 3008 /*
3009 * In order to support SR-IOV and eventually VMDq it is necessary to set
3010 * the VMOLR to enable the appropriate modes. Without this workaround
3011 * we will have issues with VLAN tag stripping not being done for frames
3012 * that are only arriving because we are the default pool
3013 */
3014 if (hw->mac.type < e1000_82576)
28fc06f5 3015 return;
9d5c8243 3016
68d480c4
AD		 3017 vmolr |= rd32(E1000_VMOLR(vfn)) &
3018 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3019 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3020 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3021}
3022
3023/* Need to wait a few seconds after link up to get diagnostic information from
3024 * the phy */
3025static void igb_update_phy_info(unsigned long data)
3026{
3027 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3028 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3029}
3030
4d6b725e
AD		 3031/**
3032 * igb_has_link - check shared code for link and determine up/down
3033 * @adapter: pointer to driver private info
3034 **/
3145535a 3035bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3036{
3037 struct e1000_hw *hw = &adapter->hw;
3038 bool link_active = false;
3039 s32 ret_val = 0;
3040
3041 /* get_link_status is set on LSC (link status) interrupt or
3042 * rx sequence error interrupt. get_link_status will stay
3043 * false until the e1000_check_for_link establishes link
3044 * for copper adapters ONLY
3045 */
3046 switch (hw->phy.media_type) {
3047 case e1000_media_type_copper:
3048 if (hw->mac.get_link_status) {
3049 ret_val = hw->mac.ops.check_for_link(hw);
3050 link_active = !hw->mac.get_link_status;
3051 } else {
3052 link_active = true;
3053 }
3054 break;
4d6b725e
AD		 3055 case e1000_media_type_internal_serdes:
3056 ret_val = hw->mac.ops.check_for_link(hw);
3057 link_active = hw->mac.serdes_has_link;
3058 break;
3059 default:
3060 case e1000_media_type_unknown:
3061 break;
3062 }
3063
3064 return link_active;
3065}
3066
9d5c8243
AK		 3067/**
3068 * igb_watchdog - Timer Call-back
3069 * @data: pointer to adapter cast into an unsigned long
3070 **/
3071static void igb_watchdog(unsigned long data)
3072{
3073 struct igb_adapter *adapter = (struct igb_adapter *)data;
3074 /* Do the rest outside of interrupt context */
3075 schedule_work(&adapter->watchdog_task);
3076}
3077
3078static void igb_watchdog_task(struct work_struct *work)
3079{
3080 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3081 struct igb_adapter,
3082 watchdog_task);
9d5c8243 3083 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3084 struct net_device *netdev = adapter->netdev;
9d5c8243 3085 u32 link;
7a6ea550 3086 int i;
9d5c8243 3087
4d6b725e 3088 link = igb_has_link(adapter);
9d5c8243
AK		 3089 if (link) {
3090 if (!netif_carrier_ok(netdev)) {
3091 u32 ctrl;
330a6d6a
AD		 3092 hw->mac.ops.get_speed_and_duplex(hw,
3093 &adapter->link_speed,
3094 &adapter->link_duplex);
9d5c8243
AK		 3095
3096 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3097 /* Links status message must follow this format */
3098 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3099 "Flow Control: %s\n",
559e9c49
AD		 3100 netdev->name,
3101 adapter->link_speed,
3102 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3103 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3104 ((ctrl & E1000_CTRL_TFCE) &&
3105 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3106 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3107 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243
AK		 3108
3109 /* tweak tx_queue_len according to speed/duplex and
3110 * adjust the timeout factor */
3111 netdev->tx_queue_len = adapter->tx_queue_len;
3112 adapter->tx_timeout_factor = 1;
3113 switch (adapter->link_speed) {
3114 case SPEED_10:
3115 netdev->tx_queue_len = 10;
3116 adapter->tx_timeout_factor = 14;
3117 break;
3118 case SPEED_100:
3119 netdev->tx_queue_len = 100;
3120 /* maybe add some timeout factor ? */
3121 break;
3122 }
3123
3124 netif_carrier_on(netdev);
9d5c8243 3125
4ae196df
AD		 3126 igb_ping_all_vfs(adapter);
3127
4b1a9877 3128 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3129 if (!test_bit(__IGB_DOWN, &adapter->state))
3130 mod_timer(&adapter->phy_info_timer,
3131 round_jiffies(jiffies + 2 * HZ));
3132 }
3133 } else {
3134 if (netif_carrier_ok(netdev)) {
3135 adapter->link_speed = 0;
3136 adapter->link_duplex = 0;
527d47c1
AD		 3137 /* Links status message must follow this format */
3138 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3139 netdev->name);
9d5c8243 3140 netif_carrier_off(netdev);
4b1a9877 3141
4ae196df
AD		 3142 igb_ping_all_vfs(adapter);
3143
4b1a9877 3144 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3145 if (!test_bit(__IGB_DOWN, &adapter->state))
3146 mod_timer(&adapter->phy_info_timer,
3147 round_jiffies(jiffies + 2 * HZ));
3148 }
3149 }
3150
9d5c8243 3151 igb_update_stats(adapter);
9d5c8243 3152
dbabb065 3153 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3154 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3155 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3156 /* We've lost link, so the controller stops DMA,
3157 * but we've got queued Tx work that's never going
3158 * to get done, so reset controller to flush Tx.
3159 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3160 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3161 adapter->tx_timeout_count++;
3162 schedule_work(&adapter->reset_task);
3163 /* return immediately since reset is imminent */
3164 return;
3165 }
9d5c8243 3166 }
9d5c8243 3167
dbabb065
AD		 3168 /* Force detection of hung controller every watchdog period */
3169 tx_ring->detect_tx_hung = true;
3170 }
f7ba205e 3171
9d5c8243 3172 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3173 if (adapter->msix_entries) {
047e0030
AD		 3174 u32 eics = 0;
3175 for (i = 0; i < adapter->num_q_vectors; i++) {
3176 struct igb_q_vector *q_vector = adapter->q_vector[i];
3177 eics |= q_vector->eims_value;
3178 }
7a6ea550
AD		 3179 wr32(E1000_EICS, eics);
3180 } else {
3181 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3182 }
9d5c8243 3183
9d5c8243
AK		 3184 /* Reset the timer */
3185 if (!test_bit(__IGB_DOWN, &adapter->state))
3186 mod_timer(&adapter->watchdog_timer,
3187 round_jiffies(jiffies + 2 * HZ));
3188}
3189
3190enum latency_range {
3191 lowest_latency = 0,
3192 low_latency = 1,
3193 bulk_latency = 2,
3194 latency_invalid = 255
3195};
3196
6eb5a7f1
AD		 3197/**
3198 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3199 *
3200 * Stores a new ITR value based on strictly on packet size. This
3201 * algorithm is less sophisticated than that used in igb_update_itr,
3202 * due to the difficulty of synchronizing statistics across multiple
3203 * receive rings. The divisors and thresholds used by this fuction
3204 * were determined based on theoretical maximum wire speed and testing
3205 * data, in order to minimize response time while increasing bulk
3206 * throughput.
3207 * This functionality is controlled by the InterruptThrottleRate module
3208 * parameter (see igb_param.c)
3209 * NOTE: This function is called only when operating in a multiqueue
3210 * receive environment.
047e0030 3211 * @q_vector: pointer to q_vector
6eb5a7f1 3212 **/
047e0030 3213static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3214{
047e0030 3215 int new_val = q_vector->itr_val;
6eb5a7f1 3216 int avg_wire_size = 0;
047e0030 3217 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3218
6eb5a7f1
AD		 3219 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3220 * ints/sec - ITR timer value of 120 ticks.
3221 */
3222 if (adapter->link_speed != SPEED_1000) {
047e0030 3223 new_val = 976;
6eb5a7f1 3224 goto set_itr_val;
9d5c8243 3225 }
047e0030
AD		 3226
3227 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3228 struct igb_ring *ring = q_vector->rx_ring;
3229 avg_wire_size = ring->total_bytes / ring->total_packets;
3230 }
3231
3232 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3233 struct igb_ring *ring = q_vector->tx_ring;
3234 avg_wire_size = max_t(u32, avg_wire_size,
3235 (ring->total_bytes /
3236 ring->total_packets));
3237 }
3238
3239 /* if avg_wire_size isn't set no work was done */
3240 if (!avg_wire_size)
3241 goto clear_counts;
9d5c8243 3242
6eb5a7f1
AD		 3243 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3244 avg_wire_size += 24;
3245
3246 /* Don't starve jumbo frames */
3247 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3248
6eb5a7f1
AD		 3249 /* Give a little boost to mid-size frames */
3250 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3251 new_val = avg_wire_size / 3;
3252 else
3253 new_val = avg_wire_size / 2;
9d5c8243 3254
abe1c363
NN		 3255 /* when in itr mode 3 do not exceed 20K ints/sec */
3256 if (adapter->rx_itr_setting == 3 && new_val < 196)
3257 new_val = 196;
3258
6eb5a7f1 3259set_itr_val:
047e0030
AD		 3260 if (new_val != q_vector->itr_val) {
3261 q_vector->itr_val = new_val;
3262 q_vector->set_itr = 1;
9d5c8243 3263 }
6eb5a7f1 3264clear_counts:
047e0030
AD		 3265 if (q_vector->rx_ring) {
3266 q_vector->rx_ring->total_bytes = 0;
3267 q_vector->rx_ring->total_packets = 0;
3268 }
3269 if (q_vector->tx_ring) {
3270 q_vector->tx_ring->total_bytes = 0;
3271 q_vector->tx_ring->total_packets = 0;
3272 }
9d5c8243
AK		 3273}
3274
3275/**
3276 * igb_update_itr - update the dynamic ITR value based on statistics
3277 * Stores a new ITR value based on packets and byte
3278 * counts during the last interrupt. The advantage of per interrupt
3279 * computation is faster updates and more accurate ITR for the current
3280 * traffic pattern. Constants in this function were computed
3281 * based on theoretical maximum wire speed and thresholds were set based
3282 * on testing data as well as attempting to minimize response time
3283 * while increasing bulk throughput.
3284 * this functionality is controlled by the InterruptThrottleRate module
3285 * parameter (see igb_param.c)
3286 * NOTE: These calculations are only valid when operating in a single-
3287 * queue environment.
3288 * @adapter: pointer to adapter
047e0030 3289 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3290 * @packets: the number of packets during this measurement interval
3291 * @bytes: the number of bytes during this measurement interval
3292 **/
3293static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3294 int packets, int bytes)
3295{
3296 unsigned int retval = itr_setting;
3297
3298 if (packets == 0)
3299 goto update_itr_done;
3300
3301 switch (itr_setting) {
3302 case lowest_latency:
3303 /* handle TSO and jumbo frames */
3304 if (bytes/packets > 8000)
3305 retval = bulk_latency;
3306 else if ((packets < 5) && (bytes > 512))
3307 retval = low_latency;
3308 break;
3309 case low_latency: /* 50 usec aka 20000 ints/s */
3310 if (bytes > 10000) {
3311 /* this if handles the TSO accounting */
3312 if (bytes/packets > 8000) {
3313 retval = bulk_latency;
3314 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3315 retval = bulk_latency;
3316 } else if ((packets > 35)) {
3317 retval = lowest_latency;
3318 }
3319 } else if (bytes/packets > 2000) {
3320 retval = bulk_latency;
3321 } else if (packets <= 2 && bytes < 512) {
3322 retval = lowest_latency;
3323 }
3324 break;
3325 case bulk_latency: /* 250 usec aka 4000 ints/s */
3326 if (bytes > 25000) {
3327 if (packets > 35)
3328 retval = low_latency;
1e5c3d21 3329 } else if (bytes < 1500) {
9d5c8243
AK		 3330 retval = low_latency;
3331 }
3332 break;
3333 }
3334
3335update_itr_done:
3336 return retval;
3337}
3338
6eb5a7f1 3339static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3340{
047e0030 3341 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3342 u16 current_itr;
047e0030 3343 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3344
3345 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3346 if (adapter->link_speed != SPEED_1000) {
3347 current_itr = 0;
3348 new_itr = 4000;
3349 goto set_itr_now;
3350 }
3351
3352 adapter->rx_itr = igb_update_itr(adapter,
3353 adapter->rx_itr,
3025a446
AD		 3354 q_vector->rx_ring->total_packets,
3355 q_vector->rx_ring->total_bytes);
9d5c8243 3356
047e0030
AD		 3357 adapter->tx_itr = igb_update_itr(adapter,
3358 adapter->tx_itr,
3025a446
AD		 3359 q_vector->tx_ring->total_packets,
3360 q_vector->tx_ring->total_bytes);
047e0030 3361 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3362
6eb5a7f1 3363 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3364 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3365 current_itr = low_latency;
3366
9d5c8243
AK		 3367 switch (current_itr) {
3368 /* counts and packets in update_itr are dependent on these numbers */
3369 case lowest_latency:
78b1f607 3370 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3371 break;
3372 case low_latency:
78b1f607 3373 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3374 break;
3375 case bulk_latency:
78b1f607 3376 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3377 break;
3378 default:
3379 break;
3380 }
3381
3382set_itr_now:
3025a446
AD		 3383 q_vector->rx_ring->total_bytes = 0;
3384 q_vector->rx_ring->total_packets = 0;
3385 q_vector->tx_ring->total_bytes = 0;
3386 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3387
047e0030 3388 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3389 /* this attempts to bias the interrupt rate towards Bulk
3390 * by adding intermediate steps when interrupt rate is
3391 * increasing */
047e0030
AD		 3392 new_itr = new_itr > q_vector->itr_val ?
3393 max((new_itr * q_vector->itr_val) /
3394 (new_itr + (q_vector->itr_val >> 2)),
3395 new_itr) :
9d5c8243
AK		 3396 new_itr;
3397 /* Don't write the value here; it resets the adapter's
3398 * internal timer, and causes us to delay far longer than
3399 * we should between interrupts. Instead, we write the ITR
3400 * value at the beginning of the next interrupt so the timing
3401 * ends up being correct.
3402 */
047e0030
AD		 3403 q_vector->itr_val = new_itr;
3404 q_vector->set_itr = 1;
9d5c8243
AK		 3405 }
3406
3407 return;
3408}
3409
9d5c8243
AK		 3410#define IGB_TX_FLAGS_CSUM 0x00000001
3411#define IGB_TX_FLAGS_VLAN 0x00000002
3412#define IGB_TX_FLAGS_TSO 0x00000004
3413#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3414#define IGB_TX_FLAGS_TSTAMP 0x00000010
3415#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3416#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3417
85ad76b2 3418static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3419 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3420{
3421 struct e1000_adv_tx_context_desc *context_desc;
3422 unsigned int i;
3423 int err;
3424 struct igb_buffer *buffer_info;
3425 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3426 u32 mss_l4len_idx;
3427 u8 l4len;
9d5c8243
AK		 3428
3429 if (skb_header_cloned(skb)) {
3430 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3431 if (err)
3432 return err;
3433 }
3434
3435 l4len = tcp_hdrlen(skb);
3436 *hdr_len += l4len;
3437
3438 if (skb->protocol == htons(ETH_P_IP)) {
3439 struct iphdr *iph = ip_hdr(skb);
3440 iph->tot_len = 0;
3441 iph->check = 0;
3442 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3443 iph->daddr, 0,
3444 IPPROTO_TCP,
3445 0);
8e1e8a47 3446 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3447 ipv6_hdr(skb)->payload_len = 0;
3448 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3449 &ipv6_hdr(skb)->daddr,
3450 0, IPPROTO_TCP, 0);
3451 }
3452
3453 i = tx_ring->next_to_use;
3454
3455 buffer_info = &tx_ring->buffer_info[i];
3456 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3457 /* VLAN MACLEN IPLEN */
3458 if (tx_flags & IGB_TX_FLAGS_VLAN)
3459 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3460 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3461 *hdr_len += skb_network_offset(skb);
3462 info |= skb_network_header_len(skb);
3463 *hdr_len += skb_network_header_len(skb);
3464 context_desc->vlan_macip_lens = cpu_to_le32(info);
3465
3466 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3467 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3468
3469 if (skb->protocol == htons(ETH_P_IP))
3470 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3471 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3472
3473 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3474
3475 /* MSS L4LEN IDX */
3476 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3477 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3478
73cd78f1 3479 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3480 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3481 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3482
3483 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3484 context_desc->seqnum_seed = 0;
3485
3486 buffer_info->time_stamp = jiffies;
0e014cb1 3487 buffer_info->next_to_watch = i;
9d5c8243
AK		 3488 buffer_info->dma = 0;
3489 i++;
3490 if (i == tx_ring->count)
3491 i = 0;
3492
3493 tx_ring->next_to_use = i;
3494
3495 return true;
3496}
3497
85ad76b2
AD		 3498static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3499 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3500{
3501 struct e1000_adv_tx_context_desc *context_desc;
80785298 3502 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3503 struct igb_buffer *buffer_info;
3504 u32 info = 0, tu_cmd = 0;
80785298 3505 unsigned int i;
9d5c8243
AK		 3506
3507 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3508 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3509 i = tx_ring->next_to_use;
3510 buffer_info = &tx_ring->buffer_info[i];
3511 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3512
3513 if (tx_flags & IGB_TX_FLAGS_VLAN)
3514 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3515
9d5c8243
AK		 3516 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3517 if (skb->ip_summed == CHECKSUM_PARTIAL)
3518 info |= skb_network_header_len(skb);
3519
3520 context_desc->vlan_macip_lens = cpu_to_le32(info);
3521
3522 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3523
3524 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3525 __be16 protocol;
3526
3527 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3528 const struct vlan_ethhdr *vhdr =
3529 (const struct vlan_ethhdr*)skb->data;
3530
3531 protocol = vhdr->h_vlan_encapsulated_proto;
3532 } else {
3533 protocol = skb->protocol;
3534 }
3535
3536 switch (protocol) {
09640e63 3537 case cpu_to_be16(ETH_P_IP):
9d5c8243 3538 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3539 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3540 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3541 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3542 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3543 break;
09640e63 3544 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3545 /* XXX what about other V6 headers?? */
3546 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3547 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3548 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3549 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3550 break;
3551 default:
3552 if (unlikely(net_ratelimit()))
80785298 3553 dev_warn(&pdev->dev,
44b0cda3
MW		 3554 "partial checksum but proto=%x!\n",
3555 skb->protocol);
3556 break;
3557 }
9d5c8243
AK		 3558 }
3559
3560 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3561 context_desc->seqnum_seed = 0;
85ad76b2 3562 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3563 context_desc->mss_l4len_idx =
85ad76b2 3564 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3565
3566 buffer_info->time_stamp = jiffies;
0e014cb1 3567 buffer_info->next_to_watch = i;
9d5c8243
AK		 3568 buffer_info->dma = 0;
3569
3570 i++;
3571 if (i == tx_ring->count)
3572 i = 0;
3573 tx_ring->next_to_use = i;
3574
3575 return true;
3576 }
9d5c8243
AK		 3577 return false;
3578}
3579
3580#define IGB_MAX_TXD_PWR 16
3581#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3582
80785298 3583static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3584 unsigned int first)
9d5c8243
AK		 3585{
3586 struct igb_buffer *buffer_info;
80785298 3587 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3588 unsigned int len = skb_headlen(skb);
3589 unsigned int count = 0, i;
3590 unsigned int f;
3591
3592 i = tx_ring->next_to_use;
3593
3594 buffer_info = &tx_ring->buffer_info[i];
3595 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3596 buffer_info->length = len;
3597 /* set time_stamp *before* dma to help avoid a possible race */
3598 buffer_info->time_stamp = jiffies;
0e014cb1 3599 buffer_info->next_to_watch = i;
6366ad33
AD		 3600 buffer_info->dma = pci_map_single(pdev, skb->data, len,
3601 PCI_DMA_TODEVICE);
3602 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3603 goto dma_error;
9d5c8243
AK		 3604
3605 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3606 struct skb_frag_struct *frag;
3607
8581145f 3608 count++;
65689fef
AD		 3609 i++;
3610 if (i == tx_ring->count)
3611 i = 0;
3612
9d5c8243
AK		 3613 frag = &skb_shinfo(skb)->frags[f];
3614 len = frag->size;
3615
3616 buffer_info = &tx_ring->buffer_info[i];
3617 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3618 buffer_info->length = len;
3619 buffer_info->time_stamp = jiffies;
0e014cb1 3620 buffer_info->next_to_watch = i;
6366ad33
AD		 3621 buffer_info->mapped_as_page = true;
3622 buffer_info->dma = pci_map_page(pdev,
3623 frag->page,
3624 frag->page_offset,
3625 len,
3626 PCI_DMA_TODEVICE);
3627 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3628 goto dma_error;
3629
9d5c8243
AK		 3630 }
3631
9d5c8243 3632 tx_ring->buffer_info[i].skb = skb;
40e90c26 3633 tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
0e014cb1 3634 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3635
cdfd01fc 3636 return ++count;
6366ad33
AD		 3637
3638dma_error:
3639 dev_err(&pdev->dev, "TX DMA map failed\n");
3640
3641 /* clear timestamp and dma mappings for failed buffer_info mapping */
3642 buffer_info->dma = 0;
3643 buffer_info->time_stamp = 0;
3644 buffer_info->length = 0;
3645 buffer_info->next_to_watch = 0;
3646 buffer_info->mapped_as_page = false;
6366ad33
AD		 3647
3648 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3649 while (count--) {
3650 if (i == 0)
3651 i = tx_ring->count;
6366ad33 3652 i--;
6366ad33
AD		 3653 buffer_info = &tx_ring->buffer_info[i];
3654 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3655 }
3656
3657 return 0;
9d5c8243
AK		 3658}
3659
85ad76b2 3660static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3661 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3662 u8 hdr_len)
3663{
cdfd01fc 3664 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3665 struct igb_buffer *buffer_info;
3666 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3667 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3668
3669 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3670 E1000_ADVTXD_DCMD_DEXT);
3671
3672 if (tx_flags & IGB_TX_FLAGS_VLAN)
3673 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3674
33af6bcc
PO		 3675 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3676 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3677
9d5c8243
AK		 3678 if (tx_flags & IGB_TX_FLAGS_TSO) {
3679 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3680
3681 /* insert tcp checksum */
3682 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3683
3684 /* insert ip checksum */
3685 if (tx_flags & IGB_TX_FLAGS_IPV4)
3686 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3687
3688 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3689 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3690 }
3691
85ad76b2
AD		 3692 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3693 (tx_flags & (IGB_TX_FLAGS_CSUM |
3694 IGB_TX_FLAGS_TSO |
7dfc16fa 3695 IGB_TX_FLAGS_VLAN)))
85ad76b2 3696 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3697
3698 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3699
cdfd01fc 3700 do {
9d5c8243
AK		 3701 buffer_info = &tx_ring->buffer_info[i];
3702 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3703 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3704 tx_desc->read.cmd_type_len =
3705 cpu_to_le32(cmd_type_len | buffer_info->length);
3706 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3707 count--;
9d5c8243
AK		 3708 i++;
3709 if (i == tx_ring->count)
3710 i = 0;
cdfd01fc 3711 } while (count > 0);
9d5c8243 3712
85ad76b2 3713 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3714 /* Force memory writes to complete before letting h/w
3715 * know there are new descriptors to fetch. (Only
3716 * applicable for weak-ordered memory model archs,
3717 * such as IA-64). */
3718 wmb();
3719
3720 tx_ring->next_to_use = i;
fce99e34 3721 writel(i, tx_ring->tail);
9d5c8243
AK		 3722 /* we need this if more than one processor can write to our tail
3723 * at a time, it syncronizes IO on IA64/Altix systems */
3724 mmiowb();
3725}
3726
e694e964 3727static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3728{
e694e964
AD		 3729 struct net_device *netdev = tx_ring->netdev;
3730
661086df 3731 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3732
9d5c8243
AK		 3733 /* Herbert's original patch had:
3734 * smp_mb__after_netif_stop_queue();
3735 * but since that doesn't exist yet, just open code it. */
3736 smp_mb();
3737
3738 /* We need to check again in a case another CPU has just
3739 * made room available. */
c493ea45 3740 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3741 return -EBUSY;
3742
3743 /* A reprieve! */
661086df 3744 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3745 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3746 return 0;
3747}
3748
717ba089 3749static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3750{
c493ea45 3751 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3752 return 0;
e694e964 3753 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3754}
3755
b1a436c3
AD		 3756netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3757 struct igb_ring *tx_ring)
9d5c8243 3758{
e694e964 3759 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 3760 int tso = 0, count;
91d4ee33
NN		 3761 u32 tx_flags = 0;
3762 u16 first;
3763 u8 hdr_len = 0;
c5b9bd5e 3764 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3765
9d5c8243
AK		 3766 /* need: 1 descriptor per page,
3767 * + 2 desc gap to keep tail from touching head,
3768 * + 1 desc for skb->data,
3769 * + 1 desc for context descriptor,
3770 * otherwise try next time */
e694e964 3771 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3772 /* this is a hard error */
9d5c8243
AK		 3773 return NETDEV_TX_BUSY;
3774 }
33af6bcc 3775
33af6bcc
PO		 3776 if (unlikely(shtx->hardware)) {
3777 shtx->in_progress = 1;
3778 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3779 }
9d5c8243 3780
cdfd01fc 3781 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3782 tx_flags |= IGB_TX_FLAGS_VLAN;
3783 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3784 }
3785
661086df
PWJ		 3786 if (skb->protocol == htons(ETH_P_IP))
3787 tx_flags |= IGB_TX_FLAGS_IPV4;
3788
0e014cb1 3789 first = tx_ring->next_to_use;
85ad76b2
AD		 3790 if (skb_is_gso(skb)) {
3791 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3792
85ad76b2
AD		 3793 if (tso < 0) {
3794 dev_kfree_skb_any(skb);
3795 return NETDEV_TX_OK;
3796 }
9d5c8243
AK		 3797 }
3798
3799 if (tso)
3800 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3801 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3802 (skb->ip_summed == CHECKSUM_PARTIAL))
3803 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3804
65689fef 3805 /*
cdfd01fc 3806 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3807 * has occured and we need to rewind the descriptor queue
3808 */
80785298 3809 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 3810 if (!count) {
65689fef
AD		 3811 dev_kfree_skb_any(skb);
3812 tx_ring->buffer_info[first].time_stamp = 0;
3813 tx_ring->next_to_use = first;
85ad76b2 3814 return NETDEV_TX_OK;
65689fef 3815 }
9d5c8243 3816
85ad76b2
AD		 3817 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3818
3819 /* Make sure there is space in the ring for the next send. */
e694e964 3820 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3821
9d5c8243
AK		 3822 return NETDEV_TX_OK;
3823}
3824
3b29a56d
SH		 3825static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3826 struct net_device *netdev)
9d5c8243
AK		 3827{
3828 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3829 struct igb_ring *tx_ring;
661086df 3830 int r_idx = 0;
b1a436c3
AD		 3831
3832 if (test_bit(__IGB_DOWN, &adapter->state)) {
3833 dev_kfree_skb_any(skb);
3834 return NETDEV_TX_OK;
3835 }
3836
3837 if (skb->len <= 0) {
3838 dev_kfree_skb_any(skb);
3839 return NETDEV_TX_OK;
3840 }
3841
1bfaf07b 3842 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3843 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3844
3845 /* This goes back to the question of how to logically map a tx queue
3846 * to a flow. Right now, performance is impacted slightly negatively
3847 * if using multiple tx queues. If the stack breaks away from a
3848 * single qdisc implementation, we can look at this again. */
e694e964 3849 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3850}
3851
3852/**
3853 * igb_tx_timeout - Respond to a Tx Hang
3854 * @netdev: network interface device structure
3855 **/
3856static void igb_tx_timeout(struct net_device *netdev)
3857{
3858 struct igb_adapter *adapter = netdev_priv(netdev);
3859 struct e1000_hw *hw = &adapter->hw;
3860
3861 /* Do the reset outside of interrupt context */
3862 adapter->tx_timeout_count++;
f7ba205e 3863
55cac248
AD		 3864 if (hw->mac.type == e1000_82580)
3865 hw->dev_spec._82575.global_device_reset = true;
3866
9d5c8243 3867 schedule_work(&adapter->reset_task);
265de409
AD		 3868 wr32(E1000_EICS,
3869 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3870}
3871
3872static void igb_reset_task(struct work_struct *work)
3873{
3874 struct igb_adapter *adapter;
3875 adapter = container_of(work, struct igb_adapter, reset_task);
3876
3877 igb_reinit_locked(adapter);
3878}
3879
3880/**
3881 * igb_get_stats - Get System Network Statistics
3882 * @netdev: network interface device structure
3883 *
3884 * Returns the address of the device statistics structure.
3885 * The statistics are actually updated from the timer callback.
3886 **/
73cd78f1 3887static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3888{
9d5c8243 3889 /* only return the current stats */
8d24e933 3890 return &netdev->stats;
9d5c8243
AK		 3891}
3892
3893/**
3894 * igb_change_mtu - Change the Maximum Transfer Unit
3895 * @netdev: network interface device structure
3896 * @new_mtu: new value for maximum frame size
3897 *
3898 * Returns 0 on success, negative on failure
3899 **/
3900static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3901{
3902 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3903 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3904 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3905 u32 rx_buffer_len, i;
9d5c8243 3906
c809d227 3907 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3908 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3909 return -EINVAL;
3910 }
3911
9d5c8243 3912 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3913 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3914 return -EINVAL;
3915 }
3916
3917 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3918 msleep(1);
73cd78f1 3919
9d5c8243
AK		 3920 /* igb_down has a dependency on max_frame_size */
3921 adapter->max_frame_size = max_frame;
559e9c49 3922
9d5c8243
AK		 3923 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3924 * means we reserve 2 more, this pushes us to allocate from the next
3925 * larger slab size.
3926 * i.e. RXBUFFER_2048 --> size-4096 slab
3927 */
3928
7d95b717 3929 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3930 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3931 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3932 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3933 else
4c844851
AD		 3934 rx_buffer_len = IGB_RXBUFFER_128;
3935
3936 if (netif_running(netdev))
3937 igb_down(adapter);
9d5c8243 3938
090b1795 3939 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3940 netdev->mtu, new_mtu);
3941 netdev->mtu = new_mtu;
3942
4c844851 3943 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3944 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 3945
9d5c8243
AK		 3946 if (netif_running(netdev))
3947 igb_up(adapter);
3948 else
3949 igb_reset(adapter);
3950
3951 clear_bit(__IGB_RESETTING, &adapter->state);
3952
3953 return 0;
3954}
3955
3956/**
3957 * igb_update_stats - Update the board statistics counters
3958 * @adapter: board private structure
3959 **/
3960
3961void igb_update_stats(struct igb_adapter *adapter)
3962{
128e45eb 3963 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 3964 struct e1000_hw *hw = &adapter->hw;
3965 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 3966 u32 reg, mpc;
9d5c8243 3967 u16 phy_tmp;
3f9c0164
AD		 3968 int i;
3969 u64 bytes, packets;
9d5c8243
AK		 3970
3971#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3972
3973 /*
3974 * Prevent stats update while adapter is being reset, or if the pci
3975 * connection is down.
3976 */
3977 if (adapter->link_speed == 0)
3978 return;
3979 if (pci_channel_offline(pdev))
3980 return;
3981
3f9c0164
AD		 3982 bytes = 0;
3983 packets = 0;
3984 for (i = 0; i < adapter->num_rx_queues; i++) {
3985 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 3986 struct igb_ring *ring = adapter->rx_ring[i];
3987 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 3988 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 3989 bytes += ring->rx_stats.bytes;
3990 packets += ring->rx_stats.packets;
3f9c0164
AD		 3991 }
3992
128e45eb
AD		 3993 net_stats->rx_bytes = bytes;
3994 net_stats->rx_packets = packets;
3f9c0164
AD		 3995
3996 bytes = 0;
3997 packets = 0;
3998 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 3999 struct igb_ring *ring = adapter->tx_ring[i];
4000 bytes += ring->tx_stats.bytes;
4001 packets += ring->tx_stats.packets;
3f9c0164 4002 }
128e45eb
AD		 4003 net_stats->tx_bytes = bytes;
4004 net_stats->tx_packets = packets;
3f9c0164
AD		 4005
4006 /* read stats registers */
9d5c8243
AK		 4007 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4008 adapter->stats.gprc += rd32(E1000_GPRC);
4009 adapter->stats.gorc += rd32(E1000_GORCL);
4010 rd32(E1000_GORCH); /* clear GORCL */
4011 adapter->stats.bprc += rd32(E1000_BPRC);
4012 adapter->stats.mprc += rd32(E1000_MPRC);
4013 adapter->stats.roc += rd32(E1000_ROC);
4014
4015 adapter->stats.prc64 += rd32(E1000_PRC64);
4016 adapter->stats.prc127 += rd32(E1000_PRC127);
4017 adapter->stats.prc255 += rd32(E1000_PRC255);
4018 adapter->stats.prc511 += rd32(E1000_PRC511);
4019 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4020 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4021 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4022 adapter->stats.sec += rd32(E1000_SEC);
4023
fa3d9a6d
MW		 4024 mpc = rd32(E1000_MPC);
4025 adapter->stats.mpc += mpc;
4026 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4027 adapter->stats.scc += rd32(E1000_SCC);
4028 adapter->stats.ecol += rd32(E1000_ECOL);
4029 adapter->stats.mcc += rd32(E1000_MCC);
4030 adapter->stats.latecol += rd32(E1000_LATECOL);
4031 adapter->stats.dc += rd32(E1000_DC);
4032 adapter->stats.rlec += rd32(E1000_RLEC);
4033 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4034 adapter->stats.xontxc += rd32(E1000_XONTXC);
4035 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4036 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4037 adapter->stats.fcruc += rd32(E1000_FCRUC);
4038 adapter->stats.gptc += rd32(E1000_GPTC);
4039 adapter->stats.gotc += rd32(E1000_GOTCL);
4040 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4041 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4042 adapter->stats.ruc += rd32(E1000_RUC);
4043 adapter->stats.rfc += rd32(E1000_RFC);
4044 adapter->stats.rjc += rd32(E1000_RJC);
4045 adapter->stats.tor += rd32(E1000_TORH);
4046 adapter->stats.tot += rd32(E1000_TOTH);
4047 adapter->stats.tpr += rd32(E1000_TPR);
4048
4049 adapter->stats.ptc64 += rd32(E1000_PTC64);
4050 adapter->stats.ptc127 += rd32(E1000_PTC127);
4051 adapter->stats.ptc255 += rd32(E1000_PTC255);
4052 adapter->stats.ptc511 += rd32(E1000_PTC511);
4053 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4054 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4055
4056 adapter->stats.mptc += rd32(E1000_MPTC);
4057 adapter->stats.bptc += rd32(E1000_BPTC);
4058
2d0b0f69
NN		 4059 adapter->stats.tpt += rd32(E1000_TPT);
4060 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4061
4062 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4063 /* read internal phy specific stats */
4064 reg = rd32(E1000_CTRL_EXT);
4065 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4066 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4067 adapter->stats.tncrs += rd32(E1000_TNCRS);
4068 }
4069
9d5c8243
AK		 4070 adapter->stats.tsctc += rd32(E1000_TSCTC);
4071 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4072
4073 adapter->stats.iac += rd32(E1000_IAC);
4074 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4075 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4076 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4077 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4078 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4079 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4080 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4081 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4082
4083 /* Fill out the OS statistics structure */
128e45eb
AD		 4084 net_stats->multicast = adapter->stats.mprc;
4085 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4086
4087 /* Rx Errors */
4088
4089 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4090 * our own version based on RUC and ROC */
128e45eb 4091 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4092 adapter->stats.crcerrs + adapter->stats.algnerrc +
4093 adapter->stats.ruc + adapter->stats.roc +
4094 adapter->stats.cexterr;
128e45eb
AD		 4095 net_stats->rx_length_errors = adapter->stats.ruc +
4096 adapter->stats.roc;
4097 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4098 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4099 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4100
4101 /* Tx Errors */
128e45eb
AD		 4102 net_stats->tx_errors = adapter->stats.ecol +
4103 adapter->stats.latecol;
4104 net_stats->tx_aborted_errors = adapter->stats.ecol;
4105 net_stats->tx_window_errors = adapter->stats.latecol;
4106 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4107
4108 /* Tx Dropped needs to be maintained elsewhere */
4109
4110 /* Phy Stats */
4111 if (hw->phy.media_type == e1000_media_type_copper) {
4112 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4113 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4114 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4115 adapter->phy_stats.idle_errors += phy_tmp;
4116 }
4117 }
4118
4119 /* Management Stats */
4120 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4121 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4122 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4123}
4124
9d5c8243
AK		 4125static irqreturn_t igb_msix_other(int irq, void *data)
4126{
047e0030 4127 struct igb_adapter *adapter = data;
9d5c8243 4128 struct e1000_hw *hw = &adapter->hw;
844290e5 4129 u32 icr = rd32(E1000_ICR);
844290e5 4130 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4131
7f081d40
AD		 4132 if (icr & E1000_ICR_DRSTA)
4133 schedule_work(&adapter->reset_task);
4134
047e0030 4135 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4136 /* HW is reporting DMA is out of sync */
4137 adapter->stats.doosync++;
4138 }
eebbbdba 4139
4ae196df
AD		 4140 /* Check for a mailbox event */
4141 if (icr & E1000_ICR_VMMB)
4142 igb_msg_task(adapter);
4143
4144 if (icr & E1000_ICR_LSC) {
4145 hw->mac.get_link_status = 1;
4146 /* guard against interrupt when we're going down */
4147 if (!test_bit(__IGB_DOWN, &adapter->state))
4148 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4149 }
4150
25568a53
AD		 4151 if (adapter->vfs_allocated_count)
4152 wr32(E1000_IMS, E1000_IMS_LSC |
4153 E1000_IMS_VMMB |
4154 E1000_IMS_DOUTSYNC);
4155 else
4156 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4157 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4158
4159 return IRQ_HANDLED;
4160}
4161
047e0030 4162static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4163{
26b39276 4164 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4165 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4166
047e0030
AD		 4167 if (!q_vector->set_itr)
4168 return;
73cd78f1 4169
047e0030
AD		 4170 if (!itr_val)
4171 itr_val = 0x4;
661086df 4172
26b39276
AD		 4173 if (adapter->hw.mac.type == e1000_82575)
4174 itr_val |= itr_val << 16;
661086df 4175 else
047e0030 4176 itr_val |= 0x8000000;
661086df 4177
047e0030
AD		 4178 writel(itr_val, q_vector->itr_register);
4179 q_vector->set_itr = 0;
6eb5a7f1
AD		 4180}
4181
047e0030 4182static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4183{
047e0030 4184 struct igb_q_vector *q_vector = data;
9d5c8243 4185
047e0030
AD		 4186 /* Write the ITR value calculated from the previous interrupt. */
4187 igb_write_itr(q_vector);
9d5c8243 4188
047e0030 4189 napi_schedule(&q_vector->napi);
844290e5 4190
047e0030 4191 return IRQ_HANDLED;
fe4506b6
JC		 4192}
4193
421e02f0 4194#ifdef CONFIG_IGB_DCA
047e0030 4195static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4196{
047e0030 4197 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4198 struct e1000_hw *hw = &adapter->hw;
4199 int cpu = get_cpu();
fe4506b6 4200
047e0030
AD		 4201 if (q_vector->cpu == cpu)
4202 goto out_no_update;
4203
4204 if (q_vector->tx_ring) {
4205 int q = q_vector->tx_ring->reg_idx;
4206 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4207 if (hw->mac.type == e1000_82575) {
4208 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4209 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4210 } else {
047e0030
AD		 4211 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4212 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4213 E1000_DCA_TXCTRL_CPUID_SHIFT;
4214 }
4215 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4216 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4217 }
4218 if (q_vector->rx_ring) {
4219 int q = q_vector->rx_ring->reg_idx;
4220 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4221 if (hw->mac.type == e1000_82575) {
2d064c06 4222 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4223 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4224 } else {
4225 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4226 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4227 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4228 }
fe4506b6
JC		 4229 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4230 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4231 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4232 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4233 }
047e0030
AD		 4234 q_vector->cpu = cpu;
4235out_no_update:
fe4506b6
JC		 4236 put_cpu();
4237}
4238
4239static void igb_setup_dca(struct igb_adapter *adapter)
4240{
7e0e99ef 4241 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4242 int i;
4243
7dfc16fa 4244 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4245 return;
4246
7e0e99ef
AD		 4247 /* Always use CB2 mode, difference is masked in the CB driver. */
4248 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4249
047e0030 4250 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4251 adapter->q_vector[i]->cpu = -1;
4252 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4253 }
4254}
4255
4256static int __igb_notify_dca(struct device *dev, void *data)
4257{
4258 struct net_device *netdev = dev_get_drvdata(dev);
4259 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4260 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4261 struct e1000_hw *hw = &adapter->hw;
4262 unsigned long event = *(unsigned long *)data;
4263
4264 switch (event) {
4265 case DCA_PROVIDER_ADD:
4266 /* if already enabled, don't do it again */
7dfc16fa 4267 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4268 break;
fe4506b6 4269 if (dca_add_requester(dev) == 0) {
bbd98fe4 4270 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4271 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4272 igb_setup_dca(adapter);
4273 break;
4274 }
4275 /* Fall Through since DCA is disabled. */
4276 case DCA_PROVIDER_REMOVE:
7dfc16fa 4277 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4278 /* without this a class_device is left
047e0030 4279 * hanging around in the sysfs model */
fe4506b6 4280 dca_remove_requester(dev);
090b1795 4281 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4282 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4283 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4284 }
4285 break;
4286 }
bbd98fe4 4287
fe4506b6 4288 return 0;
9d5c8243
AK		 4289}
4290
fe4506b6
JC		 4291static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4292 void *p)
4293{
4294 int ret_val;
4295
4296 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4297 __igb_notify_dca);
4298
4299 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4300}
421e02f0 4301#endif /* CONFIG_IGB_DCA */
9d5c8243 4302
4ae196df
AD		 4303static void igb_ping_all_vfs(struct igb_adapter *adapter)
4304{
4305 struct e1000_hw *hw = &adapter->hw;
4306 u32 ping;
4307 int i;
4308
4309 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4310 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4311 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4312 ping |= E1000_VT_MSGTYPE_CTS;
4313 igb_write_mbx(hw, &ping, 1, i);
4314 }
4315}
4316
7d5753f0
AD		 4317static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4318{
4319 struct e1000_hw *hw = &adapter->hw;
4320 u32 vmolr = rd32(E1000_VMOLR(vf));
4321 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4322
4323 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4324 IGB_VF_FLAG_MULTI_PROMISC);
4325 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4326
4327 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4328 vmolr |= E1000_VMOLR_MPME;
4329 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4330 } else {
4331 /*
4332 * if we have hashes and we are clearing a multicast promisc
4333 * flag we need to write the hashes to the MTA as this step
4334 * was previously skipped
4335 */
4336 if (vf_data->num_vf_mc_hashes > 30) {
4337 vmolr |= E1000_VMOLR_MPME;
4338 } else if (vf_data->num_vf_mc_hashes) {
4339 int j;
4340 vmolr |= E1000_VMOLR_ROMPE;
4341 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4342 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4343 }
4344 }
4345
4346 wr32(E1000_VMOLR(vf), vmolr);
4347
4348 /* there are flags left unprocessed, likely not supported */
4349 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4350 return -EINVAL;
4351
4352 return 0;
4353
4354}
4355
4ae196df
AD		 4356static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4357 u32 *msgbuf, u32 vf)
4358{
4359 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4360 u16 *hash_list = (u16 *)&msgbuf[1];
4361 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4362 int i;
4363
7d5753f0 4364 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4365 * to this VF for later use to restore when the PF multi cast
4366 * list changes
4367 */
4368 vf_data->num_vf_mc_hashes = n;
4369
7d5753f0
AD		 4370 /* only up to 30 hash values supported */
4371 if (n > 30)
4372 n = 30;
4373
4374 /* store the hashes for later use */
4ae196df 4375 for (i = 0; i < n; i++)
a419aef8 4376 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4377
4378 /* Flush and reset the mta with the new values */
ff41f8dc 4379 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4380
4381 return 0;
4382}
4383
4384static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4385{
4386 struct e1000_hw *hw = &adapter->hw;
4387 struct vf_data_storage *vf_data;
4388 int i, j;
4389
4390 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4391 u32 vmolr = rd32(E1000_VMOLR(i));
4392 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4393
4ae196df 4394 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4395
4396 if ((vf_data->num_vf_mc_hashes > 30) ||
4397 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4398 vmolr |= E1000_VMOLR_MPME;
4399 } else if (vf_data->num_vf_mc_hashes) {
4400 vmolr |= E1000_VMOLR_ROMPE;
4401 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4402 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4403 }
4404 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4405 }
4406}
4407
4408static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4409{
4410 struct e1000_hw *hw = &adapter->hw;
4411 u32 pool_mask, reg, vid;
4412 int i;
4413
4414 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4415
4416 /* Find the vlan filter for this id */
4417 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4418 reg = rd32(E1000_VLVF(i));
4419
4420 /* remove the vf from the pool */
4421 reg &= ~pool_mask;
4422
4423 /* if pool is empty then remove entry from vfta */
4424 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4425 (reg & E1000_VLVF_VLANID_ENABLE)) {
4426 reg = 0;
4427 vid = reg & E1000_VLVF_VLANID_MASK;
4428 igb_vfta_set(hw, vid, false);
4429 }
4430
4431 wr32(E1000_VLVF(i), reg);
4432 }
ae641bdc
AD		 4433
4434 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4435}
4436
4437static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4438{
4439 struct e1000_hw *hw = &adapter->hw;
4440 u32 reg, i;
4441
51466239
AD		 4442 /* The vlvf table only exists on 82576 hardware and newer */
4443 if (hw->mac.type < e1000_82576)
4444 return -1;
4445
4446 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4447 if (!adapter->vfs_allocated_count)
4448 return -1;
4449
4450 /* Find the vlan filter for this id */
4451 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4452 reg = rd32(E1000_VLVF(i));
4453 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4454 vid == (reg & E1000_VLVF_VLANID_MASK))
4455 break;
4456 }
4457
4458 if (add) {
4459 if (i == E1000_VLVF_ARRAY_SIZE) {
4460 /* Did not find a matching VLAN ID entry that was
4461 * enabled. Search for a free filter entry, i.e.
4462 * one without the enable bit set
4463 */
4464 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4465 reg = rd32(E1000_VLVF(i));
4466 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4467 break;
4468 }
4469 }
4470 if (i < E1000_VLVF_ARRAY_SIZE) {
4471 /* Found an enabled/available entry */
4472 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4473
4474 /* if !enabled we need to set this up in vfta */
4475 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4476 /* add VID to filter table */
4477 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4478 reg |= E1000_VLVF_VLANID_ENABLE;
4479 }
cad6d05f
AD		 4480 reg &= ~E1000_VLVF_VLANID_MASK;
4481 reg |= vid;
4ae196df 4482 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4483
4484 /* do not modify RLPML for PF devices */
4485 if (vf >= adapter->vfs_allocated_count)
4486 return 0;
4487
4488 if (!adapter->vf_data[vf].vlans_enabled) {
4489 u32 size;
4490 reg = rd32(E1000_VMOLR(vf));
4491 size = reg & E1000_VMOLR_RLPML_MASK;
4492 size += 4;
4493 reg &= ~E1000_VMOLR_RLPML_MASK;
4494 reg |= size;
4495 wr32(E1000_VMOLR(vf), reg);
4496 }
ae641bdc 4497
51466239 4498 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4499 return 0;
4500 }
4501 } else {
4502 if (i < E1000_VLVF_ARRAY_SIZE) {
4503 /* remove vf from the pool */
4504 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4505 /* if pool is empty then remove entry from vfta */
4506 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4507 reg = 0;
4508 igb_vfta_set(hw, vid, false);
4509 }
4510 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4511
4512 /* do not modify RLPML for PF devices */
4513 if (vf >= adapter->vfs_allocated_count)
4514 return 0;
4515
4516 adapter->vf_data[vf].vlans_enabled--;
4517 if (!adapter->vf_data[vf].vlans_enabled) {
4518 u32 size;
4519 reg = rd32(E1000_VMOLR(vf));
4520 size = reg & E1000_VMOLR_RLPML_MASK;
4521 size -= 4;
4522 reg &= ~E1000_VMOLR_RLPML_MASK;
4523 reg |= size;
4524 wr32(E1000_VMOLR(vf), reg);
4525 }
4ae196df
AD		 4526 }
4527 }
8151d294
WM		 4528 return 0;
4529}
4530
4531static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4532{
4533 struct e1000_hw *hw = &adapter->hw;
4534
4535 if (vid)
4536 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4537 else
4538 wr32(E1000_VMVIR(vf), 0);
4539}
4540
4541static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4542 int vf, u16 vlan, u8 qos)
4543{
4544 int err = 0;
4545 struct igb_adapter *adapter = netdev_priv(netdev);
4546
4547 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4548 return -EINVAL;
4549 if (vlan || qos) {
4550 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4551 if (err)
4552 goto out;
4553 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4554 igb_set_vmolr(adapter, vf, !vlan);
4555 adapter->vf_data[vf].pf_vlan = vlan;
4556 adapter->vf_data[vf].pf_qos = qos;
4557 dev_info(&adapter->pdev->dev,
4558 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4559 if (test_bit(__IGB_DOWN, &adapter->state)) {
4560 dev_warn(&adapter->pdev->dev,
4561 "The VF VLAN has been set,"
4562 " but the PF device is not up.\n");
4563 dev_warn(&adapter->pdev->dev,
4564 "Bring the PF device up before"
4565 " attempting to use the VF device.\n");
4566 }
4567 } else {
4568 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4569 false, vf);
4570 igb_set_vmvir(adapter, vlan, vf);
4571 igb_set_vmolr(adapter, vf, true);
4572 adapter->vf_data[vf].pf_vlan = 0;
4573 adapter->vf_data[vf].pf_qos = 0;
4574 }
4575out:
4576 return err;
4ae196df
AD		 4577}
4578
4579static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4580{
4581 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4582 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4583
4584 return igb_vlvf_set(adapter, vid, add, vf);
4585}
4586
f2ca0dbe 4587static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4588{
8151d294
WM		 4589 /* clear flags */
4590 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4591 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4592
4593 /* reset offloads to defaults */
8151d294 4594 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4595
4596 /* reset vlans for device */
4597 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4598 if (adapter->vf_data[vf].pf_vlan)
4599 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4600 adapter->vf_data[vf].pf_vlan,
4601 adapter->vf_data[vf].pf_qos);
4602 else
4603 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4604
4605 /* reset multicast table array for vf */
4606 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4607
4608 /* Flush and reset the mta with the new values */
ff41f8dc 4609 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4610}
4611
f2ca0dbe
AD		 4612static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4613{
4614 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4615
4616 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4617 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4618 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4619
4620 /* process remaining reset events */
4621 igb_vf_reset(adapter, vf);
4622}
4623
4624static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4625{
4626 struct e1000_hw *hw = &adapter->hw;
4627 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4628 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4629 u32 reg, msgbuf[3];
4630 u8 *addr = (u8 *)(&msgbuf[1]);
4631
4632 /* process all the same items cleared in a function level reset */
f2ca0dbe 4633 igb_vf_reset(adapter, vf);
4ae196df
AD		 4634
4635 /* set vf mac address */
26ad9178 4636 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4637
4638 /* enable transmit and receive for vf */
4639 reg = rd32(E1000_VFTE);
4640 wr32(E1000_VFTE, reg | (1 << vf));
4641 reg = rd32(E1000_VFRE);
4642 wr32(E1000_VFRE, reg | (1 << vf));
4643
f2ca0dbe 4644 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4645
4646 /* reply to reset with ack and vf mac address */
4647 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4648 memcpy(addr, vf_mac, 6);
4649 igb_write_mbx(hw, msgbuf, 3, vf);
4650}
4651
4652static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4653{
f2ca0dbe
AD		 4654 unsigned char *addr = (char *)&msg[1];
4655 int err = -1;
4ae196df 4656
f2ca0dbe
AD		 4657 if (is_valid_ether_addr(addr))
4658 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4659
f2ca0dbe 4660 return err;
4ae196df
AD		 4661}
4662
4663static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4664{
4665 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4666 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4667 u32 msg = E1000_VT_MSGTYPE_NACK;
4668
4669 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4670 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4671 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4672 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4673 vf_data->last_nack = jiffies;
4ae196df
AD		 4674 }
4675}
4676
f2ca0dbe 4677static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4678{
f2ca0dbe
AD		 4679 struct pci_dev *pdev = adapter->pdev;
4680 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4681 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4682 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4683 s32 retval;
4684
f2ca0dbe 4685 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 4686
fef45f4c
AD		 4687 if (retval) {
4688 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 4689 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 4690 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4691 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4692 return;
4693 goto out;
4694 }
4ae196df
AD		 4695
4696 /* this is a message we already processed, do nothing */
4697 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4698 return;
4ae196df
AD		 4699
4700 /*
4701 * until the vf completes a reset it should not be
4702 * allowed to start any configuration.
4703 */
4704
4705 if (msgbuf[0] == E1000_VF_RESET) {
4706 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4707 return;
4ae196df
AD		 4708 }
4709
f2ca0dbe 4710 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 4711 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4712 return;
4713 retval = -1;
4714 goto out;
4ae196df
AD		 4715 }
4716
4717 switch ((msgbuf[0] & 0xFFFF)) {
4718 case E1000_VF_SET_MAC_ADDR:
4719 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4720 break;
7d5753f0
AD		 4721 case E1000_VF_SET_PROMISC:
4722 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4723 break;
4ae196df
AD		 4724 case E1000_VF_SET_MULTICAST:
4725 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4726 break;
4727 case E1000_VF_SET_LPE:
4728 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4729 break;
4730 case E1000_VF_SET_VLAN:
8151d294
WM		 4731 if (adapter->vf_data[vf].pf_vlan)
4732 retval = -1;
4733 else
4734 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 4735 break;
4736 default:
090b1795 4737 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4738 retval = -1;
4739 break;
4740 }
4741
fef45f4c
AD		 4742 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4743out:
4ae196df
AD		 4744 /* notify the VF of the results of what it sent us */
4745 if (retval)
4746 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4747 else
4748 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4749
4ae196df 4750 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4751}
4ae196df 4752
f2ca0dbe
AD		 4753static void igb_msg_task(struct igb_adapter *adapter)
4754{
4755 struct e1000_hw *hw = &adapter->hw;
4756 u32 vf;
4757
4758 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4759 /* process any reset requests */
4760 if (!igb_check_for_rst(hw, vf))
4761 igb_vf_reset_event(adapter, vf);
4762
4763 /* process any messages pending */
4764 if (!igb_check_for_msg(hw, vf))
4765 igb_rcv_msg_from_vf(adapter, vf);
4766
4767 /* process any acks */
4768 if (!igb_check_for_ack(hw, vf))
4769 igb_rcv_ack_from_vf(adapter, vf);
4770 }
4ae196df
AD		 4771}
4772
68d480c4
AD		 4773/**
4774 * igb_set_uta - Set unicast filter table address
4775 * @adapter: board private structure
4776 *
4777 * The unicast table address is a register array of 32-bit registers.
4778 * The table is meant to be used in a way similar to how the MTA is used
4779 * however due to certain limitations in the hardware it is necessary to
4780 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4781 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4782 **/
4783static void igb_set_uta(struct igb_adapter *adapter)
4784{
4785 struct e1000_hw *hw = &adapter->hw;
4786 int i;
4787
4788 /* The UTA table only exists on 82576 hardware and newer */
4789 if (hw->mac.type < e1000_82576)
4790 return;
4791
4792 /* we only need to do this if VMDq is enabled */
4793 if (!adapter->vfs_allocated_count)
4794 return;
4795
4796 for (i = 0; i < hw->mac.uta_reg_count; i++)
4797 array_wr32(E1000_UTA, i, ~0);
4798}
4799
9d5c8243
AK		 4800/**
4801 * igb_intr_msi - Interrupt Handler
4802 * @irq: interrupt number
4803 * @data: pointer to a network interface device structure
4804 **/
4805static irqreturn_t igb_intr_msi(int irq, void *data)
4806{
047e0030
AD		 4807 struct igb_adapter *adapter = data;
4808 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4809 struct e1000_hw *hw = &adapter->hw;
4810 /* read ICR disables interrupts using IAM */
4811 u32 icr = rd32(E1000_ICR);
4812
047e0030 4813 igb_write_itr(q_vector);
9d5c8243 4814
7f081d40
AD		 4815 if (icr & E1000_ICR_DRSTA)
4816 schedule_work(&adapter->reset_task);
4817
047e0030 4818 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4819 /* HW is reporting DMA is out of sync */
4820 adapter->stats.doosync++;
4821 }
4822
9d5c8243
AK		 4823 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4824 hw->mac.get_link_status = 1;
4825 if (!test_bit(__IGB_DOWN, &adapter->state))
4826 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4827 }
4828
047e0030 4829 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4830
4831 return IRQ_HANDLED;
4832}
4833
4834/**
4a3c6433 4835 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4836 * @irq: interrupt number
4837 * @data: pointer to a network interface device structure
4838 **/
4839static irqreturn_t igb_intr(int irq, void *data)
4840{
047e0030
AD		 4841 struct igb_adapter *adapter = data;
4842 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4843 struct e1000_hw *hw = &adapter->hw;
4844 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4845 * need for the IMC write */
4846 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4847 if (!icr)
4848 return IRQ_NONE; /* Not our interrupt */
4849
047e0030 4850 igb_write_itr(q_vector);
9d5c8243
AK		 4851
4852 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4853 * not set, then the adapter didn't send an interrupt */
4854 if (!(icr & E1000_ICR_INT_ASSERTED))
4855 return IRQ_NONE;
4856
7f081d40
AD		 4857 if (icr & E1000_ICR_DRSTA)
4858 schedule_work(&adapter->reset_task);
4859
047e0030 4860 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4861 /* HW is reporting DMA is out of sync */
4862 adapter->stats.doosync++;
4863 }
4864
9d5c8243
AK		 4865 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4866 hw->mac.get_link_status = 1;
4867 /* guard against interrupt when we're going down */
4868 if (!test_bit(__IGB_DOWN, &adapter->state))
4869 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4870 }
4871
047e0030 4872 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4873
4874 return IRQ_HANDLED;
4875}
4876
047e0030 4877static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4878{
047e0030 4879 struct igb_adapter *adapter = q_vector->adapter;
46544258 4880 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4881
4fc82adf
AD		 4882 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4883 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4884 if (!adapter->msix_entries)
6eb5a7f1 4885 igb_set_itr(adapter);
46544258 4886 else
047e0030 4887 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4888 }
4889
46544258
AD		 4890 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4891 if (adapter->msix_entries)
047e0030 4892 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4893 else
4894 igb_irq_enable(adapter);
4895 }
9d5c8243
AK		 4896}
4897
46544258
AD		 4898/**
4899 * igb_poll - NAPI Rx polling callback
4900 * @napi: napi polling structure
4901 * @budget: count of how many packets we should handle
4902 **/
4903static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4904{
047e0030
AD		 4905 struct igb_q_vector *q_vector = container_of(napi,
4906 struct igb_q_vector,
4907 napi);
4908 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4909
421e02f0 4910#ifdef CONFIG_IGB_DCA
047e0030
AD		 4911 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4912 igb_update_dca(q_vector);
fe4506b6 4913#endif
047e0030
AD		 4914 if (q_vector->tx_ring)
4915 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4916
047e0030
AD		 4917 if (q_vector->rx_ring)
4918 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4919
4920 if (!tx_clean_complete)
4921 work_done = budget;
46544258 4922
9d5c8243 4923 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4924 if (work_done < budget) {
288379f0 4925 napi_complete(napi);
047e0030 4926 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4927 }
4928
46544258 4929 return work_done;
9d5c8243 4930}
6d8126f9 4931
33af6bcc 4932/**
c5b9bd5e 4933 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4934 * @adapter: board private structure
c5b9bd5e
AD		 4935 * @shhwtstamps: timestamp structure to update
4936 * @regval: unsigned 64bit system time value.
4937 *
4938 * We need to convert the system time value stored in the RX/TXSTMP registers
4939 * into a hwtstamp which can be used by the upper level timestamping functions
4940 */
4941static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4942 struct skb_shared_hwtstamps *shhwtstamps,
4943 u64 regval)
4944{
4945 u64 ns;
4946
55cac248
AD		 4947 /*
4948 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4949 * 24 to match clock shift we setup earlier.
4950 */
4951 if (adapter->hw.mac.type == e1000_82580)
4952 regval <<= IGB_82580_TSYNC_SHIFT;
4953
c5b9bd5e
AD		 4954 ns = timecounter_cyc2time(&adapter->clock, regval);
4955 timecompare_update(&adapter->compare, ns);
4956 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4957 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4958 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4959}
4960
4961/**
4962 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4963 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4964 * @skb: packet that was just sent
4965 *
4966 * If we were asked to do hardware stamping and such a time stamp is
4967 * available, then it must have been for this skb here because we only
4968 * allow only one such packet into the queue.
4969 */
c5b9bd5e 4970static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4971{
c5b9bd5e 4972 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4973 union skb_shared_tx *shtx = skb_tx(skb);
4974 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4975 struct skb_shared_hwtstamps shhwtstamps;
4976 u64 regval;
33af6bcc 4977
c5b9bd5e
AD		 4978 /* if skb does not support hw timestamp or TX stamp not valid exit */
4979 if (likely(!shtx->hardware) ||
4980 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4981 return;
4982
4983 regval = rd32(E1000_TXSTMPL);
4984 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4985
4986 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4987 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4988}
4989
9d5c8243
AK		 4990/**
4991 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4992 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4993 * returns true if ring is completely cleaned
4994 **/
047e0030 4995static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4996{
047e0030
AD		 4997 struct igb_adapter *adapter = q_vector->adapter;
4998 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4999 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5000 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 5001 struct igb_buffer *buffer_info;
5002 struct sk_buff *skb;
0e014cb1 5003 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5004 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5005 unsigned int i, eop, count = 0;
5006 bool cleaned = false;
9d5c8243 5007
9d5c8243 5008 i = tx_ring->next_to_clean;
0e014cb1
AD		 5009 eop = tx_ring->buffer_info[i].next_to_watch;
5010 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5011
5012 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5013 (count < tx_ring->count)) {
5014 for (cleaned = false; !cleaned; count++) {
5015 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5016 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5017 cleaned = (i == eop);
9d5c8243
AK		 5018 skb = buffer_info->skb;
5019
5020 if (skb) {
5021 unsigned int segs, bytecount;
5022 /* gso_segs is currently only valid for tcp */
40e90c26 5023 segs = buffer_info->gso_segs;
9d5c8243
AK		 5024 /* multiply data chunks by size of headers */
5025 bytecount = ((segs - 1) * skb_headlen(skb)) +
5026 skb->len;
5027 total_packets += segs;
5028 total_bytes += bytecount;
33af6bcc 5029
c5b9bd5e 5030 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 5031 }
5032
80785298 5033 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5034 tx_desc->wb.status = 0;
9d5c8243
AK		 5035
5036 i++;
5037 if (i == tx_ring->count)
5038 i = 0;
9d5c8243 5039 }
0e014cb1
AD		 5040 eop = tx_ring->buffer_info[i].next_to_watch;
5041 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5042 }
5043
9d5c8243
AK		 5044 tx_ring->next_to_clean = i;
5045
fc7d345d 5046 if (unlikely(count &&
9d5c8243 5047 netif_carrier_ok(netdev) &&
c493ea45 5048 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5049 /* Make sure that anybody stopping the queue after this
5050 * sees the new next_to_clean.
5051 */
5052 smp_mb();
661086df
PWJ		 5053 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5054 !(test_bit(__IGB_DOWN, &adapter->state))) {
5055 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5056 tx_ring->tx_stats.restart_queue++;
661086df 5057 }
9d5c8243
AK		 5058 }
5059
5060 if (tx_ring->detect_tx_hung) {
5061 /* Detect a transmit hang in hardware, this serializes the
5062 * check with the clearing of time_stamp and movement of i */
5063 tx_ring->detect_tx_hung = false;
5064 if (tx_ring->buffer_info[i].time_stamp &&
5065 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5066 (adapter->tx_timeout_factor * HZ)) &&
5067 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5068
9d5c8243 5069 /* detected Tx unit hang */
80785298 5070 dev_err(&tx_ring->pdev->dev,
9d5c8243 5071 "Detected Tx Unit Hang\n"
2d064c06 5072 " Tx Queue <%d>\n"
9d5c8243
AK		 5073 " TDH <%x>\n"
5074 " TDT <%x>\n"
5075 " next_to_use <%x>\n"
5076 " next_to_clean <%x>\n"
9d5c8243
AK		 5077 "buffer_info[next_to_clean]\n"
5078 " time_stamp <%lx>\n"
0e014cb1 5079 " next_to_watch <%x>\n"
9d5c8243
AK		 5080 " jiffies <%lx>\n"
5081 " desc.status <%x>\n",
2d064c06 5082 tx_ring->queue_index,
fce99e34
AD		 5083 readl(tx_ring->head),
5084 readl(tx_ring->tail),
9d5c8243
AK		 5085 tx_ring->next_to_use,
5086 tx_ring->next_to_clean,
f7ba205e 5087 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5088 eop,
9d5c8243 5089 jiffies,
0e014cb1 5090 eop_desc->wb.status);
661086df 5091 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5092 }
5093 }
5094 tx_ring->total_bytes += total_bytes;
5095 tx_ring->total_packets += total_packets;
e21ed353
AD		 5096 tx_ring->tx_stats.bytes += total_bytes;
5097 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5098 return (count < tx_ring->count);
9d5c8243
AK		 5099}
5100
9d5c8243
AK		 5101/**
5102 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5103 * @q_vector: structure containing interrupt and ring information
5104 * @skb: packet to send up
5105 * @vlan_tag: vlan tag for packet
9d5c8243 5106 **/
047e0030
AD		 5107static void igb_receive_skb(struct igb_q_vector *q_vector,
5108 struct sk_buff *skb,
5109 u16 vlan_tag)
5110{
5111 struct igb_adapter *adapter = q_vector->adapter;
5112
5113 if (vlan_tag)
5114 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5115 vlan_tag, skb);
182ff8df 5116 else
047e0030 5117 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5118}
5119
04a5fcaa 5120static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5121 u32 status_err, struct sk_buff *skb)
5122{
5123 skb->ip_summed = CHECKSUM_NONE;
5124
5125 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5126 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5127 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5128 return;
85ad76b2 5129
9d5c8243
AK		 5130 /* TCP/UDP checksum error bit is set */
5131 if (status_err &
5132 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5133 /*
5134 * work around errata with sctp packets where the TCPE aka
5135 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5136 * packets, (aka let the stack check the crc32c)
5137 */
85ad76b2
AD		 5138 if ((skb->len == 60) &&
5139 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5140 ring->rx_stats.csum_err++;
85ad76b2 5141
9d5c8243 5142 /* let the stack verify checksum errors */
9d5c8243
AK		 5143 return;
5144 }
5145 /* It must be a TCP or UDP packet with a valid checksum */
5146 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5147 skb->ip_summed = CHECKSUM_UNNECESSARY;
5148
85ad76b2 5149 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5150}
5151
c5b9bd5e
AD		 5152static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5153 struct sk_buff *skb)
5154{
5155 struct igb_adapter *adapter = q_vector->adapter;
5156 struct e1000_hw *hw = &adapter->hw;
5157 u64 regval;
5158
5159 /*
5160 * If this bit is set, then the RX registers contain the time stamp. No
5161 * other packet will be time stamped until we read these registers, so
5162 * read the registers to make them available again. Because only one
5163 * packet can be time stamped at a time, we know that the register
5164 * values must belong to this one here and therefore we don't need to
5165 * compare any of the additional attributes stored for it.
5166 *
5167 * If nothing went wrong, then it should have a skb_shared_tx that we
5168 * can turn into a skb_shared_hwtstamps.
5169 */
5170 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5171 return;
5172 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5173 return;
5174
5175 regval = rd32(E1000_RXSTMPL);
5176 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5177
5178 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5179}
4c844851 5180static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5181 union e1000_adv_rx_desc *rx_desc)
5182{
5183 /* HW will not DMA in data larger than the given buffer, even if it
5184 * parses the (NFS, of course) header to be larger. In that case, it
5185 * fills the header buffer and spills the rest into the page.
5186 */
5187 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5188 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5189 if (hlen > rx_ring->rx_buffer_len)
5190 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5191 return hlen;
5192}
5193
047e0030
AD		 5194static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5195 int *work_done, int budget)
9d5c8243 5196{
047e0030 5197 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5198 struct net_device *netdev = rx_ring->netdev;
80785298 5199 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 5200 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5201 struct igb_buffer *buffer_info , *next_buffer;
5202 struct sk_buff *skb;
9d5c8243
AK		 5203 bool cleaned = false;
5204 int cleaned_count = 0;
d1eff350 5205 int current_node = numa_node_id();
9d5c8243 5206 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5207 unsigned int i;
2d94d8ab
AD		 5208 u32 staterr;
5209 u16 length;
047e0030 5210 u16 vlan_tag;
9d5c8243
AK		 5211
5212 i = rx_ring->next_to_clean;
69d3ca53 5213 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5214 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5215 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5216
5217 while (staterr & E1000_RXD_STAT_DD) {
5218 if (*work_done >= budget)
5219 break;
5220 (*work_done)++;
9d5c8243 5221
69d3ca53
AD		 5222 skb = buffer_info->skb;
5223 prefetch(skb->data - NET_IP_ALIGN);
5224 buffer_info->skb = NULL;
5225
5226 i++;
5227 if (i == rx_ring->count)
5228 i = 0;
42d0781a 5229
69d3ca53
AD		 5230 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5231 prefetch(next_rxd);
5232 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5233
5234 length = le16_to_cpu(rx_desc->wb.upper.length);
5235 cleaned = true;
5236 cleaned_count++;
5237
2d94d8ab 5238 if (buffer_info->dma) {
bf36c1a0 5239 pci_unmap_single(pdev, buffer_info->dma,
4c844851 5240 rx_ring->rx_buffer_len,
bf36c1a0 5241 PCI_DMA_FROMDEVICE);
91615f76 5242 buffer_info->dma = 0;
4c844851 5243 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5244 skb_put(skb, length);
5245 goto send_up;
5246 }
4c844851 5247 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5248 }
5249
5250 if (length) {
9d5c8243 5251 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 5252 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 5253 buffer_info->page_dma = 0;
bf36c1a0
AD		 5254
5255 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5256 buffer_info->page,
5257 buffer_info->page_offset,
5258 length);
5259
d1eff350
AD		 5260 if ((page_count(buffer_info->page) != 1) ||
5261 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5262 buffer_info->page = NULL;
5263 else
5264 get_page(buffer_info->page);
9d5c8243
AK		 5265
5266 skb->len += length;
5267 skb->data_len += length;
bf36c1a0 5268 skb->truesize += length;
9d5c8243 5269 }
9d5c8243 5270
bf36c1a0 5271 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5272 buffer_info->skb = next_buffer->skb;
5273 buffer_info->dma = next_buffer->dma;
5274 next_buffer->skb = skb;
5275 next_buffer->dma = 0;
bf36c1a0
AD		 5276 goto next_desc;
5277 }
69d3ca53 5278send_up:
9d5c8243
AK		 5279 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5280 dev_kfree_skb_irq(skb);
5281 goto next_desc;
5282 }
9d5c8243 5283
c5b9bd5e 5284 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5285 total_bytes += skb->len;
5286 total_packets++;
5287
85ad76b2 5288 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5289
5290 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5291 skb_record_rx_queue(skb, rx_ring->queue_index);
5292
5293 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5294 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5295
047e0030 5296 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5297
9d5c8243
AK		 5298next_desc:
5299 rx_desc->wb.upper.status_error = 0;
5300
5301 /* return some buffers to hardware, one at a time is too slow */
5302 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5303 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5304 cleaned_count = 0;
5305 }
5306
5307 /* use prefetched values */
5308 rx_desc = next_rxd;
5309 buffer_info = next_buffer;
9d5c8243
AK		 5310 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5311 }
bf36c1a0 5312
9d5c8243 5313 rx_ring->next_to_clean = i;
c493ea45 5314 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5315
5316 if (cleaned_count)
3b644cf6 5317 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5318
5319 rx_ring->total_packets += total_packets;
5320 rx_ring->total_bytes += total_bytes;
5321 rx_ring->rx_stats.packets += total_packets;
5322 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5323 return cleaned;
5324}
5325
9d5c8243
AK		 5326/**
5327 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5328 * @adapter: address of board private structure
5329 **/
d7ee5b3a 5330void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5331{
e694e964 5332 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5333 union e1000_adv_rx_desc *rx_desc;
5334 struct igb_buffer *buffer_info;
5335 struct sk_buff *skb;
5336 unsigned int i;
db761762 5337 int bufsz;
9d5c8243
AK		 5338
5339 i = rx_ring->next_to_use;
5340 buffer_info = &rx_ring->buffer_info[i];
5341
4c844851 5342 bufsz = rx_ring->rx_buffer_len;
db761762 5343
9d5c8243
AK		 5344 while (cleaned_count--) {
5345 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5346
6ec43fe6 5347 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5348 if (!buffer_info->page) {
42d0781a 5349 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5350 if (!buffer_info->page) {
04a5fcaa 5351 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5352 goto no_buffers;
5353 }
5354 buffer_info->page_offset = 0;
5355 } else {
5356 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5357 }
5358 buffer_info->page_dma =
80785298 5359 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5360 buffer_info->page_offset,
5361 PAGE_SIZE / 2,
9d5c8243 5362 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5363 if (pci_dma_mapping_error(rx_ring->pdev,
5364 buffer_info->page_dma)) {
5365 buffer_info->page_dma = 0;
5366 rx_ring->rx_stats.alloc_failed++;
5367 goto no_buffers;
5368 }
9d5c8243
AK		 5369 }
5370
42d0781a
AD		 5371 skb = buffer_info->skb;
5372 if (!skb) {
89d71a66 5373 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5374 if (!skb) {
04a5fcaa 5375 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5376 goto no_buffers;
5377 }
5378
9d5c8243 5379 buffer_info->skb = skb;
42d0781a
AD		 5380 }
5381 if (!buffer_info->dma) {
80785298
AD		 5382 buffer_info->dma = pci_map_single(rx_ring->pdev,
5383 skb->data,
9d5c8243
AK		 5384 bufsz,
5385 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5386 if (pci_dma_mapping_error(rx_ring->pdev,
5387 buffer_info->dma)) {
5388 buffer_info->dma = 0;
5389 rx_ring->rx_stats.alloc_failed++;
5390 goto no_buffers;
5391 }
9d5c8243
AK		 5392 }
5393 /* Refresh the desc even if buffer_addrs didn't change because
5394 * each write-back erases this info. */
6ec43fe6 5395 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5396 rx_desc->read.pkt_addr =
5397 cpu_to_le64(buffer_info->page_dma);
5398 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5399 } else {
42d0781a 5400 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5401 rx_desc->read.hdr_addr = 0;
5402 }
5403
5404 i++;
5405 if (i == rx_ring->count)
5406 i = 0;
5407 buffer_info = &rx_ring->buffer_info[i];
5408 }
5409
5410no_buffers:
5411 if (rx_ring->next_to_use != i) {
5412 rx_ring->next_to_use = i;
5413 if (i == 0)
5414 i = (rx_ring->count - 1);
5415 else
5416 i--;
5417
5418 /* Force memory writes to complete before letting h/w
5419 * know there are new descriptors to fetch. (Only
5420 * applicable for weak-ordered memory model archs,
5421 * such as IA-64). */
5422 wmb();
fce99e34 5423 writel(i, rx_ring->tail);
9d5c8243
AK		 5424 }
5425}
5426
5427/**
5428 * igb_mii_ioctl -
5429 * @netdev:
5430 * @ifreq:
5431 * @cmd:
5432 **/
5433static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5434{
5435 struct igb_adapter *adapter = netdev_priv(netdev);
5436 struct mii_ioctl_data *data = if_mii(ifr);
5437
5438 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5439 return -EOPNOTSUPP;
5440
5441 switch (cmd) {
5442 case SIOCGMIIPHY:
5443 data->phy_id = adapter->hw.phy.addr;
5444 break;
5445 case SIOCGMIIREG:
f5f4cf08
AD		 5446 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5447 &data->val_out))
9d5c8243
AK		 5448 return -EIO;
5449 break;
5450 case SIOCSMIIREG:
5451 default:
5452 return -EOPNOTSUPP;
5453 }
5454 return 0;
5455}
5456
c6cb090b
PO		 5457/**
5458 * igb_hwtstamp_ioctl - control hardware time stamping
5459 * @netdev:
5460 * @ifreq:
5461 * @cmd:
5462 *
33af6bcc
PO		 5463 * Outgoing time stamping can be enabled and disabled. Play nice and
5464 * disable it when requested, although it shouldn't case any overhead
5465 * when no packet needs it. At most one packet in the queue may be
5466 * marked for time stamping, otherwise it would be impossible to tell
5467 * for sure to which packet the hardware time stamp belongs.
5468 *
5469 * Incoming time stamping has to be configured via the hardware
5470 * filters. Not all combinations are supported, in particular event
5471 * type has to be specified. Matching the kind of event packet is
5472 * not supported, with the exception of "all V2 events regardless of
5473 * level 2 or 4".
5474 *
c6cb090b
PO		 5475 **/
5476static int igb_hwtstamp_ioctl(struct net_device *netdev,
5477 struct ifreq *ifr, int cmd)
5478{
33af6bcc
PO		 5479 struct igb_adapter *adapter = netdev_priv(netdev);
5480 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5481 struct hwtstamp_config config;
c5b9bd5e
AD		 5482 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5483 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5484 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5485 bool is_l4 = false;
5486 bool is_l2 = false;
33af6bcc 5487 u32 regval;
c6cb090b
PO		 5488
5489 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5490 return -EFAULT;
5491
5492 /* reserved for future extensions */
5493 if (config.flags)
5494 return -EINVAL;
5495
33af6bcc
PO		 5496 switch (config.tx_type) {
5497 case HWTSTAMP_TX_OFF:
c5b9bd5e 5498 tsync_tx_ctl = 0;
33af6bcc 5499 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5500 break;
5501 default:
5502 return -ERANGE;
5503 }
5504
5505 switch (config.rx_filter) {
5506 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5507 tsync_rx_ctl = 0;
33af6bcc
PO		 5508 break;
5509 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5510 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5511 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5512 case HWTSTAMP_FILTER_ALL:
5513 /*
5514 * register TSYNCRXCFG must be set, therefore it is not
5515 * possible to time stamp both Sync and Delay_Req messages
5516 * => fall back to time stamping all packets
5517 */
c5b9bd5e 5518 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5519 config.rx_filter = HWTSTAMP_FILTER_ALL;
5520 break;
5521 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5522 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5523 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5524 is_l4 = true;
33af6bcc
PO		 5525 break;
5526 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5527 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5528 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5529 is_l4 = true;
33af6bcc
PO		 5530 break;
5531 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5532 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5533 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5534 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5535 is_l2 = true;
5536 is_l4 = true;
33af6bcc
PO		 5537 config.rx_filter = HWTSTAMP_FILTER_SOME;
5538 break;
5539 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5540 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5541 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5542 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5543 is_l2 = true;
5544 is_l4 = true;
33af6bcc
PO		 5545 config.rx_filter = HWTSTAMP_FILTER_SOME;
5546 break;
5547 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5548 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5549 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5550 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5551 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5552 is_l2 = true;
33af6bcc
PO		 5553 break;
5554 default:
5555 return -ERANGE;
5556 }
5557
c5b9bd5e
AD		 5558 if (hw->mac.type == e1000_82575) {
5559 if (tsync_rx_ctl | tsync_tx_ctl)
5560 return -EINVAL;
5561 return 0;
5562 }
5563
33af6bcc
PO		 5564 /* enable/disable TX */
5565 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5566 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5567 regval |= tsync_tx_ctl;
33af6bcc
PO		 5568 wr32(E1000_TSYNCTXCTL, regval);
5569
c5b9bd5e 5570 /* enable/disable RX */
33af6bcc 5571 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5572 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5573 regval |= tsync_rx_ctl;
33af6bcc 5574 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5575
c5b9bd5e
AD		 5576 /* define which PTP packets are time stamped */
5577 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5578
c5b9bd5e
AD		 5579 /* define ethertype filter for timestamped packets */
5580 if (is_l2)
5581 wr32(E1000_ETQF(3),
5582 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5583 E1000_ETQF_1588 | /* enable timestamping */
5584 ETH_P_1588)); /* 1588 eth protocol type */
5585 else
5586 wr32(E1000_ETQF(3), 0);
5587
5588#define PTP_PORT 319
5589 /* L4 Queue Filter[3]: filter by destination port and protocol */
5590 if (is_l4) {
5591 u32 ftqf = (IPPROTO_UDP /* UDP */
5592 | E1000_FTQF_VF_BP /* VF not compared */
5593 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5594 | E1000_FTQF_MASK); /* mask all inputs */
5595 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5596
5597 wr32(E1000_IMIR(3), htons(PTP_PORT));
5598 wr32(E1000_IMIREXT(3),
5599 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5600 if (hw->mac.type == e1000_82576) {
5601 /* enable source port check */
5602 wr32(E1000_SPQF(3), htons(PTP_PORT));
5603 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5604 }
5605 wr32(E1000_FTQF(3), ftqf);
5606 } else {
5607 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5608 }
33af6bcc
PO		 5609 wrfl();
5610
5611 adapter->hwtstamp_config = config;
5612
5613 /* clear TX/RX time stamp registers, just to be sure */
5614 regval = rd32(E1000_TXSTMPH);
5615 regval = rd32(E1000_RXSTMPH);
c6cb090b 5616
33af6bcc
PO		 5617 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5618 -EFAULT : 0;
c6cb090b
PO		 5619}
5620
9d5c8243
AK		 5621/**
5622 * igb_ioctl -
5623 * @netdev:
5624 * @ifreq:
5625 * @cmd:
5626 **/
5627static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5628{
5629 switch (cmd) {
5630 case SIOCGMIIPHY:
5631 case SIOCGMIIREG:
5632 case SIOCSMIIREG:
5633 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5634 case SIOCSHWTSTAMP:
5635 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5636 default:
5637 return -EOPNOTSUPP;
5638 }
5639}
5640
009bc06e
AD		 5641s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5642{
5643 struct igb_adapter *adapter = hw->back;
5644 u16 cap_offset;
5645
5646 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5647 if (!cap_offset)
5648 return -E1000_ERR_CONFIG;
5649
5650 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5651
5652 return 0;
5653}
5654
5655s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5656{
5657 struct igb_adapter *adapter = hw->back;
5658 u16 cap_offset;
5659
5660 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5661 if (!cap_offset)
5662 return -E1000_ERR_CONFIG;
5663
5664 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5665
5666 return 0;
5667}
5668
9d5c8243
AK		 5669static void igb_vlan_rx_register(struct net_device *netdev,
5670 struct vlan_group *grp)
5671{
5672 struct igb_adapter *adapter = netdev_priv(netdev);
5673 struct e1000_hw *hw = &adapter->hw;
5674 u32 ctrl, rctl;
5675
5676 igb_irq_disable(adapter);
5677 adapter->vlgrp = grp;
5678
5679 if (grp) {
5680 /* enable VLAN tag insert/strip */
5681 ctrl = rd32(E1000_CTRL);
5682 ctrl |= E1000_CTRL_VME;
5683 wr32(E1000_CTRL, ctrl);
5684
51466239 5685 /* Disable CFI check */
9d5c8243 5686 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5687 rctl &= ~E1000_RCTL_CFIEN;
5688 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5689 } else {
5690 /* disable VLAN tag insert/strip */
5691 ctrl = rd32(E1000_CTRL);
5692 ctrl &= ~E1000_CTRL_VME;
5693 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5694 }
5695
e1739522
AD		 5696 igb_rlpml_set(adapter);
5697
9d5c8243
AK		 5698 if (!test_bit(__IGB_DOWN, &adapter->state))
5699 igb_irq_enable(adapter);
5700}
5701
5702static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5703{
5704 struct igb_adapter *adapter = netdev_priv(netdev);
5705 struct e1000_hw *hw = &adapter->hw;
4ae196df 5706 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5707
51466239
AD		 5708 /* attempt to add filter to vlvf array */
5709 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5710
51466239
AD		 5711 /* add the filter since PF can receive vlans w/o entry in vlvf */
5712 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5713}
5714
5715static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5716{
5717 struct igb_adapter *adapter = netdev_priv(netdev);
5718 struct e1000_hw *hw = &adapter->hw;
4ae196df 5719 int pf_id = adapter->vfs_allocated_count;
51466239 5720 s32 err;
9d5c8243
AK		 5721
5722 igb_irq_disable(adapter);
5723 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5724
5725 if (!test_bit(__IGB_DOWN, &adapter->state))
5726 igb_irq_enable(adapter);
5727
51466239
AD		 5728 /* remove vlan from VLVF table array */
5729 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5730
51466239
AD		 5731 /* if vid was not present in VLVF just remove it from table */
5732 if (err)
4ae196df 5733 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5734}
5735
5736static void igb_restore_vlan(struct igb_adapter *adapter)
5737{
5738 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5739
5740 if (adapter->vlgrp) {
5741 u16 vid;
5742 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5743 if (!vlan_group_get_device(adapter->vlgrp, vid))
5744 continue;
5745 igb_vlan_rx_add_vid(adapter->netdev, vid);
5746 }
5747 }
5748}
5749
5750int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5751{
090b1795 5752 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5753 struct e1000_mac_info *mac = &adapter->hw.mac;
5754
5755 mac->autoneg = 0;
5756
9d5c8243
AK		 5757 switch (spddplx) {
5758 case SPEED_10 + DUPLEX_HALF:
5759 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5760 break;
5761 case SPEED_10 + DUPLEX_FULL:
5762 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5763 break;
5764 case SPEED_100 + DUPLEX_HALF:
5765 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5766 break;
5767 case SPEED_100 + DUPLEX_FULL:
5768 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5769 break;
5770 case SPEED_1000 + DUPLEX_FULL:
5771 mac->autoneg = 1;
5772 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5773 break;
5774 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5775 default:
090b1795 5776 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5777 return -EINVAL;
5778 }
5779 return 0;
5780}
5781
3fe7c4c9 5782static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5783{
5784 struct net_device *netdev = pci_get_drvdata(pdev);
5785 struct igb_adapter *adapter = netdev_priv(netdev);
5786 struct e1000_hw *hw = &adapter->hw;
2d064c06 5787 u32 ctrl, rctl, status;
9d5c8243
AK		 5788 u32 wufc = adapter->wol;
5789#ifdef CONFIG_PM
5790 int retval = 0;
5791#endif
5792
5793 netif_device_detach(netdev);
5794
a88f10ec
AD		 5795 if (netif_running(netdev))
5796 igb_close(netdev);
5797
047e0030 5798 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5799
5800#ifdef CONFIG_PM
5801 retval = pci_save_state(pdev);
5802 if (retval)
5803 return retval;
5804#endif
5805
5806 status = rd32(E1000_STATUS);
5807 if (status & E1000_STATUS_LU)
5808 wufc &= ~E1000_WUFC_LNKC;
5809
5810 if (wufc) {
5811 igb_setup_rctl(adapter);
ff41f8dc 5812 igb_set_rx_mode(netdev);
9d5c8243
AK		 5813
5814 /* turn on all-multi mode if wake on multicast is enabled */
5815 if (wufc & E1000_WUFC_MC) {
5816 rctl = rd32(E1000_RCTL);
5817 rctl |= E1000_RCTL_MPE;
5818 wr32(E1000_RCTL, rctl);
5819 }
5820
5821 ctrl = rd32(E1000_CTRL);
5822 /* advertise wake from D3Cold */
5823 #define E1000_CTRL_ADVD3WUC 0x00100000
5824 /* phy power management enable */
5825 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5826 ctrl |= E1000_CTRL_ADVD3WUC;
5827 wr32(E1000_CTRL, ctrl);
5828
9d5c8243 5829 /* Allow time for pending master requests to run */
330a6d6a 5830 igb_disable_pcie_master(hw);
9d5c8243
AK		 5831
5832 wr32(E1000_WUC, E1000_WUC_PME_EN);
5833 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5834 } else {
5835 wr32(E1000_WUC, 0);
5836 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5837 }
5838
3fe7c4c9
RW		 5839 *enable_wake = wufc || adapter->en_mng_pt;
5840 if (!*enable_wake)
88a268c1
NN		 5841 igb_power_down_link(adapter);
5842 else
5843 igb_power_up_link(adapter);
9d5c8243
AK		 5844
5845 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5846 * would have already happened in close and is redundant. */
5847 igb_release_hw_control(adapter);
5848
5849 pci_disable_device(pdev);
5850
9d5c8243
AK		 5851 return 0;
5852}
5853
5854#ifdef CONFIG_PM
3fe7c4c9
RW		 5855static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5856{
5857 int retval;
5858 bool wake;
5859
5860 retval = __igb_shutdown(pdev, &wake);
5861 if (retval)
5862 return retval;
5863
5864 if (wake) {
5865 pci_prepare_to_sleep(pdev);
5866 } else {
5867 pci_wake_from_d3(pdev, false);
5868 pci_set_power_state(pdev, PCI_D3hot);
5869 }
5870
5871 return 0;
5872}
5873
9d5c8243
AK		 5874static int igb_resume(struct pci_dev *pdev)
5875{
5876 struct net_device *netdev = pci_get_drvdata(pdev);
5877 struct igb_adapter *adapter = netdev_priv(netdev);
5878 struct e1000_hw *hw = &adapter->hw;
5879 u32 err;
5880
5881 pci_set_power_state(pdev, PCI_D0);
5882 pci_restore_state(pdev);
b94f2d77 5883 pci_save_state(pdev);
42bfd33a 5884
aed5dec3 5885 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5886 if (err) {
5887 dev_err(&pdev->dev,
5888 "igb: Cannot enable PCI device from suspend\n");
5889 return err;
5890 }
5891 pci_set_master(pdev);
5892
5893 pci_enable_wake(pdev, PCI_D3hot, 0);
5894 pci_enable_wake(pdev, PCI_D3cold, 0);
5895
047e0030 5896 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5897 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5898 return -ENOMEM;
9d5c8243
AK		 5899 }
5900
9d5c8243 5901 igb_reset(adapter);
a8564f03
AD		 5902
5903 /* let the f/w know that the h/w is now under the control of the
5904 * driver. */
5905 igb_get_hw_control(adapter);
5906
9d5c8243
AK		 5907 wr32(E1000_WUS, ~0);
5908
a88f10ec
AD		 5909 if (netif_running(netdev)) {
5910 err = igb_open(netdev);
5911 if (err)
5912 return err;
5913 }
9d5c8243
AK		 5914
5915 netif_device_attach(netdev);
5916
9d5c8243
AK		 5917 return 0;
5918}
5919#endif
5920
5921static void igb_shutdown(struct pci_dev *pdev)
5922{
3fe7c4c9
RW		 5923 bool wake;
5924
5925 __igb_shutdown(pdev, &wake);
5926
5927 if (system_state == SYSTEM_POWER_OFF) {
5928 pci_wake_from_d3(pdev, wake);
5929 pci_set_power_state(pdev, PCI_D3hot);
5930 }
9d5c8243
AK		 5931}
5932
5933#ifdef CONFIG_NET_POLL_CONTROLLER
5934/*
5935 * Polling 'interrupt' - used by things like netconsole to send skbs
5936 * without having to re-enable interrupts. It's not called while
5937 * the interrupt routine is executing.
5938 */
5939static void igb_netpoll(struct net_device *netdev)
5940{
5941 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5942 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5943 int i;
9d5c8243 5944
eebbbdba 5945 if (!adapter->msix_entries) {
047e0030 5946 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5947 igb_irq_disable(adapter);
047e0030 5948 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5949 return;
5950 }
9d5c8243 5951
047e0030
AD		 5952 for (i = 0; i < adapter->num_q_vectors; i++) {
5953 struct igb_q_vector *q_vector = adapter->q_vector[i];
5954 wr32(E1000_EIMC, q_vector->eims_value);
5955 napi_schedule(&q_vector->napi);
eebbbdba 5956 }
9d5c8243
AK		 5957}
5958#endif /* CONFIG_NET_POLL_CONTROLLER */
5959
5960/**
5961 * igb_io_error_detected - called when PCI error is detected
5962 * @pdev: Pointer to PCI device
5963 * @state: The current pci connection state
5964 *
5965 * This function is called after a PCI bus error affecting
5966 * this device has been detected.
5967 */
5968static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5969 pci_channel_state_t state)
5970{
5971 struct net_device *netdev = pci_get_drvdata(pdev);
5972 struct igb_adapter *adapter = netdev_priv(netdev);
5973
5974 netif_device_detach(netdev);
5975
59ed6eec
AD		 5976 if (state == pci_channel_io_perm_failure)
5977 return PCI_ERS_RESULT_DISCONNECT;
5978
9d5c8243
AK		 5979 if (netif_running(netdev))
5980 igb_down(adapter);
5981 pci_disable_device(pdev);
5982
5983 /* Request a slot slot reset. */
5984 return PCI_ERS_RESULT_NEED_RESET;
5985}
5986
5987/**
5988 * igb_io_slot_reset - called after the pci bus has been reset.
5989 * @pdev: Pointer to PCI device
5990 *
5991 * Restart the card from scratch, as if from a cold-boot. Implementation
5992 * resembles the first-half of the igb_resume routine.
5993 */
5994static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5995{
5996 struct net_device *netdev = pci_get_drvdata(pdev);
5997 struct igb_adapter *adapter = netdev_priv(netdev);
5998 struct e1000_hw *hw = &adapter->hw;
40a914fa 5999 pci_ers_result_t result;
42bfd33a 6000 int err;
9d5c8243 6001
aed5dec3 6002 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6003 dev_err(&pdev->dev,
6004 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6005 result = PCI_ERS_RESULT_DISCONNECT;
6006 } else {
6007 pci_set_master(pdev);
6008 pci_restore_state(pdev);
b94f2d77 6009 pci_save_state(pdev);
9d5c8243 6010
40a914fa
AD		 6011 pci_enable_wake(pdev, PCI_D3hot, 0);
6012 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6013
40a914fa
AD		 6014 igb_reset(adapter);
6015 wr32(E1000_WUS, ~0);
6016 result = PCI_ERS_RESULT_RECOVERED;
6017 }
9d5c8243 6018
ea943d41
JK		 6019 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6020 if (err) {
6021 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6022 "failed 0x%0x\n", err);
6023 /* non-fatal, continue */
6024 }
40a914fa
AD		 6025
6026 return result;
9d5c8243
AK		 6027}
6028
6029/**
6030 * igb_io_resume - called when traffic can start flowing again.
6031 * @pdev: Pointer to PCI device
6032 *
6033 * This callback is called when the error recovery driver tells us that
6034 * its OK to resume normal operation. Implementation resembles the
6035 * second-half of the igb_resume routine.
6036 */
6037static void igb_io_resume(struct pci_dev *pdev)
6038{
6039 struct net_device *netdev = pci_get_drvdata(pdev);
6040 struct igb_adapter *adapter = netdev_priv(netdev);
6041
9d5c8243
AK		 6042 if (netif_running(netdev)) {
6043 if (igb_up(adapter)) {
6044 dev_err(&pdev->dev, "igb_up failed after reset\n");
6045 return;
6046 }
6047 }
6048
6049 netif_device_attach(netdev);
6050
6051 /* let the f/w know that the h/w is now under the control of the
6052 * driver. */
6053 igb_get_hw_control(adapter);
9d5c8243
AK		 6054}
6055
26ad9178
AD		 6056static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6057 u8 qsel)
6058{
6059 u32 rar_low, rar_high;
6060 struct e1000_hw *hw = &adapter->hw;
6061
6062 /* HW expects these in little endian so we reverse the byte order
6063 * from network order (big endian) to little endian
6064 */
6065 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6066 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6067 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6068
6069 /* Indicate to hardware the Address is Valid. */
6070 rar_high |= E1000_RAH_AV;
6071
6072 if (hw->mac.type == e1000_82575)
6073 rar_high |= E1000_RAH_POOL_1 * qsel;
6074 else
6075 rar_high |= E1000_RAH_POOL_1 << qsel;
6076
6077 wr32(E1000_RAL(index), rar_low);
6078 wrfl();
6079 wr32(E1000_RAH(index), rar_high);
6080 wrfl();
6081}
6082
4ae196df
AD		 6083static int igb_set_vf_mac(struct igb_adapter *adapter,
6084 int vf, unsigned char *mac_addr)
6085{
6086 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6087 /* VF MAC addresses start at end of receive addresses and moves
6088 * torwards the first, as a result a collision should not be possible */
6089 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6090
37680117 6091 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6092
26ad9178 6093 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6094
6095 return 0;
6096}
6097
8151d294
WM		 6098static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6099{
6100 struct igb_adapter *adapter = netdev_priv(netdev);
6101 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6102 return -EINVAL;
6103 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6104 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6105 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6106 " change effective.");
6107 if (test_bit(__IGB_DOWN, &adapter->state)) {
6108 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6109 " but the PF device is not up.\n");
6110 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6111 " attempting to use the VF device.\n");
6112 }
6113 return igb_set_vf_mac(adapter, vf, mac);
6114}
6115
6116static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6117{
6118 return -EOPNOTSUPP;
6119}
6120
6121static int igb_ndo_get_vf_config(struct net_device *netdev,
6122 int vf, struct ifla_vf_info *ivi)
6123{
6124 struct igb_adapter *adapter = netdev_priv(netdev);
6125 if (vf >= adapter->vfs_allocated_count)
6126 return -EINVAL;
6127 ivi->vf = vf;
6128 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6129 ivi->tx_rate = 0;
6130 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6131 ivi->qos = adapter->vf_data[vf].pf_qos;
6132 return 0;
6133}
6134
4ae196df
AD		 6135static void igb_vmm_control(struct igb_adapter *adapter)
6136{
6137 struct e1000_hw *hw = &adapter->hw;
10d8e907 6138 u32 reg;
4ae196df 6139
d4960307
AD		 6140 /* replication is not supported for 82575 */
6141 if (hw->mac.type == e1000_82575)
4ae196df
AD		 6142 return;
6143
10d8e907
AD		 6144 /* enable replication vlan tag stripping */
6145 reg = rd32(E1000_RPLOLR);
6146 reg |= E1000_RPLOLR_STRVLAN;
6147 wr32(E1000_RPLOLR, reg);
6148
6149 /* notify HW that the MAC is adding vlan tags */
6150 reg = rd32(E1000_DTXCTL);
6151 reg |= E1000_DTXCTL_VLAN_ADDED;
6152 wr32(E1000_DTXCTL, reg);
6153
d4960307
AD		 6154 if (adapter->vfs_allocated_count) {
6155 igb_vmdq_set_loopback_pf(hw, true);
6156 igb_vmdq_set_replication_pf(hw, true);
6157 } else {
6158 igb_vmdq_set_loopback_pf(hw, false);
6159 igb_vmdq_set_replication_pf(hw, false);
6160 }
4ae196df
AD		 6161}
6162
9d5c8243 6163/* igb_main.c */
Clone of davem's net-next-2.6 tree