]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/igb/igb_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: minor type cleanups
[net-next-2.6.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
55cac248 52#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
a3aa1884 63static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
55cac248
AD		 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
79 /* required last entry */
80 {0, }
81};
82
83MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
84
85void igb_reset(struct igb_adapter *);
86static int igb_setup_all_tx_resources(struct igb_adapter *);
87static int igb_setup_all_rx_resources(struct igb_adapter *);
88static void igb_free_all_tx_resources(struct igb_adapter *);
89static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 90static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 91void igb_update_stats(struct igb_adapter *);
92static int igb_probe(struct pci_dev *, const struct pci_device_id *);
93static void __devexit igb_remove(struct pci_dev *pdev);
94static int igb_sw_init(struct igb_adapter *);
95static int igb_open(struct net_device *);
96static int igb_close(struct net_device *);
97static void igb_configure_tx(struct igb_adapter *);
98static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 99static void igb_clean_all_tx_rings(struct igb_adapter *);
100static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 101static void igb_clean_tx_ring(struct igb_ring *);
102static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 103static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 104static void igb_update_phy_info(unsigned long);
105static void igb_watchdog(unsigned long);
106static void igb_watchdog_task(struct work_struct *);
b1a436c3 107static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 108static struct net_device_stats *igb_get_stats(struct net_device *);
109static int igb_change_mtu(struct net_device *, int);
110static int igb_set_mac(struct net_device *, void *);
68d480c4 111static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 112static irqreturn_t igb_intr(int irq, void *);
113static irqreturn_t igb_intr_msi(int irq, void *);
114static irqreturn_t igb_msix_other(int irq, void *);
047e0030 115static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 116#ifdef CONFIG_IGB_DCA
047e0030 117static void igb_update_dca(struct igb_q_vector *);
fe4506b6 118static void igb_setup_dca(struct igb_adapter *);
421e02f0 119#endif /* CONFIG_IGB_DCA */
047e0030 120static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 121static int igb_poll(struct napi_struct *, int);
047e0030 122static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 123static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
124static void igb_tx_timeout(struct net_device *);
125static void igb_reset_task(struct work_struct *);
126static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
127static void igb_vlan_rx_add_vid(struct net_device *, u16);
128static void igb_vlan_rx_kill_vid(struct net_device *, u16);
129static void igb_restore_vlan(struct igb_adapter *);
26ad9178 130static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 131static void igb_ping_all_vfs(struct igb_adapter *);
132static void igb_msg_task(struct igb_adapter *);
4ae196df 133static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 134static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 135static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 136static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
137static int igb_ndo_set_vf_vlan(struct net_device *netdev,
138 int vf, u16 vlan, u8 qos);
139static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
140static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
141 struct ifla_vf_info *ivi);
9d5c8243 142
9d5c8243 143#ifdef CONFIG_PM
3fe7c4c9 144static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 145static int igb_resume(struct pci_dev *);
146#endif
147static void igb_shutdown(struct pci_dev *);
421e02f0 148#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 149static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
150static struct notifier_block dca_notifier = {
151 .notifier_call = igb_notify_dca,
152 .next = NULL,
153 .priority = 0
154};
155#endif
9d5c8243
AK		 156#ifdef CONFIG_NET_POLL_CONTROLLER
157/* for netdump / net console */
158static void igb_netpoll(struct net_device *);
159#endif
37680117 160#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 161static unsigned int max_vfs = 0;
162module_param(max_vfs, uint, 0);
163MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
164 "per physical function");
165#endif /* CONFIG_PCI_IOV */
166
9d5c8243
AK		 167static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
168 pci_channel_state_t);
169static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
170static void igb_io_resume(struct pci_dev *);
171
172static struct pci_error_handlers igb_err_handler = {
173 .error_detected = igb_io_error_detected,
174 .slot_reset = igb_io_slot_reset,
175 .resume = igb_io_resume,
176};
177
178
179static struct pci_driver igb_driver = {
180 .name = igb_driver_name,
181 .id_table = igb_pci_tbl,
182 .probe = igb_probe,
183 .remove = __devexit_p(igb_remove),
184#ifdef CONFIG_PM
185 /* Power Managment Hooks */
186 .suspend = igb_suspend,
187 .resume = igb_resume,
188#endif
189 .shutdown = igb_shutdown,
190 .err_handler = &igb_err_handler
191};
192
193MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
194MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
195MODULE_LICENSE("GPL");
196MODULE_VERSION(DRV_VERSION);
197
38c845c7
PO		 198/**
199 * igb_read_clock - read raw cycle counter (to be used by time counter)
200 */
201static cycle_t igb_read_clock(const struct cyclecounter *tc)
202{
203 struct igb_adapter *adapter =
204 container_of(tc, struct igb_adapter, cycles);
205 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 206 u64 stamp = 0;
207 int shift = 0;
38c845c7 208
55cac248
AD		 209 /*
210 * The timestamp latches on lowest register read. For the 82580
211 * the lowest register is SYSTIMR instead of SYSTIML. However we never
212 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
213 */
214 if (hw->mac.type == e1000_82580) {
215 stamp = rd32(E1000_SYSTIMR) >> 8;
216 shift = IGB_82580_TSYNC_SHIFT;
217 }
218
c5b9bd5e
AD		 219 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
220 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 221 return stamp;
222}
223
9d5c8243
AK		 224#ifdef DEBUG
225/**
226 * igb_get_hw_dev_name - return device name string
227 * used by hardware layer to print debugging information
228 **/
229char *igb_get_hw_dev_name(struct e1000_hw *hw)
230{
231 struct igb_adapter *adapter = hw->back;
232 return adapter->netdev->name;
233}
38c845c7
PO		 234
235/**
236 * igb_get_time_str - format current NIC and system time as string
237 */
238static char *igb_get_time_str(struct igb_adapter *adapter,
239 char buffer[160])
240{
241 cycle_t hw = adapter->cycles.read(&adapter->cycles);
242 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
243 struct timespec sys;
244 struct timespec delta;
245 getnstimeofday(&sys);
246
247 delta = timespec_sub(nic, sys);
248
249 sprintf(buffer,
33af6bcc
PO		 250 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
251 hw,
38c845c7
PO		 252 (long)nic.tv_sec, nic.tv_nsec,
253 (long)sys.tv_sec, sys.tv_nsec,
254 (long)delta.tv_sec, delta.tv_nsec);
255
256 return buffer;
257}
9d5c8243
AK		 258#endif
259
260/**
261 * igb_init_module - Driver Registration Routine
262 *
263 * igb_init_module is the first routine called when the driver is
264 * loaded. All it does is register with the PCI subsystem.
265 **/
266static int __init igb_init_module(void)
267{
268 int ret;
269 printk(KERN_INFO "%s - version %s\n",
270 igb_driver_string, igb_driver_version);
271
272 printk(KERN_INFO "%s\n", igb_copyright);
273
421e02f0 274#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 275 dca_register_notify(&dca_notifier);
276#endif
bbd98fe4 277 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 278 return ret;
279}
280
281module_init(igb_init_module);
282
283/**
284 * igb_exit_module - Driver Exit Cleanup Routine
285 *
286 * igb_exit_module is called just before the driver is removed
287 * from memory.
288 **/
289static void __exit igb_exit_module(void)
290{
421e02f0 291#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 292 dca_unregister_notify(&dca_notifier);
293#endif
9d5c8243
AK		 294 pci_unregister_driver(&igb_driver);
295}
296
297module_exit(igb_exit_module);
298
26bc19ec
AD		 299#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
300/**
301 * igb_cache_ring_register - Descriptor ring to register mapping
302 * @adapter: board private structure to initialize
303 *
304 * Once we know the feature-set enabled for the device, we'll cache
305 * the register offset the descriptor ring is assigned to.
306 **/
307static void igb_cache_ring_register(struct igb_adapter *adapter)
308{
ee1b9f06 309 int i = 0, j = 0;
047e0030 310 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 311
312 switch (adapter->hw.mac.type) {
313 case e1000_82576:
314 /* The queues are allocated for virtualization such that VF 0
315 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
316 * In order to avoid collision we start at the first free queue
317 * and continue consuming queues in the same sequence
318 */
ee1b9f06 319 if (adapter->vfs_allocated_count) {
a99955fc 320 for (; i < adapter->rss_queues; i++)
3025a446
AD		 321 adapter->rx_ring[i]->reg_idx = rbase_offset +
322 Q_IDX_82576(i);
a99955fc 323 for (; j < adapter->rss_queues; j++)
3025a446
AD		 324 adapter->tx_ring[j]->reg_idx = rbase_offset +
325 Q_IDX_82576(j);
ee1b9f06 326 }
26bc19ec 327 case e1000_82575:
55cac248 328 case e1000_82580:
26bc19ec 329 default:
ee1b9f06 330 for (; i < adapter->num_rx_queues; i++)
3025a446 331 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 332 for (; j < adapter->num_tx_queues; j++)
3025a446 333 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 334 break;
335 }
336}
337
047e0030
AD		 338static void igb_free_queues(struct igb_adapter *adapter)
339{
3025a446 340 int i;
047e0030 341
3025a446
AD		 342 for (i = 0; i < adapter->num_tx_queues; i++) {
343 kfree(adapter->tx_ring[i]);
344 adapter->tx_ring[i] = NULL;
345 }
346 for (i = 0; i < adapter->num_rx_queues; i++) {
347 kfree(adapter->rx_ring[i]);
348 adapter->rx_ring[i] = NULL;
349 }
047e0030
AD		 350 adapter->num_rx_queues = 0;
351 adapter->num_tx_queues = 0;
352}
353
9d5c8243
AK		 354/**
355 * igb_alloc_queues - Allocate memory for all rings
356 * @adapter: board private structure to initialize
357 *
358 * We allocate one ring per queue at run-time since we don't know the
359 * number of queues at compile-time.
360 **/
361static int igb_alloc_queues(struct igb_adapter *adapter)
362{
3025a446 363 struct igb_ring *ring;
9d5c8243
AK		 364 int i;
365
661086df 366 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 367 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
368 if (!ring)
369 goto err;
68fd9910 370 ring->count = adapter->tx_ring_count;
661086df 371 ring->queue_index = i;
80785298 372 ring->pdev = adapter->pdev;
e694e964 373 ring->netdev = adapter->netdev;
85ad76b2
AD		 374 /* For 82575, context index must be unique per ring. */
375 if (adapter->hw.mac.type == e1000_82575)
376 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 377 adapter->tx_ring[i] = ring;
661086df 378 }
85ad76b2 379
9d5c8243 380 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 381 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
382 if (!ring)
383 goto err;
68fd9910 384 ring->count = adapter->rx_ring_count;
844290e5 385 ring->queue_index = i;
80785298 386 ring->pdev = adapter->pdev;
e694e964 387 ring->netdev = adapter->netdev;
4c844851 388 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 389 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
390 /* set flag indicating ring supports SCTP checksum offload */
391 if (adapter->hw.mac.type >= e1000_82576)
392 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 393 adapter->rx_ring[i] = ring;
9d5c8243 394 }
26bc19ec
AD		 395
396 igb_cache_ring_register(adapter);
9d5c8243 397
047e0030 398 return 0;
a88f10ec 399
047e0030
AD		 400err:
401 igb_free_queues(adapter);
d1a8c9e1 402
047e0030 403 return -ENOMEM;
a88f10ec
AD		 404}
405
9d5c8243 406#define IGB_N0_QUEUE -1
047e0030 407static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 408{
409 u32 msixbm = 0;
047e0030 410 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 411 struct e1000_hw *hw = &adapter->hw;
2d064c06 412 u32 ivar, index;
047e0030
AD		 413 int rx_queue = IGB_N0_QUEUE;
414 int tx_queue = IGB_N0_QUEUE;
415
416 if (q_vector->rx_ring)
417 rx_queue = q_vector->rx_ring->reg_idx;
418 if (q_vector->tx_ring)
419 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 420
421 switch (hw->mac.type) {
422 case e1000_82575:
9d5c8243
AK		 423 /* The 82575 assigns vectors using a bitmask, which matches the
424 bitmask for the EICR/EIMS/EIMC registers. To assign one
425 or more queues to a vector, we write the appropriate bits
426 into the MSIXBM register for that vector. */
047e0030 427 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 428 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 429 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 430 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 431 if (!adapter->msix_entries && msix_vector == 0)
432 msixbm |= E1000_EIMS_OTHER;
9d5c8243 433 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 434 q_vector->eims_value = msixbm;
2d064c06
AD		 435 break;
436 case e1000_82576:
26bc19ec 437 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 438 Each queue has a single entry in the table to which we write
439 a vector number along with a "valid" bit. Sadly, the layout
440 of the table is somewhat counterintuitive. */
441 if (rx_queue > IGB_N0_QUEUE) {
047e0030 442 index = (rx_queue & 0x7);
2d064c06 443 ivar = array_rd32(E1000_IVAR0, index);
047e0030 444 if (rx_queue < 8) {
26bc19ec
AD		 445 /* vector goes into low byte of register */
446 ivar = ivar & 0xFFFFFF00;
447 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 448 } else {
449 /* vector goes into third byte of register */
450 ivar = ivar & 0xFF00FFFF;
451 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 452 }
2d064c06
AD		 453 array_wr32(E1000_IVAR0, index, ivar);
454 }
455 if (tx_queue > IGB_N0_QUEUE) {
047e0030 456 index = (tx_queue & 0x7);
2d064c06 457 ivar = array_rd32(E1000_IVAR0, index);
047e0030 458 if (tx_queue < 8) {
26bc19ec
AD		 459 /* vector goes into second byte of register */
460 ivar = ivar & 0xFFFF00FF;
461 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 462 } else {
463 /* vector goes into high byte of register */
464 ivar = ivar & 0x00FFFFFF;
465 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 466 }
2d064c06
AD		 467 array_wr32(E1000_IVAR0, index, ivar);
468 }
047e0030 469 q_vector->eims_value = 1 << msix_vector;
2d064c06 470 break;
55cac248
AD		 471 case e1000_82580:
472 /* 82580 uses the same table-based approach as 82576 but has fewer
473 entries as a result we carry over for queues greater than 4. */
474 if (rx_queue > IGB_N0_QUEUE) {
475 index = (rx_queue >> 1);
476 ivar = array_rd32(E1000_IVAR0, index);
477 if (rx_queue & 0x1) {
478 /* vector goes into third byte of register */
479 ivar = ivar & 0xFF00FFFF;
480 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
481 } else {
482 /* vector goes into low byte of register */
483 ivar = ivar & 0xFFFFFF00;
484 ivar |= msix_vector | E1000_IVAR_VALID;
485 }
486 array_wr32(E1000_IVAR0, index, ivar);
487 }
488 if (tx_queue > IGB_N0_QUEUE) {
489 index = (tx_queue >> 1);
490 ivar = array_rd32(E1000_IVAR0, index);
491 if (tx_queue & 0x1) {
492 /* vector goes into high byte of register */
493 ivar = ivar & 0x00FFFFFF;
494 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
495 } else {
496 /* vector goes into second byte of register */
497 ivar = ivar & 0xFFFF00FF;
498 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
499 }
500 array_wr32(E1000_IVAR0, index, ivar);
501 }
502 q_vector->eims_value = 1 << msix_vector;
503 break;
2d064c06
AD		 504 default:
505 BUG();
506 break;
507 }
26b39276
AD		 508
509 /* add q_vector eims value to global eims_enable_mask */
510 adapter->eims_enable_mask |= q_vector->eims_value;
511
512 /* configure q_vector to set itr on first interrupt */
513 q_vector->set_itr = 1;
9d5c8243
AK		 514}
515
516/**
517 * igb_configure_msix - Configure MSI-X hardware
518 *
519 * igb_configure_msix sets up the hardware to properly
520 * generate MSI-X interrupts.
521 **/
522static void igb_configure_msix(struct igb_adapter *adapter)
523{
524 u32 tmp;
525 int i, vector = 0;
526 struct e1000_hw *hw = &adapter->hw;
527
528 adapter->eims_enable_mask = 0;
9d5c8243
AK		 529
530 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 531 switch (hw->mac.type) {
532 case e1000_82575:
9d5c8243
AK		 533 tmp = rd32(E1000_CTRL_EXT);
534 /* enable MSI-X PBA support*/
535 tmp |= E1000_CTRL_EXT_PBA_CLR;
536
537 /* Auto-Mask interrupts upon ICR read. */
538 tmp |= E1000_CTRL_EXT_EIAME;
539 tmp |= E1000_CTRL_EXT_IRCA;
540
541 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 542
543 /* enable msix_other interrupt */
544 array_wr32(E1000_MSIXBM(0), vector++,
545 E1000_EIMS_OTHER);
844290e5 546 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 547
2d064c06
AD		 548 break;
549
550 case e1000_82576:
55cac248 551 case e1000_82580:
047e0030
AD		 552 /* Turn on MSI-X capability first, or our settings
553 * won't stick. And it will take days to debug. */
554 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
555 E1000_GPIE_PBA | E1000_GPIE_EIAME |
556 E1000_GPIE_NSICR);
557
558 /* enable msix_other interrupt */
559 adapter->eims_other = 1 << vector;
2d064c06 560 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 561
047e0030 562 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 563 break;
564 default:
565 /* do nothing, since nothing else supports MSI-X */
566 break;
567 } /* switch (hw->mac.type) */
047e0030
AD		 568
569 adapter->eims_enable_mask |= adapter->eims_other;
570
26b39276
AD		 571 for (i = 0; i < adapter->num_q_vectors; i++)
572 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 573
9d5c8243
AK		 574 wrfl();
575}
576
577/**
578 * igb_request_msix - Initialize MSI-X interrupts
579 *
580 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
581 * kernel.
582 **/
583static int igb_request_msix(struct igb_adapter *adapter)
584{
585 struct net_device *netdev = adapter->netdev;
047e0030 586 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 587 int i, err = 0, vector = 0;
588
047e0030 589 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 590 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 591 if (err)
592 goto out;
593 vector++;
594
595 for (i = 0; i < adapter->num_q_vectors; i++) {
596 struct igb_q_vector *q_vector = adapter->q_vector[i];
597
598 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
599
600 if (q_vector->rx_ring && q_vector->tx_ring)
601 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
602 q_vector->rx_ring->queue_index);
603 else if (q_vector->tx_ring)
604 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
605 q_vector->tx_ring->queue_index);
606 else if (q_vector->rx_ring)
607 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
608 q_vector->rx_ring->queue_index);
9d5c8243 609 else
047e0030
AD		 610 sprintf(q_vector->name, "%s-unused", netdev->name);
611
9d5c8243 612 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 613 igb_msix_ring, 0, q_vector->name,
047e0030 614 q_vector);
9d5c8243
AK		 615 if (err)
616 goto out;
9d5c8243
AK		 617 vector++;
618 }
619
9d5c8243
AK		 620 igb_configure_msix(adapter);
621 return 0;
622out:
623 return err;
624}
625
626static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
627{
628 if (adapter->msix_entries) {
629 pci_disable_msix(adapter->pdev);
630 kfree(adapter->msix_entries);
631 adapter->msix_entries = NULL;
047e0030 632 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 633 pci_disable_msi(adapter->pdev);
047e0030 634 }
9d5c8243
AK		 635}
636
047e0030
AD		 637/**
638 * igb_free_q_vectors - Free memory allocated for interrupt vectors
639 * @adapter: board private structure to initialize
640 *
641 * This function frees the memory allocated to the q_vectors. In addition if
642 * NAPI is enabled it will delete any references to the NAPI struct prior
643 * to freeing the q_vector.
644 **/
645static void igb_free_q_vectors(struct igb_adapter *adapter)
646{
647 int v_idx;
648
649 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
650 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
651 adapter->q_vector[v_idx] = NULL;
652 netif_napi_del(&q_vector->napi);
653 kfree(q_vector);
654 }
655 adapter->num_q_vectors = 0;
656}
657
658/**
659 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
660 *
661 * This function resets the device so that it has 0 rx queues, tx queues, and
662 * MSI-X interrupts allocated.
663 */
664static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
665{
666 igb_free_queues(adapter);
667 igb_free_q_vectors(adapter);
668 igb_reset_interrupt_capability(adapter);
669}
9d5c8243
AK		 670
671/**
672 * igb_set_interrupt_capability - set MSI or MSI-X if supported
673 *
674 * Attempt to configure interrupts using the best available
675 * capabilities of the hardware and kernel.
676 **/
677static void igb_set_interrupt_capability(struct igb_adapter *adapter)
678{
679 int err;
680 int numvecs, i;
681
83b7180d 682 /* Number of supported queues. */
a99955fc
AD		 683 adapter->num_rx_queues = adapter->rss_queues;
684 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 685
047e0030
AD		 686 /* start with one vector for every rx queue */
687 numvecs = adapter->num_rx_queues;
688
689 /* if tx handler is seperate add 1 for every tx queue */
a99955fc
AD		 690 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
691 numvecs += adapter->num_tx_queues;
047e0030
AD		 692
693 /* store the number of vectors reserved for queues */
694 adapter->num_q_vectors = numvecs;
695
696 /* add 1 vector for link status interrupts */
697 numvecs++;
9d5c8243
AK		 698 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
699 GFP_KERNEL);
700 if (!adapter->msix_entries)
701 goto msi_only;
702
703 for (i = 0; i < numvecs; i++)
704 adapter->msix_entries[i].entry = i;
705
706 err = pci_enable_msix(adapter->pdev,
707 adapter->msix_entries,
708 numvecs);
709 if (err == 0)
34a20e89 710 goto out;
9d5c8243
AK		 711
712 igb_reset_interrupt_capability(adapter);
713
714 /* If we can't do MSI-X, try MSI */
715msi_only:
2a3abf6d
AD		 716#ifdef CONFIG_PCI_IOV
717 /* disable SR-IOV for non MSI-X configurations */
718 if (adapter->vf_data) {
719 struct e1000_hw *hw = &adapter->hw;
720 /* disable iov and allow time for transactions to clear */
721 pci_disable_sriov(adapter->pdev);
722 msleep(500);
723
724 kfree(adapter->vf_data);
725 adapter->vf_data = NULL;
726 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
727 msleep(100);
728 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
729 }
730#endif
4fc82adf 731 adapter->vfs_allocated_count = 0;
a99955fc 732 adapter->rss_queues = 1;
4fc82adf 733 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 734 adapter->num_rx_queues = 1;
661086df 735 adapter->num_tx_queues = 1;
047e0030 736 adapter->num_q_vectors = 1;
9d5c8243 737 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 738 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 739out:
661086df 740 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 741 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 742 return;
743}
744
047e0030
AD		 745/**
746 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
747 * @adapter: board private structure to initialize
748 *
749 * We allocate one q_vector per queue interrupt. If allocation fails we
750 * return -ENOMEM.
751 **/
752static int igb_alloc_q_vectors(struct igb_adapter *adapter)
753{
754 struct igb_q_vector *q_vector;
755 struct e1000_hw *hw = &adapter->hw;
756 int v_idx;
757
758 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
759 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
760 if (!q_vector)
761 goto err_out;
762 q_vector->adapter = adapter;
047e0030
AD		 763 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
764 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 765 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
766 adapter->q_vector[v_idx] = q_vector;
767 }
768 return 0;
769
770err_out:
771 while (v_idx) {
772 v_idx--;
773 q_vector = adapter->q_vector[v_idx];
774 netif_napi_del(&q_vector->napi);
775 kfree(q_vector);
776 adapter->q_vector[v_idx] = NULL;
777 }
778 return -ENOMEM;
779}
780
781static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
782 int ring_idx, int v_idx)
783{
3025a446 784 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 785
3025a446 786 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 787 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 788 q_vector->itr_val = adapter->rx_itr_setting;
789 if (q_vector->itr_val && q_vector->itr_val <= 3)
790 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 791}
792
793static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
794 int ring_idx, int v_idx)
795{
3025a446 796 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 797
3025a446 798 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 799 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 800 q_vector->itr_val = adapter->tx_itr_setting;
801 if (q_vector->itr_val && q_vector->itr_val <= 3)
802 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 803}
804
805/**
806 * igb_map_ring_to_vector - maps allocated queues to vectors
807 *
808 * This function maps the recently allocated queues to vectors.
809 **/
810static int igb_map_ring_to_vector(struct igb_adapter *adapter)
811{
812 int i;
813 int v_idx = 0;
814
815 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
816 (adapter->num_q_vectors < adapter->num_tx_queues))
817 return -ENOMEM;
818
819 if (adapter->num_q_vectors >=
820 (adapter->num_rx_queues + adapter->num_tx_queues)) {
821 for (i = 0; i < adapter->num_rx_queues; i++)
822 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
823 for (i = 0; i < adapter->num_tx_queues; i++)
824 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
825 } else {
826 for (i = 0; i < adapter->num_rx_queues; i++) {
827 if (i < adapter->num_tx_queues)
828 igb_map_tx_ring_to_vector(adapter, i, v_idx);
829 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
830 }
831 for (; i < adapter->num_tx_queues; i++)
832 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
833 }
834 return 0;
835}
836
837/**
838 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
839 *
840 * This function initializes the interrupts and allocates all of the queues.
841 **/
842static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
843{
844 struct pci_dev *pdev = adapter->pdev;
845 int err;
846
847 igb_set_interrupt_capability(adapter);
848
849 err = igb_alloc_q_vectors(adapter);
850 if (err) {
851 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
852 goto err_alloc_q_vectors;
853 }
854
855 err = igb_alloc_queues(adapter);
856 if (err) {
857 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
858 goto err_alloc_queues;
859 }
860
861 err = igb_map_ring_to_vector(adapter);
862 if (err) {
863 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
864 goto err_map_queues;
865 }
866
867
868 return 0;
869err_map_queues:
870 igb_free_queues(adapter);
871err_alloc_queues:
872 igb_free_q_vectors(adapter);
873err_alloc_q_vectors:
874 igb_reset_interrupt_capability(adapter);
875 return err;
876}
877
9d5c8243
AK		 878/**
879 * igb_request_irq - initialize interrupts
880 *
881 * Attempts to configure interrupts using the best available
882 * capabilities of the hardware and kernel.
883 **/
884static int igb_request_irq(struct igb_adapter *adapter)
885{
886 struct net_device *netdev = adapter->netdev;
047e0030 887 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 888 int err = 0;
889
890 if (adapter->msix_entries) {
891 err = igb_request_msix(adapter);
844290e5 892 if (!err)
9d5c8243 893 goto request_done;
9d5c8243 894 /* fall back to MSI */
047e0030 895 igb_clear_interrupt_scheme(adapter);
9d5c8243 896 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 897 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 898 igb_free_all_tx_resources(adapter);
899 igb_free_all_rx_resources(adapter);
047e0030 900 adapter->num_tx_queues = 1;
9d5c8243 901 adapter->num_rx_queues = 1;
047e0030
AD		 902 adapter->num_q_vectors = 1;
903 err = igb_alloc_q_vectors(adapter);
904 if (err) {
905 dev_err(&pdev->dev,
906 "Unable to allocate memory for vectors\n");
907 goto request_done;
908 }
909 err = igb_alloc_queues(adapter);
910 if (err) {
911 dev_err(&pdev->dev,
912 "Unable to allocate memory for queues\n");
913 igb_free_q_vectors(adapter);
914 goto request_done;
915 }
916 igb_setup_all_tx_resources(adapter);
917 igb_setup_all_rx_resources(adapter);
844290e5 918 } else {
feeb2721 919 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 920 }
844290e5 921
7dfc16fa 922 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 923 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 924 netdev->name, adapter);
9d5c8243
AK		 925 if (!err)
926 goto request_done;
047e0030 927
9d5c8243
AK		 928 /* fall back to legacy interrupts */
929 igb_reset_interrupt_capability(adapter);
7dfc16fa 930 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 931 }
932
a0607fd3 933 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 934 netdev->name, adapter);
9d5c8243 935
6cb5e577 936 if (err)
9d5c8243
AK		 937 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
938 err);
9d5c8243
AK		 939
940request_done:
941 return err;
942}
943
944static void igb_free_irq(struct igb_adapter *adapter)
945{
9d5c8243
AK		 946 if (adapter->msix_entries) {
947 int vector = 0, i;
948
047e0030 949 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 950
047e0030
AD		 951 for (i = 0; i < adapter->num_q_vectors; i++) {
952 struct igb_q_vector *q_vector = adapter->q_vector[i];
953 free_irq(adapter->msix_entries[vector++].vector,
954 q_vector);
955 }
956 } else {
957 free_irq(adapter->pdev->irq, adapter);
9d5c8243 958 }
9d5c8243
AK		 959}
960
961/**
962 * igb_irq_disable - Mask off interrupt generation on the NIC
963 * @adapter: board private structure
964 **/
965static void igb_irq_disable(struct igb_adapter *adapter)
966{
967 struct e1000_hw *hw = &adapter->hw;
968
25568a53
AD		 969 /*
970 * we need to be careful when disabling interrupts. The VFs are also
971 * mapped into these registers and so clearing the bits can cause
972 * issues on the VF drivers so we only need to clear what we set
973 */
9d5c8243 974 if (adapter->msix_entries) {
2dfd1212
AD		 975 u32 regval = rd32(E1000_EIAM);
976 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
977 wr32(E1000_EIMC, adapter->eims_enable_mask);
978 regval = rd32(E1000_EIAC);
979 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 980 }
844290e5
PW		 981
982 wr32(E1000_IAM, 0);
9d5c8243
AK		 983 wr32(E1000_IMC, ~0);
984 wrfl();
985 synchronize_irq(adapter->pdev->irq);
986}
987
988/**
989 * igb_irq_enable - Enable default interrupt generation settings
990 * @adapter: board private structure
991 **/
992static void igb_irq_enable(struct igb_adapter *adapter)
993{
994 struct e1000_hw *hw = &adapter->hw;
995
996 if (adapter->msix_entries) {
25568a53 997 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 998 u32 regval = rd32(E1000_EIAC);
999 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1000 regval = rd32(E1000_EIAM);
1001 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1002 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1003 if (adapter->vfs_allocated_count) {
4ae196df 1004 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1005 ims |= E1000_IMS_VMMB;
1006 }
55cac248
AD		 1007 if (adapter->hw.mac.type == e1000_82580)
1008 ims |= E1000_IMS_DRSTA;
1009
25568a53 1010 wr32(E1000_IMS, ims);
844290e5 1011 } else {
55cac248
AD		 1012 wr32(E1000_IMS, IMS_ENABLE_MASK |
1013 E1000_IMS_DRSTA);
1014 wr32(E1000_IAM, IMS_ENABLE_MASK |
1015 E1000_IMS_DRSTA);
844290e5 1016 }
9d5c8243
AK		 1017}
1018
1019static void igb_update_mng_vlan(struct igb_adapter *adapter)
1020{
51466239 1021 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1022 u16 vid = adapter->hw.mng_cookie.vlan_id;
1023 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1024
1025 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1026 /* add VID to filter table */
1027 igb_vfta_set(hw, vid, true);
1028 adapter->mng_vlan_id = vid;
1029 } else {
1030 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1031 }
1032
1033 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1034 (vid != old_vid) &&
1035 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1036 /* remove VID from filter table */
1037 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1038 }
1039}
1040
1041/**
1042 * igb_release_hw_control - release control of the h/w to f/w
1043 * @adapter: address of board private structure
1044 *
1045 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1046 * For ASF and Pass Through versions of f/w this means that the
1047 * driver is no longer loaded.
1048 *
1049 **/
1050static void igb_release_hw_control(struct igb_adapter *adapter)
1051{
1052 struct e1000_hw *hw = &adapter->hw;
1053 u32 ctrl_ext;
1054
1055 /* Let firmware take over control of h/w */
1056 ctrl_ext = rd32(E1000_CTRL_EXT);
1057 wr32(E1000_CTRL_EXT,
1058 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1059}
1060
9d5c8243
AK		 1061/**
1062 * igb_get_hw_control - get control of the h/w from f/w
1063 * @adapter: address of board private structure
1064 *
1065 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1066 * For ASF and Pass Through versions of f/w this means that
1067 * the driver is loaded.
1068 *
1069 **/
1070static void igb_get_hw_control(struct igb_adapter *adapter)
1071{
1072 struct e1000_hw *hw = &adapter->hw;
1073 u32 ctrl_ext;
1074
1075 /* Let firmware know the driver has taken over */
1076 ctrl_ext = rd32(E1000_CTRL_EXT);
1077 wr32(E1000_CTRL_EXT,
1078 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1079}
1080
9d5c8243
AK		 1081/**
1082 * igb_configure - configure the hardware for RX and TX
1083 * @adapter: private board structure
1084 **/
1085static void igb_configure(struct igb_adapter *adapter)
1086{
1087 struct net_device *netdev = adapter->netdev;
1088 int i;
1089
1090 igb_get_hw_control(adapter);
ff41f8dc 1091 igb_set_rx_mode(netdev);
9d5c8243
AK		 1092
1093 igb_restore_vlan(adapter);
9d5c8243 1094
85b430b4 1095 igb_setup_tctl(adapter);
06cf2666 1096 igb_setup_mrqc(adapter);
9d5c8243 1097 igb_setup_rctl(adapter);
85b430b4
AD		 1098
1099 igb_configure_tx(adapter);
9d5c8243 1100 igb_configure_rx(adapter);
662d7205
AD		 1101
1102 igb_rx_fifo_flush_82575(&adapter->hw);
1103
c493ea45 1104 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1105 * at least 1 descriptor unused to make sure
1106 * next_to_use != next_to_clean */
1107 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1108 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1109 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1110 }
1111
1112
1113 adapter->tx_queue_len = netdev->tx_queue_len;
1114}
1115
88a268c1
NN		 1116/**
1117 * igb_power_up_link - Power up the phy/serdes link
1118 * @adapter: address of board private structure
1119 **/
1120void igb_power_up_link(struct igb_adapter *adapter)
1121{
1122 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1123 igb_power_up_phy_copper(&adapter->hw);
1124 else
1125 igb_power_up_serdes_link_82575(&adapter->hw);
1126}
1127
1128/**
1129 * igb_power_down_link - Power down the phy/serdes link
1130 * @adapter: address of board private structure
1131 */
1132static void igb_power_down_link(struct igb_adapter *adapter)
1133{
1134 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1135 igb_power_down_phy_copper_82575(&adapter->hw);
1136 else
1137 igb_shutdown_serdes_link_82575(&adapter->hw);
1138}
9d5c8243
AK		 1139
1140/**
1141 * igb_up - Open the interface and prepare it to handle traffic
1142 * @adapter: board private structure
1143 **/
9d5c8243
AK		 1144int igb_up(struct igb_adapter *adapter)
1145{
1146 struct e1000_hw *hw = &adapter->hw;
1147 int i;
1148
1149 /* hardware has been reset, we need to reload some things */
1150 igb_configure(adapter);
1151
1152 clear_bit(__IGB_DOWN, &adapter->state);
1153
047e0030
AD		 1154 for (i = 0; i < adapter->num_q_vectors; i++) {
1155 struct igb_q_vector *q_vector = adapter->q_vector[i];
1156 napi_enable(&q_vector->napi);
1157 }
844290e5 1158 if (adapter->msix_entries)
9d5c8243 1159 igb_configure_msix(adapter);
feeb2721
AD		 1160 else
1161 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1162
1163 /* Clear any pending interrupts. */
1164 rd32(E1000_ICR);
1165 igb_irq_enable(adapter);
1166
d4960307
AD		 1167 /* notify VFs that reset has been completed */
1168 if (adapter->vfs_allocated_count) {
1169 u32 reg_data = rd32(E1000_CTRL_EXT);
1170 reg_data |= E1000_CTRL_EXT_PFRSTD;
1171 wr32(E1000_CTRL_EXT, reg_data);
1172 }
1173
4cb9be7a
JB		 1174 netif_tx_start_all_queues(adapter->netdev);
1175
25568a53
AD		 1176 /* start the watchdog. */
1177 hw->mac.get_link_status = 1;
1178 schedule_work(&adapter->watchdog_task);
1179
9d5c8243
AK		 1180 return 0;
1181}
1182
1183void igb_down(struct igb_adapter *adapter)
1184{
9d5c8243 1185 struct net_device *netdev = adapter->netdev;
330a6d6a 1186 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1187 u32 tctl, rctl;
1188 int i;
1189
1190 /* signal that we're down so the interrupt handler does not
1191 * reschedule our watchdog timer */
1192 set_bit(__IGB_DOWN, &adapter->state);
1193
1194 /* disable receives in the hardware */
1195 rctl = rd32(E1000_RCTL);
1196 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1197 /* flush and sleep below */
1198
fd2ea0a7 1199 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1200
1201 /* disable transmits in the hardware */
1202 tctl = rd32(E1000_TCTL);
1203 tctl &= ~E1000_TCTL_EN;
1204 wr32(E1000_TCTL, tctl);
1205 /* flush both disables and wait for them to finish */
1206 wrfl();
1207 msleep(10);
1208
047e0030
AD		 1209 for (i = 0; i < adapter->num_q_vectors; i++) {
1210 struct igb_q_vector *q_vector = adapter->q_vector[i];
1211 napi_disable(&q_vector->napi);
1212 }
9d5c8243 1213
9d5c8243
AK		 1214 igb_irq_disable(adapter);
1215
1216 del_timer_sync(&adapter->watchdog_timer);
1217 del_timer_sync(&adapter->phy_info_timer);
1218
1219 netdev->tx_queue_len = adapter->tx_queue_len;
1220 netif_carrier_off(netdev);
04fe6358
AD		 1221
1222 /* record the stats before reset*/
1223 igb_update_stats(adapter);
1224
9d5c8243
AK		 1225 adapter->link_speed = 0;
1226 adapter->link_duplex = 0;
1227
3023682e
JK		 1228 if (!pci_channel_offline(adapter->pdev))
1229 igb_reset(adapter);
9d5c8243
AK		 1230 igb_clean_all_tx_rings(adapter);
1231 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1232#ifdef CONFIG_IGB_DCA
1233
1234 /* since we reset the hardware DCA settings were cleared */
1235 igb_setup_dca(adapter);
1236#endif
9d5c8243
AK		 1237}
1238
1239void igb_reinit_locked(struct igb_adapter *adapter)
1240{
1241 WARN_ON(in_interrupt());
1242 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1243 msleep(1);
1244 igb_down(adapter);
1245 igb_up(adapter);
1246 clear_bit(__IGB_RESETTING, &adapter->state);
1247}
1248
1249void igb_reset(struct igb_adapter *adapter)
1250{
090b1795 1251 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1252 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1253 struct e1000_mac_info *mac = &hw->mac;
1254 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1255 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1256 u16 hwm;
1257
1258 /* Repartition Pba for greater than 9k mtu
1259 * To take effect CTRL.RST is required.
1260 */
fa4dfae0 1261 switch (mac->type) {
55cac248
AD		 1262 case e1000_82580:
1263 pba = rd32(E1000_RXPBS);
1264 pba = igb_rxpbs_adjust_82580(pba);
1265 break;
fa4dfae0 1266 case e1000_82576:
d249be54
AD		 1267 pba = rd32(E1000_RXPBS);
1268 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1269 break;
1270 case e1000_82575:
1271 default:
1272 pba = E1000_PBA_34K;
1273 break;
2d064c06 1274 }
9d5c8243 1275
2d064c06
AD		 1276 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1277 (mac->type < e1000_82576)) {
9d5c8243
AK		 1278 /* adjust PBA for jumbo frames */
1279 wr32(E1000_PBA, pba);
1280
1281 /* To maintain wire speed transmits, the Tx FIFO should be
1282 * large enough to accommodate two full transmit packets,
1283 * rounded up to the next 1KB and expressed in KB. Likewise,
1284 * the Rx FIFO should be large enough to accommodate at least
1285 * one full receive packet and is similarly rounded up and
1286 * expressed in KB. */
1287 pba = rd32(E1000_PBA);
1288 /* upper 16 bits has Tx packet buffer allocation size in KB */
1289 tx_space = pba >> 16;
1290 /* lower 16 bits has Rx packet buffer allocation size in KB */
1291 pba &= 0xffff;
1292 /* the tx fifo also stores 16 bytes of information about the tx
1293 * but don't include ethernet FCS because hardware appends it */
1294 min_tx_space = (adapter->max_frame_size +
85e8d004 1295 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1296 ETH_FCS_LEN) * 2;
1297 min_tx_space = ALIGN(min_tx_space, 1024);
1298 min_tx_space >>= 10;
1299 /* software strips receive CRC, so leave room for it */
1300 min_rx_space = adapter->max_frame_size;
1301 min_rx_space = ALIGN(min_rx_space, 1024);
1302 min_rx_space >>= 10;
1303
1304 /* If current Tx allocation is less than the min Tx FIFO size,
1305 * and the min Tx FIFO size is less than the current Rx FIFO
1306 * allocation, take space away from current Rx allocation */
1307 if (tx_space < min_tx_space &&
1308 ((min_tx_space - tx_space) < pba)) {
1309 pba = pba - (min_tx_space - tx_space);
1310
1311 /* if short on rx space, rx wins and must trump tx
1312 * adjustment */
1313 if (pba < min_rx_space)
1314 pba = min_rx_space;
1315 }
2d064c06 1316 wr32(E1000_PBA, pba);
9d5c8243 1317 }
9d5c8243
AK		 1318
1319 /* flow control settings */
1320 /* The high water mark must be low enough to fit one full frame
1321 * (or the size used for early receive) above it in the Rx FIFO.
1322 * Set it to the lower of:
1323 * - 90% of the Rx FIFO size, or
1324 * - the full Rx FIFO size minus one full frame */
1325 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1326 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1327
d405ea3e
AD		 1328 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1329 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1330 fc->pause_time = 0xFFFF;
1331 fc->send_xon = 1;
0cce119a 1332 fc->current_mode = fc->requested_mode;
9d5c8243 1333
4ae196df
AD		 1334 /* disable receive for all VFs and wait one second */
1335 if (adapter->vfs_allocated_count) {
1336 int i;
1337 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1338 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1339
1340 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1341 igb_ping_all_vfs(adapter);
4ae196df
AD		 1342
1343 /* disable transmits and receives */
1344 wr32(E1000_VFRE, 0);
1345 wr32(E1000_VFTE, 0);
1346 }
1347
9d5c8243 1348 /* Allow time for pending master requests to run */
330a6d6a 1349 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1350 wr32(E1000_WUC, 0);
1351
330a6d6a 1352 if (hw->mac.ops.init_hw(hw))
090b1795 1353 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1354
55cac248
AD		 1355 if (hw->mac.type == e1000_82580) {
1356 u32 reg = rd32(E1000_PCIEMISC);
1357 wr32(E1000_PCIEMISC,
1358 reg & ~E1000_PCIEMISC_LX_DECISION);
1359 }
88a268c1
NN		 1360 if (!netif_running(adapter->netdev))
1361 igb_power_down_link(adapter);
1362
9d5c8243
AK		 1363 igb_update_mng_vlan(adapter);
1364
1365 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1366 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1367
330a6d6a 1368 igb_get_phy_info(hw);
9d5c8243
AK		 1369}
1370
2e5c6922 1371static const struct net_device_ops igb_netdev_ops = {
559e9c49 1372 .ndo_open = igb_open,
2e5c6922 1373 .ndo_stop = igb_close,
00829823 1374 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1375 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1376 .ndo_set_rx_mode = igb_set_rx_mode,
1377 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1378 .ndo_set_mac_address = igb_set_mac,
1379 .ndo_change_mtu = igb_change_mtu,
1380 .ndo_do_ioctl = igb_ioctl,
1381 .ndo_tx_timeout = igb_tx_timeout,
1382 .ndo_validate_addr = eth_validate_addr,
1383 .ndo_vlan_rx_register = igb_vlan_rx_register,
1384 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1385 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1386 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1387 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1388 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1389 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1390#ifdef CONFIG_NET_POLL_CONTROLLER
1391 .ndo_poll_controller = igb_netpoll,
1392#endif
1393};
1394
9d5c8243
AK		 1395/**
1396 * igb_probe - Device Initialization Routine
1397 * @pdev: PCI device information struct
1398 * @ent: entry in igb_pci_tbl
1399 *
1400 * Returns 0 on success, negative on failure
1401 *
1402 * igb_probe initializes an adapter identified by a pci_dev structure.
1403 * The OS initialization, configuring of the adapter private structure,
1404 * and a hardware reset occur.
1405 **/
1406static int __devinit igb_probe(struct pci_dev *pdev,
1407 const struct pci_device_id *ent)
1408{
1409 struct net_device *netdev;
1410 struct igb_adapter *adapter;
1411 struct e1000_hw *hw;
4337e993
AD		 1412 u16 eeprom_data = 0;
1413 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1414 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1415 unsigned long mmio_start, mmio_len;
2d6a5e95 1416 int err, pci_using_dac;
9d5c8243
AK		 1417 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1418 u32 part_num;
1419
aed5dec3 1420 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1421 if (err)
1422 return err;
1423
1424 pci_using_dac = 0;
6a35528a 1425 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1426 if (!err) {
6a35528a 1427 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1428 if (!err)
1429 pci_using_dac = 1;
1430 } else {
284901a9 1431 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1432 if (err) {
284901a9 1433 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1434 if (err) {
1435 dev_err(&pdev->dev, "No usable DMA "
1436 "configuration, aborting\n");
1437 goto err_dma;
1438 }
1439 }
1440 }
1441
aed5dec3
AD		 1442 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1443 IORESOURCE_MEM),
1444 igb_driver_name);
9d5c8243
AK		 1445 if (err)
1446 goto err_pci_reg;
1447
19d5afd4 1448 pci_enable_pcie_error_reporting(pdev);
40a914fa 1449
9d5c8243 1450 pci_set_master(pdev);
c682fc23 1451 pci_save_state(pdev);
9d5c8243
AK		 1452
1453 err = -ENOMEM;
1bfaf07b
AD		 1454 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1455 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1456 if (!netdev)
1457 goto err_alloc_etherdev;
1458
1459 SET_NETDEV_DEV(netdev, &pdev->dev);
1460
1461 pci_set_drvdata(pdev, netdev);
1462 adapter = netdev_priv(netdev);
1463 adapter->netdev = netdev;
1464 adapter->pdev = pdev;
1465 hw = &adapter->hw;
1466 hw->back = adapter;
1467 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1468
1469 mmio_start = pci_resource_start(pdev, 0);
1470 mmio_len = pci_resource_len(pdev, 0);
1471
1472 err = -EIO;
28b0759c
AD		 1473 hw->hw_addr = ioremap(mmio_start, mmio_len);
1474 if (!hw->hw_addr)
9d5c8243
AK		 1475 goto err_ioremap;
1476
2e5c6922 1477 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1478 igb_set_ethtool_ops(netdev);
9d5c8243 1479 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1480
1481 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1482
1483 netdev->mem_start = mmio_start;
1484 netdev->mem_end = mmio_start + mmio_len;
1485
9d5c8243
AK		 1486 /* PCI config space info */
1487 hw->vendor_id = pdev->vendor;
1488 hw->device_id = pdev->device;
1489 hw->revision_id = pdev->revision;
1490 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1491 hw->subsystem_device_id = pdev->subsystem_device;
1492
9d5c8243
AK		 1493 /* Copy the default MAC, PHY and NVM function pointers */
1494 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1495 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1496 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1497 /* Initialize skew-specific constants */
1498 err = ei->get_invariants(hw);
1499 if (err)
450c87c8 1500 goto err_sw_init;
9d5c8243 1501
450c87c8 1502 /* setup the private structure */
9d5c8243
AK		 1503 err = igb_sw_init(adapter);
1504 if (err)
1505 goto err_sw_init;
1506
1507 igb_get_bus_info_pcie(hw);
1508
1509 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1510
1511 /* Copper options */
1512 if (hw->phy.media_type == e1000_media_type_copper) {
1513 hw->phy.mdix = AUTO_ALL_MODES;
1514 hw->phy.disable_polarity_correction = false;
1515 hw->phy.ms_type = e1000_ms_hw_default;
1516 }
1517
1518 if (igb_check_reset_block(hw))
1519 dev_info(&pdev->dev,
1520 "PHY reset is blocked due to SOL/IDER session.\n");
1521
1522 netdev->features = NETIF_F_SG |
7d8eb29e 1523 NETIF_F_IP_CSUM |
9d5c8243
AK		 1524 NETIF_F_HW_VLAN_TX |
1525 NETIF_F_HW_VLAN_RX |
1526 NETIF_F_HW_VLAN_FILTER;
1527
7d8eb29e 1528 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1529 netdev->features |= NETIF_F_TSO;
9d5c8243 1530 netdev->features |= NETIF_F_TSO6;
5c0999b7 1531 netdev->features |= NETIF_F_GRO;
d3352520 1532
48f29ffc
JK		 1533 netdev->vlan_features |= NETIF_F_TSO;
1534 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1535 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1536 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1537 netdev->vlan_features |= NETIF_F_SG;
1538
9d5c8243
AK		 1539 if (pci_using_dac)
1540 netdev->features |= NETIF_F_HIGHDMA;
1541
5b043fb0 1542 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1543 netdev->features |= NETIF_F_SCTP_CSUM;
1544
330a6d6a 1545 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1546
1547 /* before reading the NVM, reset the controller to put the device in a
1548 * known good starting state */
1549 hw->mac.ops.reset_hw(hw);
1550
1551 /* make sure the NVM is good */
1552 if (igb_validate_nvm_checksum(hw) < 0) {
1553 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1554 err = -EIO;
1555 goto err_eeprom;
1556 }
1557
1558 /* copy the MAC address out of the NVM */
1559 if (hw->mac.ops.read_mac_addr(hw))
1560 dev_err(&pdev->dev, "NVM Read Error\n");
1561
1562 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1563 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1564
1565 if (!is_valid_ether_addr(netdev->perm_addr)) {
1566 dev_err(&pdev->dev, "Invalid MAC Address\n");
1567 err = -EIO;
1568 goto err_eeprom;
1569 }
1570
0e340485
AD		 1571 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1572 (unsigned long) adapter);
1573 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1574 (unsigned long) adapter);
9d5c8243
AK		 1575
1576 INIT_WORK(&adapter->reset_task, igb_reset_task);
1577 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1578
450c87c8 1579 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1580 adapter->fc_autoneg = true;
1581 hw->mac.autoneg = true;
1582 hw->phy.autoneg_advertised = 0x2f;
1583
0cce119a
AD		 1584 hw->fc.requested_mode = e1000_fc_default;
1585 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1586
9d5c8243
AK		 1587 igb_validate_mdi_setting(hw);
1588
9d5c8243
AK		 1589 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1590 * enable the ACPI Magic Packet filter
1591 */
1592
a2cf8b6c 1593 if (hw->bus.func == 0)
312c75ae 1594 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1595 else if (hw->mac.type == e1000_82580)
1596 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1597 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1598 &eeprom_data);
a2cf8b6c
AD		 1599 else if (hw->bus.func == 1)
1600 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1601
1602 if (eeprom_data & eeprom_apme_mask)
1603 adapter->eeprom_wol |= E1000_WUFC_MAG;
1604
1605 /* now that we have the eeprom settings, apply the special cases where
1606 * the eeprom may be wrong or the board simply won't support wake on
1607 * lan on a particular port */
1608 switch (pdev->device) {
1609 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1610 adapter->eeprom_wol = 0;
1611 break;
1612 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1613 case E1000_DEV_ID_82576_FIBER:
1614 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1615 /* Wake events only supported on port A for dual fiber
1616 * regardless of eeprom setting */
1617 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1618 adapter->eeprom_wol = 0;
1619 break;
c8ea5ea9
AD		 1620 case E1000_DEV_ID_82576_QUAD_COPPER:
1621 /* if quad port adapter, disable WoL on all but port A */
1622 if (global_quad_port_a != 0)
1623 adapter->eeprom_wol = 0;
1624 else
1625 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1626 /* Reset for multiple quad port adapters */
1627 if (++global_quad_port_a == 4)
1628 global_quad_port_a = 0;
1629 break;
9d5c8243
AK		 1630 }
1631
1632 /* initialize the wol settings based on the eeprom settings */
1633 adapter->wol = adapter->eeprom_wol;
e1b86d84 1634 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1635
1636 /* reset the hardware with the new settings */
1637 igb_reset(adapter);
1638
1639 /* let the f/w know that the h/w is now under the control of the
1640 * driver. */
1641 igb_get_hw_control(adapter);
1642
9d5c8243
AK		 1643 strcpy(netdev->name, "eth%d");
1644 err = register_netdev(netdev);
1645 if (err)
1646 goto err_register;
1647
b168dfc5
JB		 1648 /* carrier off reporting is important to ethtool even BEFORE open */
1649 netif_carrier_off(netdev);
1650
421e02f0 1651#ifdef CONFIG_IGB_DCA
bbd98fe4 1652 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1653 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1654 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1655 igb_setup_dca(adapter);
1656 }
fe4506b6 1657
38c845c7 1658#endif
9d5c8243
AK		 1659 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1660 /* print bus type/speed/width info */
7c510e4b 1661 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1662 netdev->name,
559e9c49
AD		 1663 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1664 "unknown"),
59c3de89
AD		 1665 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1666 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1667 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1668 "unknown"),
7c510e4b 1669 netdev->dev_addr);
9d5c8243
AK		 1670
1671 igb_read_part_num(hw, &part_num);
1672 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1673 (part_num >> 8), (part_num & 0xff));
1674
1675 dev_info(&pdev->dev,
1676 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1677 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1678 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1679 adapter->num_rx_queues, adapter->num_tx_queues);
1680
9d5c8243
AK		 1681 return 0;
1682
1683err_register:
1684 igb_release_hw_control(adapter);
1685err_eeprom:
1686 if (!igb_check_reset_block(hw))
f5f4cf08 1687 igb_reset_phy(hw);
9d5c8243
AK		 1688
1689 if (hw->flash_address)
1690 iounmap(hw->flash_address);
9d5c8243 1691err_sw_init:
047e0030 1692 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1693 iounmap(hw->hw_addr);
1694err_ioremap:
1695 free_netdev(netdev);
1696err_alloc_etherdev:
559e9c49
AD		 1697 pci_release_selected_regions(pdev,
1698 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1699err_pci_reg:
1700err_dma:
1701 pci_disable_device(pdev);
1702 return err;
1703}
1704
1705/**
1706 * igb_remove - Device Removal Routine
1707 * @pdev: PCI device information struct
1708 *
1709 * igb_remove is called by the PCI subsystem to alert the driver
1710 * that it should release a PCI device. The could be caused by a
1711 * Hot-Plug event, or because the driver is going to be removed from
1712 * memory.
1713 **/
1714static void __devexit igb_remove(struct pci_dev *pdev)
1715{
1716 struct net_device *netdev = pci_get_drvdata(pdev);
1717 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1718 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1719
1720 /* flush_scheduled work may reschedule our watchdog task, so
1721 * explicitly disable watchdog tasks from being rescheduled */
1722 set_bit(__IGB_DOWN, &adapter->state);
1723 del_timer_sync(&adapter->watchdog_timer);
1724 del_timer_sync(&adapter->phy_info_timer);
1725
1726 flush_scheduled_work();
1727
421e02f0 1728#ifdef CONFIG_IGB_DCA
7dfc16fa 1729 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1730 dev_info(&pdev->dev, "DCA disabled\n");
1731 dca_remove_requester(&pdev->dev);
7dfc16fa 1732 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1733 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1734 }
1735#endif
1736
9d5c8243
AK		 1737 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1738 * would have already happened in close and is redundant. */
1739 igb_release_hw_control(adapter);
1740
1741 unregister_netdev(netdev);
1742
047e0030 1743 igb_clear_interrupt_scheme(adapter);
9d5c8243 1744
37680117
AD		 1745#ifdef CONFIG_PCI_IOV
1746 /* reclaim resources allocated to VFs */
1747 if (adapter->vf_data) {
1748 /* disable iov and allow time for transactions to clear */
1749 pci_disable_sriov(pdev);
1750 msleep(500);
1751
1752 kfree(adapter->vf_data);
1753 adapter->vf_data = NULL;
1754 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1755 msleep(100);
1756 dev_info(&pdev->dev, "IOV Disabled\n");
1757 }
1758#endif
559e9c49 1759
28b0759c
AD		 1760 iounmap(hw->hw_addr);
1761 if (hw->flash_address)
1762 iounmap(hw->flash_address);
559e9c49
AD		 1763 pci_release_selected_regions(pdev,
1764 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1765
1766 free_netdev(netdev);
1767
19d5afd4 1768 pci_disable_pcie_error_reporting(pdev);
40a914fa 1769
9d5c8243
AK		 1770 pci_disable_device(pdev);
1771}
1772
a6b623e0
AD		 1773/**
1774 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1775 * @adapter: board private structure to initialize
1776 *
1777 * This function initializes the vf specific data storage and then attempts to
1778 * allocate the VFs. The reason for ordering it this way is because it is much
1779 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1780 * the memory for the VFs.
1781 **/
1782static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1783{
1784#ifdef CONFIG_PCI_IOV
1785 struct pci_dev *pdev = adapter->pdev;
1786
1787 if (adapter->vfs_allocated_count > 7)
1788 adapter->vfs_allocated_count = 7;
1789
1790 if (adapter->vfs_allocated_count) {
1791 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1792 sizeof(struct vf_data_storage),
1793 GFP_KERNEL);
1794 /* if allocation failed then we do not support SR-IOV */
1795 if (!adapter->vf_data) {
1796 adapter->vfs_allocated_count = 0;
1797 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1798 "Data Storage\n");
1799 }
1800 }
1801
1802 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1803 kfree(adapter->vf_data);
1804 adapter->vf_data = NULL;
1805#endif /* CONFIG_PCI_IOV */
1806 adapter->vfs_allocated_count = 0;
1807#ifdef CONFIG_PCI_IOV
1808 } else {
1809 unsigned char mac_addr[ETH_ALEN];
1810 int i;
1811 dev_info(&pdev->dev, "%d vfs allocated\n",
1812 adapter->vfs_allocated_count);
1813 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1814 random_ether_addr(mac_addr);
1815 igb_set_vf_mac(adapter, i, mac_addr);
1816 }
1817 }
1818#endif /* CONFIG_PCI_IOV */
1819}
1820
115f459a
AD		 1821
1822/**
1823 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1824 * @adapter: board private structure to initialize
1825 *
1826 * igb_init_hw_timer initializes the function pointer and values for the hw
1827 * timer found in hardware.
1828 **/
1829static void igb_init_hw_timer(struct igb_adapter *adapter)
1830{
1831 struct e1000_hw *hw = &adapter->hw;
1832
1833 switch (hw->mac.type) {
55cac248
AD		 1834 case e1000_82580:
1835 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1836 adapter->cycles.read = igb_read_clock;
1837 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1838 adapter->cycles.mult = 1;
1839 /*
1840 * The 82580 timesync updates the system timer every 8ns by 8ns
1841 * and the value cannot be shifted. Instead we need to shift
1842 * the registers to generate a 64bit timer value. As a result
1843 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1844 * 24 in order to generate a larger value for synchronization.
1845 */
1846 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1847 /* disable system timer temporarily by setting bit 31 */
1848 wr32(E1000_TSAUXC, 0x80000000);
1849 wrfl();
1850
1851 /* Set registers so that rollover occurs soon to test this. */
1852 wr32(E1000_SYSTIMR, 0x00000000);
1853 wr32(E1000_SYSTIML, 0x80000000);
1854 wr32(E1000_SYSTIMH, 0x000000FF);
1855 wrfl();
1856
1857 /* enable system timer by clearing bit 31 */
1858 wr32(E1000_TSAUXC, 0x0);
1859 wrfl();
1860
1861 timecounter_init(&adapter->clock,
1862 &adapter->cycles,
1863 ktime_to_ns(ktime_get_real()));
1864 /*
1865 * Synchronize our NIC clock against system wall clock. NIC
1866 * time stamp reading requires ~3us per sample, each sample
1867 * was pretty stable even under load => only require 10
1868 * samples for each offset comparison.
1869 */
1870 memset(&adapter->compare, 0, sizeof(adapter->compare));
1871 adapter->compare.source = &adapter->clock;
1872 adapter->compare.target = ktime_get_real;
1873 adapter->compare.num_samples = 10;
1874 timecompare_update(&adapter->compare, 0);
1875 break;
115f459a
AD		 1876 case e1000_82576:
1877 /*
1878 * Initialize hardware timer: we keep it running just in case
1879 * that some program needs it later on.
1880 */
1881 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1882 adapter->cycles.read = igb_read_clock;
1883 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1884 adapter->cycles.mult = 1;
1885 /**
1886 * Scale the NIC clock cycle by a large factor so that
1887 * relatively small clock corrections can be added or
1888 * substracted at each clock tick. The drawbacks of a large
1889 * factor are a) that the clock register overflows more quickly
1890 * (not such a big deal) and b) that the increment per tick has
1891 * to fit into 24 bits. As a result we need to use a shift of
1892 * 19 so we can fit a value of 16 into the TIMINCA register.
1893 */
1894 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1895 wr32(E1000_TIMINCA,
1896 (1 << E1000_TIMINCA_16NS_SHIFT) |
1897 (16 << IGB_82576_TSYNC_SHIFT));
1898
1899 /* Set registers so that rollover occurs soon to test this. */
1900 wr32(E1000_SYSTIML, 0x00000000);
1901 wr32(E1000_SYSTIMH, 0xFF800000);
1902 wrfl();
1903
1904 timecounter_init(&adapter->clock,
1905 &adapter->cycles,
1906 ktime_to_ns(ktime_get_real()));
1907 /*
1908 * Synchronize our NIC clock against system wall clock. NIC
1909 * time stamp reading requires ~3us per sample, each sample
1910 * was pretty stable even under load => only require 10
1911 * samples for each offset comparison.
1912 */
1913 memset(&adapter->compare, 0, sizeof(adapter->compare));
1914 adapter->compare.source = &adapter->clock;
1915 adapter->compare.target = ktime_get_real;
1916 adapter->compare.num_samples = 10;
1917 timecompare_update(&adapter->compare, 0);
1918 break;
1919 case e1000_82575:
1920 /* 82575 does not support timesync */
1921 default:
1922 break;
1923 }
1924
1925}
1926
9d5c8243
AK		 1927/**
1928 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1929 * @adapter: board private structure to initialize
1930 *
1931 * igb_sw_init initializes the Adapter private data structure.
1932 * Fields are initialized based on PCI device information and
1933 * OS network device settings (MTU size).
1934 **/
1935static int __devinit igb_sw_init(struct igb_adapter *adapter)
1936{
1937 struct e1000_hw *hw = &adapter->hw;
1938 struct net_device *netdev = adapter->netdev;
1939 struct pci_dev *pdev = adapter->pdev;
1940
1941 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1942
68fd9910
AD		 1943 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1944 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1945 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1946 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1947
9d5c8243
AK		 1948 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1949 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1950
a6b623e0
AD		 1951#ifdef CONFIG_PCI_IOV
1952 if (hw->mac.type == e1000_82576)
1953 adapter->vfs_allocated_count = max_vfs;
1954
1955#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 1956 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1957
1958 /*
1959 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1960 * then we should combine the queues into a queue pair in order to
1961 * conserve interrupts due to limited supply
1962 */
1963 if ((adapter->rss_queues > 4) ||
1964 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1965 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1966
a6b623e0 1967 /* This call may decrease the number of queues */
047e0030 1968 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1969 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1970 return -ENOMEM;
1971 }
1972
115f459a 1973 igb_init_hw_timer(adapter);
a6b623e0
AD		 1974 igb_probe_vfs(adapter);
1975
9d5c8243
AK		 1976 /* Explicitly disable IRQ since the NIC can be in any state. */
1977 igb_irq_disable(adapter);
1978
1979 set_bit(__IGB_DOWN, &adapter->state);
1980 return 0;
1981}
1982
1983/**
1984 * igb_open - Called when a network interface is made active
1985 * @netdev: network interface device structure
1986 *
1987 * Returns 0 on success, negative value on failure
1988 *
1989 * The open entry point is called when a network interface is made
1990 * active by the system (IFF_UP). At this point all resources needed
1991 * for transmit and receive operations are allocated, the interrupt
1992 * handler is registered with the OS, the watchdog timer is started,
1993 * and the stack is notified that the interface is ready.
1994 **/
1995static int igb_open(struct net_device *netdev)
1996{
1997 struct igb_adapter *adapter = netdev_priv(netdev);
1998 struct e1000_hw *hw = &adapter->hw;
1999 int err;
2000 int i;
2001
2002 /* disallow open during test */
2003 if (test_bit(__IGB_TESTING, &adapter->state))
2004 return -EBUSY;
2005
b168dfc5
JB		 2006 netif_carrier_off(netdev);
2007
9d5c8243
AK		 2008 /* allocate transmit descriptors */
2009 err = igb_setup_all_tx_resources(adapter);
2010 if (err)
2011 goto err_setup_tx;
2012
2013 /* allocate receive descriptors */
2014 err = igb_setup_all_rx_resources(adapter);
2015 if (err)
2016 goto err_setup_rx;
2017
88a268c1 2018 igb_power_up_link(adapter);
9d5c8243 2019
9d5c8243
AK		 2020 /* before we allocate an interrupt, we must be ready to handle it.
2021 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2022 * as soon as we call pci_request_irq, so we have to setup our
2023 * clean_rx handler before we do so. */
2024 igb_configure(adapter);
2025
2026 err = igb_request_irq(adapter);
2027 if (err)
2028 goto err_req_irq;
2029
2030 /* From here on the code is the same as igb_up() */
2031 clear_bit(__IGB_DOWN, &adapter->state);
2032
047e0030
AD		 2033 for (i = 0; i < adapter->num_q_vectors; i++) {
2034 struct igb_q_vector *q_vector = adapter->q_vector[i];
2035 napi_enable(&q_vector->napi);
2036 }
9d5c8243
AK		 2037
2038 /* Clear any pending interrupts. */
2039 rd32(E1000_ICR);
844290e5
PW		 2040
2041 igb_irq_enable(adapter);
2042
d4960307
AD		 2043 /* notify VFs that reset has been completed */
2044 if (adapter->vfs_allocated_count) {
2045 u32 reg_data = rd32(E1000_CTRL_EXT);
2046 reg_data |= E1000_CTRL_EXT_PFRSTD;
2047 wr32(E1000_CTRL_EXT, reg_data);
2048 }
2049
d55b53ff
JK		 2050 netif_tx_start_all_queues(netdev);
2051
25568a53
AD		 2052 /* start the watchdog. */
2053 hw->mac.get_link_status = 1;
2054 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2055
2056 return 0;
2057
2058err_req_irq:
2059 igb_release_hw_control(adapter);
88a268c1 2060 igb_power_down_link(adapter);
9d5c8243
AK		 2061 igb_free_all_rx_resources(adapter);
2062err_setup_rx:
2063 igb_free_all_tx_resources(adapter);
2064err_setup_tx:
2065 igb_reset(adapter);
2066
2067 return err;
2068}
2069
2070/**
2071 * igb_close - Disables a network interface
2072 * @netdev: network interface device structure
2073 *
2074 * Returns 0, this is not allowed to fail
2075 *
2076 * The close entry point is called when an interface is de-activated
2077 * by the OS. The hardware is still under the driver's control, but
2078 * needs to be disabled. A global MAC reset is issued to stop the
2079 * hardware, and all transmit and receive resources are freed.
2080 **/
2081static int igb_close(struct net_device *netdev)
2082{
2083 struct igb_adapter *adapter = netdev_priv(netdev);
2084
2085 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2086 igb_down(adapter);
2087
2088 igb_free_irq(adapter);
2089
2090 igb_free_all_tx_resources(adapter);
2091 igb_free_all_rx_resources(adapter);
2092
9d5c8243
AK		 2093 return 0;
2094}
2095
2096/**
2097 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2098 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2099 *
2100 * Return 0 on success, negative on failure
2101 **/
80785298 2102int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2103{
80785298 2104 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2105 int size;
2106
2107 size = sizeof(struct igb_buffer) * tx_ring->count;
2108 tx_ring->buffer_info = vmalloc(size);
2109 if (!tx_ring->buffer_info)
2110 goto err;
2111 memset(tx_ring->buffer_info, 0, size);
2112
2113 /* round up to nearest 4K */
85e8d004 2114 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2115 tx_ring->size = ALIGN(tx_ring->size, 4096);
2116
439705e1
AD		 2117 tx_ring->desc = pci_alloc_consistent(pdev,
2118 tx_ring->size,
9d5c8243
AK		 2119 &tx_ring->dma);
2120
2121 if (!tx_ring->desc)
2122 goto err;
2123
9d5c8243
AK		 2124 tx_ring->next_to_use = 0;
2125 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2126 return 0;
2127
2128err:
2129 vfree(tx_ring->buffer_info);
047e0030 2130 dev_err(&pdev->dev,
9d5c8243
AK		 2131 "Unable to allocate memory for the transmit descriptor ring\n");
2132 return -ENOMEM;
2133}
2134
2135/**
2136 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2137 * (Descriptors) for all queues
2138 * @adapter: board private structure
2139 *
2140 * Return 0 on success, negative on failure
2141 **/
2142static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2143{
439705e1 2144 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2145 int i, err = 0;
2146
2147 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2148 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2149 if (err) {
439705e1 2150 dev_err(&pdev->dev,
9d5c8243
AK		 2151 "Allocation for Tx Queue %u failed\n", i);
2152 for (i--; i >= 0; i--)
3025a446 2153 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2154 break;
2155 }
2156 }
2157
a99955fc 2158 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2159 int r_idx = i % adapter->num_tx_queues;
3025a446 2160 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2161 }
9d5c8243
AK		 2162 return err;
2163}
2164
2165/**
85b430b4
AD		 2166 * igb_setup_tctl - configure the transmit control registers
2167 * @adapter: Board private structure
9d5c8243 2168 **/
d7ee5b3a 2169void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2170{
9d5c8243
AK		 2171 struct e1000_hw *hw = &adapter->hw;
2172 u32 tctl;
9d5c8243 2173
85b430b4
AD		 2174 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2175 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2176
2177 /* Program the Transmit Control Register */
9d5c8243
AK		 2178 tctl = rd32(E1000_TCTL);
2179 tctl &= ~E1000_TCTL_CT;
2180 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2181 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2182
2183 igb_config_collision_dist(hw);
2184
9d5c8243
AK		 2185 /* Enable transmits */
2186 tctl |= E1000_TCTL_EN;
2187
2188 wr32(E1000_TCTL, tctl);
2189}
2190
85b430b4
AD		 2191/**
2192 * igb_configure_tx_ring - Configure transmit ring after Reset
2193 * @adapter: board private structure
2194 * @ring: tx ring to configure
2195 *
2196 * Configure a transmit ring after a reset.
2197 **/
d7ee5b3a
AD		 2198void igb_configure_tx_ring(struct igb_adapter *adapter,
2199 struct igb_ring *ring)
85b430b4
AD		 2200{
2201 struct e1000_hw *hw = &adapter->hw;
2202 u32 txdctl;
2203 u64 tdba = ring->dma;
2204 int reg_idx = ring->reg_idx;
2205
2206 /* disable the queue */
2207 txdctl = rd32(E1000_TXDCTL(reg_idx));
2208 wr32(E1000_TXDCTL(reg_idx),
2209 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2210 wrfl();
2211 mdelay(10);
2212
2213 wr32(E1000_TDLEN(reg_idx),
2214 ring->count * sizeof(union e1000_adv_tx_desc));
2215 wr32(E1000_TDBAL(reg_idx),
2216 tdba & 0x00000000ffffffffULL);
2217 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2218
fce99e34
AD		 2219 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2220 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2221 writel(0, ring->head);
2222 writel(0, ring->tail);
85b430b4
AD		 2223
2224 txdctl |= IGB_TX_PTHRESH;
2225 txdctl |= IGB_TX_HTHRESH << 8;
2226 txdctl |= IGB_TX_WTHRESH << 16;
2227
2228 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2229 wr32(E1000_TXDCTL(reg_idx), txdctl);
2230}
2231
2232/**
2233 * igb_configure_tx - Configure transmit Unit after Reset
2234 * @adapter: board private structure
2235 *
2236 * Configure the Tx unit of the MAC after a reset.
2237 **/
2238static void igb_configure_tx(struct igb_adapter *adapter)
2239{
2240 int i;
2241
2242 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2243 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2244}
2245
9d5c8243
AK		 2246/**
2247 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2248 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2249 *
2250 * Returns 0 on success, negative on failure
2251 **/
80785298 2252int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2253{
80785298 2254 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2255 int size, desc_len;
2256
2257 size = sizeof(struct igb_buffer) * rx_ring->count;
2258 rx_ring->buffer_info = vmalloc(size);
2259 if (!rx_ring->buffer_info)
2260 goto err;
2261 memset(rx_ring->buffer_info, 0, size);
2262
2263 desc_len = sizeof(union e1000_adv_rx_desc);
2264
2265 /* Round up to nearest 4K */
2266 rx_ring->size = rx_ring->count * desc_len;
2267 rx_ring->size = ALIGN(rx_ring->size, 4096);
2268
2269 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2270 &rx_ring->dma);
2271
2272 if (!rx_ring->desc)
2273 goto err;
2274
2275 rx_ring->next_to_clean = 0;
2276 rx_ring->next_to_use = 0;
9d5c8243 2277
9d5c8243
AK		 2278 return 0;
2279
2280err:
2281 vfree(rx_ring->buffer_info);
439705e1 2282 rx_ring->buffer_info = NULL;
80785298 2283 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2284 "the receive descriptor ring\n");
2285 return -ENOMEM;
2286}
2287
2288/**
2289 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2290 * (Descriptors) for all queues
2291 * @adapter: board private structure
2292 *
2293 * Return 0 on success, negative on failure
2294 **/
2295static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2296{
439705e1 2297 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2298 int i, err = 0;
2299
2300 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2301 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2302 if (err) {
439705e1 2303 dev_err(&pdev->dev,
9d5c8243
AK		 2304 "Allocation for Rx Queue %u failed\n", i);
2305 for (i--; i >= 0; i--)
3025a446 2306 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2307 break;
2308 }
2309 }
2310
2311 return err;
2312}
2313
06cf2666
AD		 2314/**
2315 * igb_setup_mrqc - configure the multiple receive queue control registers
2316 * @adapter: Board private structure
2317 **/
2318static void igb_setup_mrqc(struct igb_adapter *adapter)
2319{
2320 struct e1000_hw *hw = &adapter->hw;
2321 u32 mrqc, rxcsum;
2322 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2323 union e1000_reta {
2324 u32 dword;
2325 u8 bytes[4];
2326 } reta;
2327 static const u8 rsshash[40] = {
2328 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2329 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2330 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2331 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2332
2333 /* Fill out hash function seeds */
2334 for (j = 0; j < 10; j++) {
2335 u32 rsskey = rsshash[(j * 4)];
2336 rsskey |= rsshash[(j * 4) + 1] << 8;
2337 rsskey |= rsshash[(j * 4) + 2] << 16;
2338 rsskey |= rsshash[(j * 4) + 3] << 24;
2339 array_wr32(E1000_RSSRK(0), j, rsskey);
2340 }
2341
a99955fc 2342 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2343
2344 if (adapter->vfs_allocated_count) {
2345 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2346 switch (hw->mac.type) {
55cac248
AD		 2347 case e1000_82580:
2348 num_rx_queues = 1;
2349 shift = 0;
2350 break;
06cf2666
AD		 2351 case e1000_82576:
2352 shift = 3;
2353 num_rx_queues = 2;
2354 break;
2355 case e1000_82575:
2356 shift = 2;
2357 shift2 = 6;
2358 default:
2359 break;
2360 }
2361 } else {
2362 if (hw->mac.type == e1000_82575)
2363 shift = 6;
2364 }
2365
2366 for (j = 0; j < (32 * 4); j++) {
2367 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2368 if (shift2)
2369 reta.bytes[j & 3] |= num_rx_queues << shift2;
2370 if ((j & 3) == 3)
2371 wr32(E1000_RETA(j >> 2), reta.dword);
2372 }
2373
2374 /*
2375 * Disable raw packet checksumming so that RSS hash is placed in
2376 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2377 * offloads as they are enabled by default
2378 */
2379 rxcsum = rd32(E1000_RXCSUM);
2380 rxcsum |= E1000_RXCSUM_PCSD;
2381
2382 if (adapter->hw.mac.type >= e1000_82576)
2383 /* Enable Receive Checksum Offload for SCTP */
2384 rxcsum |= E1000_RXCSUM_CRCOFL;
2385
2386 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2387 wr32(E1000_RXCSUM, rxcsum);
2388
2389 /* If VMDq is enabled then we set the appropriate mode for that, else
2390 * we default to RSS so that an RSS hash is calculated per packet even
2391 * if we are only using one queue */
2392 if (adapter->vfs_allocated_count) {
2393 if (hw->mac.type > e1000_82575) {
2394 /* Set the default pool for the PF's first queue */
2395 u32 vtctl = rd32(E1000_VT_CTL);
2396 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2397 E1000_VT_CTL_DISABLE_DEF_POOL);
2398 vtctl |= adapter->vfs_allocated_count <<
2399 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2400 wr32(E1000_VT_CTL, vtctl);
2401 }
a99955fc 2402 if (adapter->rss_queues > 1)
06cf2666
AD		 2403 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2404 else
2405 mrqc = E1000_MRQC_ENABLE_VMDQ;
2406 } else {
2407 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2408 }
2409 igb_vmm_control(adapter);
2410
2411 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2412 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2413 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2414 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2415 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2416 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2417 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2418 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2419
2420 wr32(E1000_MRQC, mrqc);
2421}
2422
9d5c8243
AK		 2423/**
2424 * igb_setup_rctl - configure the receive control registers
2425 * @adapter: Board private structure
2426 **/
d7ee5b3a 2427void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2428{
2429 struct e1000_hw *hw = &adapter->hw;
2430 u32 rctl;
9d5c8243
AK		 2431
2432 rctl = rd32(E1000_RCTL);
2433
2434 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2435 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2436
69d728ba 2437 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2438 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2439
87cb7e8c
AK		 2440 /*
2441 * enable stripping of CRC. It's unlikely this will break BMC
2442 * redirection as it did with e1000. Newer features require
2443 * that the HW strips the CRC.
73cd78f1 2444 */
87cb7e8c 2445 rctl |= E1000_RCTL_SECRC;
9d5c8243 2446
559e9c49 2447 /* disable store bad packets and clear size bits. */
ec54d7d6 2448 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2449
6ec43fe6
AD		 2450 /* enable LPE to prevent packets larger than max_frame_size */
2451 rctl |= E1000_RCTL_LPE;
9d5c8243 2452
952f72a8
AD		 2453 /* disable queue 0 to prevent tail write w/o re-config */
2454 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2455
e1739522
AD		 2456 /* Attention!!! For SR-IOV PF driver operations you must enable
2457 * queue drop for all VF and PF queues to prevent head of line blocking
2458 * if an un-trusted VF does not provide descriptors to hardware.
2459 */
2460 if (adapter->vfs_allocated_count) {
e1739522
AD		 2461 /* set all queue drop enable bits */
2462 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2463 }
2464
9d5c8243
AK		 2465 wr32(E1000_RCTL, rctl);
2466}
2467
7d5753f0
AD		 2468static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2469 int vfn)
2470{
2471 struct e1000_hw *hw = &adapter->hw;
2472 u32 vmolr;
2473
2474 /* if it isn't the PF check to see if VFs are enabled and
2475 * increase the size to support vlan tags */
2476 if (vfn < adapter->vfs_allocated_count &&
2477 adapter->vf_data[vfn].vlans_enabled)
2478 size += VLAN_TAG_SIZE;
2479
2480 vmolr = rd32(E1000_VMOLR(vfn));
2481 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2482 vmolr |= size | E1000_VMOLR_LPE;
2483 wr32(E1000_VMOLR(vfn), vmolr);
2484
2485 return 0;
2486}
2487
e1739522
AD		 2488/**
2489 * igb_rlpml_set - set maximum receive packet size
2490 * @adapter: board private structure
2491 *
2492 * Configure maximum receivable packet size.
2493 **/
2494static void igb_rlpml_set(struct igb_adapter *adapter)
2495{
2496 u32 max_frame_size = adapter->max_frame_size;
2497 struct e1000_hw *hw = &adapter->hw;
2498 u16 pf_id = adapter->vfs_allocated_count;
2499
2500 if (adapter->vlgrp)
2501 max_frame_size += VLAN_TAG_SIZE;
2502
2503 /* if vfs are enabled we set RLPML to the largest possible request
2504 * size and set the VMOLR RLPML to the size we need */
2505 if (pf_id) {
2506 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2507 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2508 }
2509
2510 wr32(E1000_RLPML, max_frame_size);
2511}
2512
8151d294
WM		 2513static inline void igb_set_vmolr(struct igb_adapter *adapter,
2514 int vfn, bool aupe)
7d5753f0
AD		 2515{
2516 struct e1000_hw *hw = &adapter->hw;
2517 u32 vmolr;
2518
2519 /*
2520 * This register exists only on 82576 and newer so if we are older then
2521 * we should exit and do nothing
2522 */
2523 if (hw->mac.type < e1000_82576)
2524 return;
2525
2526 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2527 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2528 if (aupe)
2529 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2530 else
2531 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2532
2533 /* clear all bits that might not be set */
2534 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2535
a99955fc 2536 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2537 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2538 /*
2539 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2540 * multicast packets
2541 */
2542 if (vfn <= adapter->vfs_allocated_count)
2543 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2544
2545 wr32(E1000_VMOLR(vfn), vmolr);
2546}
2547
85b430b4
AD		 2548/**
2549 * igb_configure_rx_ring - Configure a receive ring after Reset
2550 * @adapter: board private structure
2551 * @ring: receive ring to be configured
2552 *
2553 * Configure the Rx unit of the MAC after a reset.
2554 **/
d7ee5b3a
AD		 2555void igb_configure_rx_ring(struct igb_adapter *adapter,
2556 struct igb_ring *ring)
85b430b4
AD		 2557{
2558 struct e1000_hw *hw = &adapter->hw;
2559 u64 rdba = ring->dma;
2560 int reg_idx = ring->reg_idx;
952f72a8 2561 u32 srrctl, rxdctl;
85b430b4
AD		 2562
2563 /* disable the queue */
2564 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2565 wr32(E1000_RXDCTL(reg_idx),
2566 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2567
2568 /* Set DMA base address registers */
2569 wr32(E1000_RDBAL(reg_idx),
2570 rdba & 0x00000000ffffffffULL);
2571 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2572 wr32(E1000_RDLEN(reg_idx),
2573 ring->count * sizeof(union e1000_adv_rx_desc));
2574
2575 /* initialize head and tail */
fce99e34
AD		 2576 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2577 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2578 writel(0, ring->head);
2579 writel(0, ring->tail);
85b430b4 2580
952f72a8 2581 /* set descriptor configuration */
4c844851
AD		 2582 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2583 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2584 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2585#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2586 srrctl |= IGB_RXBUFFER_16384 >>
2587 E1000_SRRCTL_BSIZEPKT_SHIFT;
2588#else
2589 srrctl |= (PAGE_SIZE / 2) >>
2590 E1000_SRRCTL_BSIZEPKT_SHIFT;
2591#endif
2592 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2593 } else {
4c844851 2594 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2595 E1000_SRRCTL_BSIZEPKT_SHIFT;
2596 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2597 }
e6bdb6fe
NN		 2598 /* Only set Drop Enable if we are supporting multiple queues */
2599 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2600 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2601
2602 wr32(E1000_SRRCTL(reg_idx), srrctl);
2603
7d5753f0 2604 /* set filtering for VMDQ pools */
8151d294 2605 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2606
85b430b4
AD		 2607 /* enable receive descriptor fetching */
2608 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2609 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2610 rxdctl &= 0xFFF00000;
2611 rxdctl |= IGB_RX_PTHRESH;
2612 rxdctl |= IGB_RX_HTHRESH << 8;
2613 rxdctl |= IGB_RX_WTHRESH << 16;
2614 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2615}
2616
9d5c8243
AK		 2617/**
2618 * igb_configure_rx - Configure receive Unit after Reset
2619 * @adapter: board private structure
2620 *
2621 * Configure the Rx unit of the MAC after a reset.
2622 **/
2623static void igb_configure_rx(struct igb_adapter *adapter)
2624{
9107584e 2625 int i;
9d5c8243 2626
68d480c4
AD		 2627 /* set UTA to appropriate mode */
2628 igb_set_uta(adapter);
2629
26ad9178
AD		 2630 /* set the correct pool for the PF default MAC address in entry 0 */
2631 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2632 adapter->vfs_allocated_count);
2633
06cf2666
AD		 2634 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2635 * the Base and Length of the Rx Descriptor Ring */
2636 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2637 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2638}
2639
2640/**
2641 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2642 * @tx_ring: Tx descriptor ring for a specific queue
2643 *
2644 * Free all transmit software resources
2645 **/
68fd9910 2646void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2647{
3b644cf6 2648 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2649
2650 vfree(tx_ring->buffer_info);
2651 tx_ring->buffer_info = NULL;
2652
439705e1
AD		 2653 /* if not set, then don't free */
2654 if (!tx_ring->desc)
2655 return;
2656
80785298
AD		 2657 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2658 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2659
2660 tx_ring->desc = NULL;
2661}
2662
2663/**
2664 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2665 * @adapter: board private structure
2666 *
2667 * Free all transmit software resources
2668 **/
2669static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2670{
2671 int i;
2672
2673 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2674 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2675}
2676
b1a436c3
AD		 2677void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2678 struct igb_buffer *buffer_info)
9d5c8243 2679{
6366ad33
AD		 2680 if (buffer_info->dma) {
2681 if (buffer_info->mapped_as_page)
2682 pci_unmap_page(tx_ring->pdev,
2683 buffer_info->dma,
2684 buffer_info->length,
2685 PCI_DMA_TODEVICE);
2686 else
2687 pci_unmap_single(tx_ring->pdev,
2688 buffer_info->dma,
2689 buffer_info->length,
2690 PCI_DMA_TODEVICE);
2691 buffer_info->dma = 0;
2692 }
9d5c8243
AK		 2693 if (buffer_info->skb) {
2694 dev_kfree_skb_any(buffer_info->skb);
2695 buffer_info->skb = NULL;
2696 }
2697 buffer_info->time_stamp = 0;
6366ad33
AD		 2698 buffer_info->length = 0;
2699 buffer_info->next_to_watch = 0;
2700 buffer_info->mapped_as_page = false;
9d5c8243
AK		 2701}
2702
2703/**
2704 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2705 * @tx_ring: ring to be cleaned
2706 **/
3b644cf6 2707static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2708{
2709 struct igb_buffer *buffer_info;
2710 unsigned long size;
2711 unsigned int i;
2712
2713 if (!tx_ring->buffer_info)
2714 return;
2715 /* Free all the Tx ring sk_buffs */
2716
2717 for (i = 0; i < tx_ring->count; i++) {
2718 buffer_info = &tx_ring->buffer_info[i];
80785298 2719 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2720 }
2721
2722 size = sizeof(struct igb_buffer) * tx_ring->count;
2723 memset(tx_ring->buffer_info, 0, size);
2724
2725 /* Zero out the descriptor ring */
9d5c8243
AK		 2726 memset(tx_ring->desc, 0, tx_ring->size);
2727
2728 tx_ring->next_to_use = 0;
2729 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2730}
2731
2732/**
2733 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2734 * @adapter: board private structure
2735 **/
2736static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2737{
2738 int i;
2739
2740 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2741 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 2742}
2743
2744/**
2745 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2746 * @rx_ring: ring to clean the resources from
2747 *
2748 * Free all receive software resources
2749 **/
68fd9910 2750void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2751{
3b644cf6 2752 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2753
2754 vfree(rx_ring->buffer_info);
2755 rx_ring->buffer_info = NULL;
2756
439705e1
AD		 2757 /* if not set, then don't free */
2758 if (!rx_ring->desc)
2759 return;
2760
80785298
AD		 2761 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2762 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2763
2764 rx_ring->desc = NULL;
2765}
2766
2767/**
2768 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2769 * @adapter: board private structure
2770 *
2771 * Free all receive software resources
2772 **/
2773static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2774{
2775 int i;
2776
2777 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2778 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2779}
2780
2781/**
2782 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2783 * @rx_ring: ring to free buffers from
2784 **/
3b644cf6 2785static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2786{
2787 struct igb_buffer *buffer_info;
9d5c8243
AK		 2788 unsigned long size;
2789 unsigned int i;
2790
2791 if (!rx_ring->buffer_info)
2792 return;
439705e1 2793
9d5c8243
AK		 2794 /* Free all the Rx ring sk_buffs */
2795 for (i = 0; i < rx_ring->count; i++) {
2796 buffer_info = &rx_ring->buffer_info[i];
2797 if (buffer_info->dma) {
80785298
AD		 2798 pci_unmap_single(rx_ring->pdev,
2799 buffer_info->dma,
4c844851 2800 rx_ring->rx_buffer_len,
6ec43fe6 2801 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2802 buffer_info->dma = 0;
2803 }
2804
2805 if (buffer_info->skb) {
2806 dev_kfree_skb(buffer_info->skb);
2807 buffer_info->skb = NULL;
2808 }
6ec43fe6 2809 if (buffer_info->page_dma) {
80785298
AD		 2810 pci_unmap_page(rx_ring->pdev,
2811 buffer_info->page_dma,
6ec43fe6
AD		 2812 PAGE_SIZE / 2,
2813 PCI_DMA_FROMDEVICE);
2814 buffer_info->page_dma = 0;
2815 }
9d5c8243 2816 if (buffer_info->page) {
9d5c8243
AK		 2817 put_page(buffer_info->page);
2818 buffer_info->page = NULL;
bf36c1a0 2819 buffer_info->page_offset = 0;
9d5c8243
AK		 2820 }
2821 }
2822
9d5c8243
AK		 2823 size = sizeof(struct igb_buffer) * rx_ring->count;
2824 memset(rx_ring->buffer_info, 0, size);
2825
2826 /* Zero out the descriptor ring */
2827 memset(rx_ring->desc, 0, rx_ring->size);
2828
2829 rx_ring->next_to_clean = 0;
2830 rx_ring->next_to_use = 0;
9d5c8243
AK		 2831}
2832
2833/**
2834 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2835 * @adapter: board private structure
2836 **/
2837static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2838{
2839 int i;
2840
2841 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2842 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 2843}
2844
2845/**
2846 * igb_set_mac - Change the Ethernet Address of the NIC
2847 * @netdev: network interface device structure
2848 * @p: pointer to an address structure
2849 *
2850 * Returns 0 on success, negative on failure
2851 **/
2852static int igb_set_mac(struct net_device *netdev, void *p)
2853{
2854 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2855 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2856 struct sockaddr *addr = p;
2857
2858 if (!is_valid_ether_addr(addr->sa_data))
2859 return -EADDRNOTAVAIL;
2860
2861 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2862 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2863
26ad9178
AD		 2864 /* set the correct pool for the new PF MAC address in entry 0 */
2865 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2866 adapter->vfs_allocated_count);
e1739522 2867
9d5c8243
AK		 2868 return 0;
2869}
2870
2871/**
68d480c4 2872 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2873 * @netdev: network interface device structure
2874 *
68d480c4
AD		 2875 * Writes multicast address list to the MTA hash table.
2876 * Returns: -ENOMEM on failure
2877 * 0 on no addresses written
2878 * X on writing X addresses to MTA
9d5c8243 2879 **/
68d480c4 2880static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2881{
2882 struct igb_adapter *adapter = netdev_priv(netdev);
2883 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2884 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2885 u8 *mta_list;
2886 u32 vmolr = 0;
9d5c8243
AK		 2887 int i;
2888
4cd24eaf 2889 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 2890 /* nothing to program, so clear mc list */
2891 igb_update_mc_addr_list(hw, NULL, 0);
2892 igb_restore_vf_multicasts(adapter);
2893 return 0;
2894 }
9d5c8243 2895
4cd24eaf 2896 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 2897 if (!mta_list)
2898 return -ENOMEM;
ff41f8dc 2899
68d480c4
AD		 2900 /* set vmolr receive overflow multicast bit */
2901 vmolr |= E1000_VMOLR_ROMPE;
2902
2903 /* The shared function expects a packed array of only addresses. */
2904 mc_ptr = netdev->mc_list;
2905
4cd24eaf 2906 for (i = 0; i < netdev_mc_count(netdev); i++) {
68d480c4
AD		 2907 if (!mc_ptr)
2908 break;
2909 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2910 mc_ptr = mc_ptr->next;
746b9f02 2911 }
68d480c4
AD		 2912 igb_update_mc_addr_list(hw, mta_list, i);
2913 kfree(mta_list);
2914
4cd24eaf 2915 return netdev_mc_count(netdev);
68d480c4
AD		 2916}
2917
2918/**
2919 * igb_write_uc_addr_list - write unicast addresses to RAR table
2920 * @netdev: network interface device structure
2921 *
2922 * Writes unicast address list to the RAR table.
2923 * Returns: -ENOMEM on failure/insufficient address space
2924 * 0 on no addresses written
2925 * X on writing X addresses to the RAR table
2926 **/
2927static int igb_write_uc_addr_list(struct net_device *netdev)
2928{
2929 struct igb_adapter *adapter = netdev_priv(netdev);
2930 struct e1000_hw *hw = &adapter->hw;
2931 unsigned int vfn = adapter->vfs_allocated_count;
2932 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2933 int count = 0;
2934
2935 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 2936 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 2937 return -ENOMEM;
9d5c8243 2938
32e7bfc4 2939 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 2940 struct netdev_hw_addr *ha;
32e7bfc4
JP		 2941
2942 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 2943 if (!rar_entries)
2944 break;
26ad9178
AD		 2945 igb_rar_set_qsel(adapter, ha->addr,
2946 rar_entries--,
68d480c4
AD		 2947 vfn);
2948 count++;
ff41f8dc
AD		 2949 }
2950 }
2951 /* write the addresses in reverse order to avoid write combining */
2952 for (; rar_entries > 0 ; rar_entries--) {
2953 wr32(E1000_RAH(rar_entries), 0);
2954 wr32(E1000_RAL(rar_entries), 0);
2955 }
2956 wrfl();
2957
68d480c4
AD		 2958 return count;
2959}
2960
2961/**
2962 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2963 * @netdev: network interface device structure
2964 *
2965 * The set_rx_mode entry point is called whenever the unicast or multicast
2966 * address lists or the network interface flags are updated. This routine is
2967 * responsible for configuring the hardware for proper unicast, multicast,
2968 * promiscuous mode, and all-multi behavior.
2969 **/
2970static void igb_set_rx_mode(struct net_device *netdev)
2971{
2972 struct igb_adapter *adapter = netdev_priv(netdev);
2973 struct e1000_hw *hw = &adapter->hw;
2974 unsigned int vfn = adapter->vfs_allocated_count;
2975 u32 rctl, vmolr = 0;
2976 int count;
2977
2978 /* Check for Promiscuous and All Multicast modes */
2979 rctl = rd32(E1000_RCTL);
2980
2981 /* clear the effected bits */
2982 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2983
2984 if (netdev->flags & IFF_PROMISC) {
2985 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2986 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2987 } else {
2988 if (netdev->flags & IFF_ALLMULTI) {
2989 rctl |= E1000_RCTL_MPE;
2990 vmolr |= E1000_VMOLR_MPME;
2991 } else {
2992 /*
2993 * Write addresses to the MTA, if the attempt fails
2994 * then we should just turn on promiscous mode so
2995 * that we can at least receive multicast traffic
2996 */
2997 count = igb_write_mc_addr_list(netdev);
2998 if (count < 0) {
2999 rctl |= E1000_RCTL_MPE;
3000 vmolr |= E1000_VMOLR_MPME;
3001 } else if (count) {
3002 vmolr |= E1000_VMOLR_ROMPE;
3003 }
3004 }
3005 /*
3006 * Write addresses to available RAR registers, if there is not
3007 * sufficient space to store all the addresses then enable
3008 * unicast promiscous mode
3009 */
3010 count = igb_write_uc_addr_list(netdev);
3011 if (count < 0) {
3012 rctl |= E1000_RCTL_UPE;
3013 vmolr |= E1000_VMOLR_ROPE;
3014 }
3015 rctl |= E1000_RCTL_VFE;
28fc06f5 3016 }
68d480c4 3017 wr32(E1000_RCTL, rctl);
28fc06f5 3018
68d480c4
AD		 3019 /*
3020 * In order to support SR-IOV and eventually VMDq it is necessary to set
3021 * the VMOLR to enable the appropriate modes. Without this workaround
3022 * we will have issues with VLAN tag stripping not being done for frames
3023 * that are only arriving because we are the default pool
3024 */
3025 if (hw->mac.type < e1000_82576)
28fc06f5 3026 return;
9d5c8243 3027
68d480c4
AD		 3028 vmolr |= rd32(E1000_VMOLR(vfn)) &
3029 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3030 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3031 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3032}
3033
3034/* Need to wait a few seconds after link up to get diagnostic information from
3035 * the phy */
3036static void igb_update_phy_info(unsigned long data)
3037{
3038 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3039 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3040}
3041
4d6b725e
AD		 3042/**
3043 * igb_has_link - check shared code for link and determine up/down
3044 * @adapter: pointer to driver private info
3045 **/
3145535a 3046bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3047{
3048 struct e1000_hw *hw = &adapter->hw;
3049 bool link_active = false;
3050 s32 ret_val = 0;
3051
3052 /* get_link_status is set on LSC (link status) interrupt or
3053 * rx sequence error interrupt. get_link_status will stay
3054 * false until the e1000_check_for_link establishes link
3055 * for copper adapters ONLY
3056 */
3057 switch (hw->phy.media_type) {
3058 case e1000_media_type_copper:
3059 if (hw->mac.get_link_status) {
3060 ret_val = hw->mac.ops.check_for_link(hw);
3061 link_active = !hw->mac.get_link_status;
3062 } else {
3063 link_active = true;
3064 }
3065 break;
4d6b725e
AD		 3066 case e1000_media_type_internal_serdes:
3067 ret_val = hw->mac.ops.check_for_link(hw);
3068 link_active = hw->mac.serdes_has_link;
3069 break;
3070 default:
3071 case e1000_media_type_unknown:
3072 break;
3073 }
3074
3075 return link_active;
3076}
3077
9d5c8243
AK		 3078/**
3079 * igb_watchdog - Timer Call-back
3080 * @data: pointer to adapter cast into an unsigned long
3081 **/
3082static void igb_watchdog(unsigned long data)
3083{
3084 struct igb_adapter *adapter = (struct igb_adapter *)data;
3085 /* Do the rest outside of interrupt context */
3086 schedule_work(&adapter->watchdog_task);
3087}
3088
3089static void igb_watchdog_task(struct work_struct *work)
3090{
3091 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3092 struct igb_adapter,
3093 watchdog_task);
9d5c8243 3094 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3095 struct net_device *netdev = adapter->netdev;
9d5c8243 3096 u32 link;
7a6ea550 3097 int i;
9d5c8243 3098
4d6b725e 3099 link = igb_has_link(adapter);
9d5c8243
AK		 3100 if (link) {
3101 if (!netif_carrier_ok(netdev)) {
3102 u32 ctrl;
330a6d6a
AD		 3103 hw->mac.ops.get_speed_and_duplex(hw,
3104 &adapter->link_speed,
3105 &adapter->link_duplex);
9d5c8243
AK		 3106
3107 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3108 /* Links status message must follow this format */
3109 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3110 "Flow Control: %s\n",
559e9c49
AD		 3111 netdev->name,
3112 adapter->link_speed,
3113 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3114 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3115 ((ctrl & E1000_CTRL_TFCE) &&
3116 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3117 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3118 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243
AK		 3119
3120 /* tweak tx_queue_len according to speed/duplex and
3121 * adjust the timeout factor */
3122 netdev->tx_queue_len = adapter->tx_queue_len;
3123 adapter->tx_timeout_factor = 1;
3124 switch (adapter->link_speed) {
3125 case SPEED_10:
3126 netdev->tx_queue_len = 10;
3127 adapter->tx_timeout_factor = 14;
3128 break;
3129 case SPEED_100:
3130 netdev->tx_queue_len = 100;
3131 /* maybe add some timeout factor ? */
3132 break;
3133 }
3134
3135 netif_carrier_on(netdev);
9d5c8243 3136
4ae196df
AD		 3137 igb_ping_all_vfs(adapter);
3138
4b1a9877 3139 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3140 if (!test_bit(__IGB_DOWN, &adapter->state))
3141 mod_timer(&adapter->phy_info_timer,
3142 round_jiffies(jiffies + 2 * HZ));
3143 }
3144 } else {
3145 if (netif_carrier_ok(netdev)) {
3146 adapter->link_speed = 0;
3147 adapter->link_duplex = 0;
527d47c1
AD		 3148 /* Links status message must follow this format */
3149 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3150 netdev->name);
9d5c8243 3151 netif_carrier_off(netdev);
4b1a9877 3152
4ae196df
AD		 3153 igb_ping_all_vfs(adapter);
3154
4b1a9877 3155 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3156 if (!test_bit(__IGB_DOWN, &adapter->state))
3157 mod_timer(&adapter->phy_info_timer,
3158 round_jiffies(jiffies + 2 * HZ));
3159 }
3160 }
3161
9d5c8243 3162 igb_update_stats(adapter);
9d5c8243 3163
dbabb065 3164 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3165 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3166 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3167 /* We've lost link, so the controller stops DMA,
3168 * but we've got queued Tx work that's never going
3169 * to get done, so reset controller to flush Tx.
3170 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3171 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3172 adapter->tx_timeout_count++;
3173 schedule_work(&adapter->reset_task);
3174 /* return immediately since reset is imminent */
3175 return;
3176 }
9d5c8243 3177 }
9d5c8243 3178
dbabb065
AD		 3179 /* Force detection of hung controller every watchdog period */
3180 tx_ring->detect_tx_hung = true;
3181 }
f7ba205e 3182
9d5c8243 3183 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3184 if (adapter->msix_entries) {
047e0030
AD		 3185 u32 eics = 0;
3186 for (i = 0; i < adapter->num_q_vectors; i++) {
3187 struct igb_q_vector *q_vector = adapter->q_vector[i];
3188 eics |= q_vector->eims_value;
3189 }
7a6ea550
AD		 3190 wr32(E1000_EICS, eics);
3191 } else {
3192 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3193 }
9d5c8243 3194
9d5c8243
AK		 3195 /* Reset the timer */
3196 if (!test_bit(__IGB_DOWN, &adapter->state))
3197 mod_timer(&adapter->watchdog_timer,
3198 round_jiffies(jiffies + 2 * HZ));
3199}
3200
3201enum latency_range {
3202 lowest_latency = 0,
3203 low_latency = 1,
3204 bulk_latency = 2,
3205 latency_invalid = 255
3206};
3207
6eb5a7f1
AD		 3208/**
3209 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3210 *
3211 * Stores a new ITR value based on strictly on packet size. This
3212 * algorithm is less sophisticated than that used in igb_update_itr,
3213 * due to the difficulty of synchronizing statistics across multiple
3214 * receive rings. The divisors and thresholds used by this fuction
3215 * were determined based on theoretical maximum wire speed and testing
3216 * data, in order to minimize response time while increasing bulk
3217 * throughput.
3218 * This functionality is controlled by the InterruptThrottleRate module
3219 * parameter (see igb_param.c)
3220 * NOTE: This function is called only when operating in a multiqueue
3221 * receive environment.
047e0030 3222 * @q_vector: pointer to q_vector
6eb5a7f1 3223 **/
047e0030 3224static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3225{
047e0030 3226 int new_val = q_vector->itr_val;
6eb5a7f1 3227 int avg_wire_size = 0;
047e0030 3228 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3229
6eb5a7f1
AD		 3230 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3231 * ints/sec - ITR timer value of 120 ticks.
3232 */
3233 if (adapter->link_speed != SPEED_1000) {
047e0030 3234 new_val = 976;
6eb5a7f1 3235 goto set_itr_val;
9d5c8243 3236 }
047e0030
AD		 3237
3238 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3239 struct igb_ring *ring = q_vector->rx_ring;
3240 avg_wire_size = ring->total_bytes / ring->total_packets;
3241 }
3242
3243 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3244 struct igb_ring *ring = q_vector->tx_ring;
3245 avg_wire_size = max_t(u32, avg_wire_size,
3246 (ring->total_bytes /
3247 ring->total_packets));
3248 }
3249
3250 /* if avg_wire_size isn't set no work was done */
3251 if (!avg_wire_size)
3252 goto clear_counts;
9d5c8243 3253
6eb5a7f1
AD		 3254 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3255 avg_wire_size += 24;
3256
3257 /* Don't starve jumbo frames */
3258 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3259
6eb5a7f1
AD		 3260 /* Give a little boost to mid-size frames */
3261 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3262 new_val = avg_wire_size / 3;
3263 else
3264 new_val = avg_wire_size / 2;
9d5c8243 3265
abe1c363
NN		 3266 /* when in itr mode 3 do not exceed 20K ints/sec */
3267 if (adapter->rx_itr_setting == 3 && new_val < 196)
3268 new_val = 196;
3269
6eb5a7f1 3270set_itr_val:
047e0030
AD		 3271 if (new_val != q_vector->itr_val) {
3272 q_vector->itr_val = new_val;
3273 q_vector->set_itr = 1;
9d5c8243 3274 }
6eb5a7f1 3275clear_counts:
047e0030
AD		 3276 if (q_vector->rx_ring) {
3277 q_vector->rx_ring->total_bytes = 0;
3278 q_vector->rx_ring->total_packets = 0;
3279 }
3280 if (q_vector->tx_ring) {
3281 q_vector->tx_ring->total_bytes = 0;
3282 q_vector->tx_ring->total_packets = 0;
3283 }
9d5c8243
AK		 3284}
3285
3286/**
3287 * igb_update_itr - update the dynamic ITR value based on statistics
3288 * Stores a new ITR value based on packets and byte
3289 * counts during the last interrupt. The advantage of per interrupt
3290 * computation is faster updates and more accurate ITR for the current
3291 * traffic pattern. Constants in this function were computed
3292 * based on theoretical maximum wire speed and thresholds were set based
3293 * on testing data as well as attempting to minimize response time
3294 * while increasing bulk throughput.
3295 * this functionality is controlled by the InterruptThrottleRate module
3296 * parameter (see igb_param.c)
3297 * NOTE: These calculations are only valid when operating in a single-
3298 * queue environment.
3299 * @adapter: pointer to adapter
047e0030 3300 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3301 * @packets: the number of packets during this measurement interval
3302 * @bytes: the number of bytes during this measurement interval
3303 **/
3304static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3305 int packets, int bytes)
3306{
3307 unsigned int retval = itr_setting;
3308
3309 if (packets == 0)
3310 goto update_itr_done;
3311
3312 switch (itr_setting) {
3313 case lowest_latency:
3314 /* handle TSO and jumbo frames */
3315 if (bytes/packets > 8000)
3316 retval = bulk_latency;
3317 else if ((packets < 5) && (bytes > 512))
3318 retval = low_latency;
3319 break;
3320 case low_latency: /* 50 usec aka 20000 ints/s */
3321 if (bytes > 10000) {
3322 /* this if handles the TSO accounting */
3323 if (bytes/packets > 8000) {
3324 retval = bulk_latency;
3325 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3326 retval = bulk_latency;
3327 } else if ((packets > 35)) {
3328 retval = lowest_latency;
3329 }
3330 } else if (bytes/packets > 2000) {
3331 retval = bulk_latency;
3332 } else if (packets <= 2 && bytes < 512) {
3333 retval = lowest_latency;
3334 }
3335 break;
3336 case bulk_latency: /* 250 usec aka 4000 ints/s */
3337 if (bytes > 25000) {
3338 if (packets > 35)
3339 retval = low_latency;
1e5c3d21 3340 } else if (bytes < 1500) {
9d5c8243
AK		 3341 retval = low_latency;
3342 }
3343 break;
3344 }
3345
3346update_itr_done:
3347 return retval;
3348}
3349
6eb5a7f1 3350static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3351{
047e0030 3352 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3353 u16 current_itr;
047e0030 3354 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3355
3356 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3357 if (adapter->link_speed != SPEED_1000) {
3358 current_itr = 0;
3359 new_itr = 4000;
3360 goto set_itr_now;
3361 }
3362
3363 adapter->rx_itr = igb_update_itr(adapter,
3364 adapter->rx_itr,
3025a446
AD		 3365 q_vector->rx_ring->total_packets,
3366 q_vector->rx_ring->total_bytes);
9d5c8243 3367
047e0030
AD		 3368 adapter->tx_itr = igb_update_itr(adapter,
3369 adapter->tx_itr,
3025a446
AD		 3370 q_vector->tx_ring->total_packets,
3371 q_vector->tx_ring->total_bytes);
047e0030 3372 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3373
6eb5a7f1 3374 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3375 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3376 current_itr = low_latency;
3377
9d5c8243
AK		 3378 switch (current_itr) {
3379 /* counts and packets in update_itr are dependent on these numbers */
3380 case lowest_latency:
78b1f607 3381 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3382 break;
3383 case low_latency:
78b1f607 3384 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3385 break;
3386 case bulk_latency:
78b1f607 3387 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3388 break;
3389 default:
3390 break;
3391 }
3392
3393set_itr_now:
3025a446
AD		 3394 q_vector->rx_ring->total_bytes = 0;
3395 q_vector->rx_ring->total_packets = 0;
3396 q_vector->tx_ring->total_bytes = 0;
3397 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3398
047e0030 3399 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3400 /* this attempts to bias the interrupt rate towards Bulk
3401 * by adding intermediate steps when interrupt rate is
3402 * increasing */
047e0030
AD		 3403 new_itr = new_itr > q_vector->itr_val ?
3404 max((new_itr * q_vector->itr_val) /
3405 (new_itr + (q_vector->itr_val >> 2)),
3406 new_itr) :
9d5c8243
AK		 3407 new_itr;
3408 /* Don't write the value here; it resets the adapter's
3409 * internal timer, and causes us to delay far longer than
3410 * we should between interrupts. Instead, we write the ITR
3411 * value at the beginning of the next interrupt so the timing
3412 * ends up being correct.
3413 */
047e0030
AD		 3414 q_vector->itr_val = new_itr;
3415 q_vector->set_itr = 1;
9d5c8243
AK		 3416 }
3417
3418 return;
3419}
3420
9d5c8243
AK		 3421#define IGB_TX_FLAGS_CSUM 0x00000001
3422#define IGB_TX_FLAGS_VLAN 0x00000002
3423#define IGB_TX_FLAGS_TSO 0x00000004
3424#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3425#define IGB_TX_FLAGS_TSTAMP 0x00000010
3426#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3427#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3428
85ad76b2 3429static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3430 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3431{
3432 struct e1000_adv_tx_context_desc *context_desc;
3433 unsigned int i;
3434 int err;
3435 struct igb_buffer *buffer_info;
3436 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3437 u32 mss_l4len_idx;
3438 u8 l4len;
9d5c8243
AK		 3439
3440 if (skb_header_cloned(skb)) {
3441 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3442 if (err)
3443 return err;
3444 }
3445
3446 l4len = tcp_hdrlen(skb);
3447 *hdr_len += l4len;
3448
3449 if (skb->protocol == htons(ETH_P_IP)) {
3450 struct iphdr *iph = ip_hdr(skb);
3451 iph->tot_len = 0;
3452 iph->check = 0;
3453 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3454 iph->daddr, 0,
3455 IPPROTO_TCP,
3456 0);
8e1e8a47 3457 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3458 ipv6_hdr(skb)->payload_len = 0;
3459 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3460 &ipv6_hdr(skb)->daddr,
3461 0, IPPROTO_TCP, 0);
3462 }
3463
3464 i = tx_ring->next_to_use;
3465
3466 buffer_info = &tx_ring->buffer_info[i];
3467 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3468 /* VLAN MACLEN IPLEN */
3469 if (tx_flags & IGB_TX_FLAGS_VLAN)
3470 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3471 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3472 *hdr_len += skb_network_offset(skb);
3473 info |= skb_network_header_len(skb);
3474 *hdr_len += skb_network_header_len(skb);
3475 context_desc->vlan_macip_lens = cpu_to_le32(info);
3476
3477 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3478 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3479
3480 if (skb->protocol == htons(ETH_P_IP))
3481 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3482 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3483
3484 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3485
3486 /* MSS L4LEN IDX */
3487 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3488 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3489
73cd78f1 3490 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3491 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3492 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3493
3494 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3495 context_desc->seqnum_seed = 0;
3496
3497 buffer_info->time_stamp = jiffies;
0e014cb1 3498 buffer_info->next_to_watch = i;
9d5c8243
AK		 3499 buffer_info->dma = 0;
3500 i++;
3501 if (i == tx_ring->count)
3502 i = 0;
3503
3504 tx_ring->next_to_use = i;
3505
3506 return true;
3507}
3508
85ad76b2
AD		 3509static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3510 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3511{
3512 struct e1000_adv_tx_context_desc *context_desc;
80785298 3513 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3514 struct igb_buffer *buffer_info;
3515 u32 info = 0, tu_cmd = 0;
80785298 3516 unsigned int i;
9d5c8243
AK		 3517
3518 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3519 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3520 i = tx_ring->next_to_use;
3521 buffer_info = &tx_ring->buffer_info[i];
3522 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3523
3524 if (tx_flags & IGB_TX_FLAGS_VLAN)
3525 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3526
9d5c8243
AK		 3527 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3528 if (skb->ip_summed == CHECKSUM_PARTIAL)
3529 info |= skb_network_header_len(skb);
3530
3531 context_desc->vlan_macip_lens = cpu_to_le32(info);
3532
3533 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3534
3535 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3536 __be16 protocol;
3537
3538 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3539 const struct vlan_ethhdr *vhdr =
3540 (const struct vlan_ethhdr*)skb->data;
3541
3542 protocol = vhdr->h_vlan_encapsulated_proto;
3543 } else {
3544 protocol = skb->protocol;
3545 }
3546
3547 switch (protocol) {
09640e63 3548 case cpu_to_be16(ETH_P_IP):
9d5c8243 3549 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3550 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3551 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3552 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3553 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3554 break;
09640e63 3555 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3556 /* XXX what about other V6 headers?? */
3557 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3558 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3559 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3560 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3561 break;
3562 default:
3563 if (unlikely(net_ratelimit()))
80785298 3564 dev_warn(&pdev->dev,
44b0cda3
MW		 3565 "partial checksum but proto=%x!\n",
3566 skb->protocol);
3567 break;
3568 }
9d5c8243
AK		 3569 }
3570
3571 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3572 context_desc->seqnum_seed = 0;
85ad76b2 3573 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3574 context_desc->mss_l4len_idx =
85ad76b2 3575 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3576
3577 buffer_info->time_stamp = jiffies;
0e014cb1 3578 buffer_info->next_to_watch = i;
9d5c8243
AK		 3579 buffer_info->dma = 0;
3580
3581 i++;
3582 if (i == tx_ring->count)
3583 i = 0;
3584 tx_ring->next_to_use = i;
3585
3586 return true;
3587 }
9d5c8243
AK		 3588 return false;
3589}
3590
3591#define IGB_MAX_TXD_PWR 16
3592#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3593
80785298 3594static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3595 unsigned int first)
9d5c8243
AK		 3596{
3597 struct igb_buffer *buffer_info;
80785298 3598 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3599 unsigned int len = skb_headlen(skb);
3600 unsigned int count = 0, i;
3601 unsigned int f;
3602
3603 i = tx_ring->next_to_use;
3604
3605 buffer_info = &tx_ring->buffer_info[i];
3606 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3607 buffer_info->length = len;
3608 /* set time_stamp *before* dma to help avoid a possible race */
3609 buffer_info->time_stamp = jiffies;
0e014cb1 3610 buffer_info->next_to_watch = i;
6366ad33
AD		 3611 buffer_info->dma = pci_map_single(pdev, skb->data, len,
3612 PCI_DMA_TODEVICE);
3613 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3614 goto dma_error;
9d5c8243
AK		 3615
3616 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3617 struct skb_frag_struct *frag;
3618
8581145f 3619 count++;
65689fef
AD		 3620 i++;
3621 if (i == tx_ring->count)
3622 i = 0;
3623
9d5c8243
AK		 3624 frag = &skb_shinfo(skb)->frags[f];
3625 len = frag->size;
3626
3627 buffer_info = &tx_ring->buffer_info[i];
3628 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3629 buffer_info->length = len;
3630 buffer_info->time_stamp = jiffies;
0e014cb1 3631 buffer_info->next_to_watch = i;
6366ad33
AD		 3632 buffer_info->mapped_as_page = true;
3633 buffer_info->dma = pci_map_page(pdev,
3634 frag->page,
3635 frag->page_offset,
3636 len,
3637 PCI_DMA_TODEVICE);
3638 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3639 goto dma_error;
3640
9d5c8243
AK		 3641 }
3642
9d5c8243 3643 tx_ring->buffer_info[i].skb = skb;
40e90c26 3644 tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
0e014cb1 3645 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3646
cdfd01fc 3647 return ++count;
6366ad33
AD		 3648
3649dma_error:
3650 dev_err(&pdev->dev, "TX DMA map failed\n");
3651
3652 /* clear timestamp and dma mappings for failed buffer_info mapping */
3653 buffer_info->dma = 0;
3654 buffer_info->time_stamp = 0;
3655 buffer_info->length = 0;
3656 buffer_info->next_to_watch = 0;
3657 buffer_info->mapped_as_page = false;
3658 count--;
3659
3660 /* clear timestamp and dma mappings for remaining portion of packet */
3661 while (count >= 0) {
3662 count--;
3663 i--;
3664 if (i < 0)
3665 i += tx_ring->count;
3666 buffer_info = &tx_ring->buffer_info[i];
3667 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3668 }
3669
3670 return 0;
9d5c8243
AK		 3671}
3672
85ad76b2 3673static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3674 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3675 u8 hdr_len)
3676{
cdfd01fc 3677 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3678 struct igb_buffer *buffer_info;
3679 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3680 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3681
3682 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3683 E1000_ADVTXD_DCMD_DEXT);
3684
3685 if (tx_flags & IGB_TX_FLAGS_VLAN)
3686 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3687
33af6bcc
PO		 3688 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3689 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3690
9d5c8243
AK		 3691 if (tx_flags & IGB_TX_FLAGS_TSO) {
3692 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3693
3694 /* insert tcp checksum */
3695 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3696
3697 /* insert ip checksum */
3698 if (tx_flags & IGB_TX_FLAGS_IPV4)
3699 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3700
3701 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3702 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3703 }
3704
85ad76b2
AD		 3705 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3706 (tx_flags & (IGB_TX_FLAGS_CSUM |
3707 IGB_TX_FLAGS_TSO |
7dfc16fa 3708 IGB_TX_FLAGS_VLAN)))
85ad76b2 3709 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3710
3711 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3712
cdfd01fc 3713 do {
9d5c8243
AK		 3714 buffer_info = &tx_ring->buffer_info[i];
3715 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3716 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3717 tx_desc->read.cmd_type_len =
3718 cpu_to_le32(cmd_type_len | buffer_info->length);
3719 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3720 count--;
9d5c8243
AK		 3721 i++;
3722 if (i == tx_ring->count)
3723 i = 0;
cdfd01fc 3724 } while (count > 0);
9d5c8243 3725
85ad76b2 3726 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3727 /* Force memory writes to complete before letting h/w
3728 * know there are new descriptors to fetch. (Only
3729 * applicable for weak-ordered memory model archs,
3730 * such as IA-64). */
3731 wmb();
3732
3733 tx_ring->next_to_use = i;
fce99e34 3734 writel(i, tx_ring->tail);
9d5c8243
AK		 3735 /* we need this if more than one processor can write to our tail
3736 * at a time, it syncronizes IO on IA64/Altix systems */
3737 mmiowb();
3738}
3739
e694e964 3740static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3741{
e694e964
AD		 3742 struct net_device *netdev = tx_ring->netdev;
3743
661086df 3744 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3745
9d5c8243
AK		 3746 /* Herbert's original patch had:
3747 * smp_mb__after_netif_stop_queue();
3748 * but since that doesn't exist yet, just open code it. */
3749 smp_mb();
3750
3751 /* We need to check again in a case another CPU has just
3752 * made room available. */
c493ea45 3753 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3754 return -EBUSY;
3755
3756 /* A reprieve! */
661086df 3757 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3758 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3759 return 0;
3760}
3761
717ba089 3762static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3763{
c493ea45 3764 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3765 return 0;
e694e964 3766 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3767}
3768
b1a436c3
AD		 3769netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3770 struct igb_ring *tx_ring)
9d5c8243 3771{
e694e964 3772 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 3773 int tso = 0, count;
91d4ee33
NN		 3774 u32 tx_flags = 0;
3775 u16 first;
3776 u8 hdr_len = 0;
c5b9bd5e 3777 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3778
9d5c8243
AK		 3779 /* need: 1 descriptor per page,
3780 * + 2 desc gap to keep tail from touching head,
3781 * + 1 desc for skb->data,
3782 * + 1 desc for context descriptor,
3783 * otherwise try next time */
e694e964 3784 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3785 /* this is a hard error */
9d5c8243
AK		 3786 return NETDEV_TX_BUSY;
3787 }
33af6bcc 3788
33af6bcc
PO		 3789 if (unlikely(shtx->hardware)) {
3790 shtx->in_progress = 1;
3791 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3792 }
9d5c8243 3793
cdfd01fc 3794 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3795 tx_flags |= IGB_TX_FLAGS_VLAN;
3796 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3797 }
3798
661086df
PWJ		 3799 if (skb->protocol == htons(ETH_P_IP))
3800 tx_flags |= IGB_TX_FLAGS_IPV4;
3801
0e014cb1 3802 first = tx_ring->next_to_use;
85ad76b2
AD		 3803 if (skb_is_gso(skb)) {
3804 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3805
85ad76b2
AD		 3806 if (tso < 0) {
3807 dev_kfree_skb_any(skb);
3808 return NETDEV_TX_OK;
3809 }
9d5c8243
AK		 3810 }
3811
3812 if (tso)
3813 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3814 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3815 (skb->ip_summed == CHECKSUM_PARTIAL))
3816 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3817
65689fef 3818 /*
cdfd01fc 3819 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3820 * has occured and we need to rewind the descriptor queue
3821 */
80785298 3822 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 3823 if (!count) {
65689fef
AD		 3824 dev_kfree_skb_any(skb);
3825 tx_ring->buffer_info[first].time_stamp = 0;
3826 tx_ring->next_to_use = first;
85ad76b2 3827 return NETDEV_TX_OK;
65689fef 3828 }
9d5c8243 3829
85ad76b2
AD		 3830 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3831
3832 /* Make sure there is space in the ring for the next send. */
e694e964 3833 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3834
9d5c8243
AK		 3835 return NETDEV_TX_OK;
3836}
3837
3b29a56d
SH		 3838static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3839 struct net_device *netdev)
9d5c8243
AK		 3840{
3841 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3842 struct igb_ring *tx_ring;
661086df 3843 int r_idx = 0;
b1a436c3
AD		 3844
3845 if (test_bit(__IGB_DOWN, &adapter->state)) {
3846 dev_kfree_skb_any(skb);
3847 return NETDEV_TX_OK;
3848 }
3849
3850 if (skb->len <= 0) {
3851 dev_kfree_skb_any(skb);
3852 return NETDEV_TX_OK;
3853 }
3854
1bfaf07b 3855 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3856 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3857
3858 /* This goes back to the question of how to logically map a tx queue
3859 * to a flow. Right now, performance is impacted slightly negatively
3860 * if using multiple tx queues. If the stack breaks away from a
3861 * single qdisc implementation, we can look at this again. */
e694e964 3862 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3863}
3864
3865/**
3866 * igb_tx_timeout - Respond to a Tx Hang
3867 * @netdev: network interface device structure
3868 **/
3869static void igb_tx_timeout(struct net_device *netdev)
3870{
3871 struct igb_adapter *adapter = netdev_priv(netdev);
3872 struct e1000_hw *hw = &adapter->hw;
3873
3874 /* Do the reset outside of interrupt context */
3875 adapter->tx_timeout_count++;
f7ba205e 3876
55cac248
AD		 3877 if (hw->mac.type == e1000_82580)
3878 hw->dev_spec._82575.global_device_reset = true;
3879
9d5c8243 3880 schedule_work(&adapter->reset_task);
265de409
AD		 3881 wr32(E1000_EICS,
3882 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3883}
3884
3885static void igb_reset_task(struct work_struct *work)
3886{
3887 struct igb_adapter *adapter;
3888 adapter = container_of(work, struct igb_adapter, reset_task);
3889
3890 igb_reinit_locked(adapter);
3891}
3892
3893/**
3894 * igb_get_stats - Get System Network Statistics
3895 * @netdev: network interface device structure
3896 *
3897 * Returns the address of the device statistics structure.
3898 * The statistics are actually updated from the timer callback.
3899 **/
73cd78f1 3900static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3901{
9d5c8243 3902 /* only return the current stats */
8d24e933 3903 return &netdev->stats;
9d5c8243
AK		 3904}
3905
3906/**
3907 * igb_change_mtu - Change the Maximum Transfer Unit
3908 * @netdev: network interface device structure
3909 * @new_mtu: new value for maximum frame size
3910 *
3911 * Returns 0 on success, negative on failure
3912 **/
3913static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3914{
3915 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3916 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3917 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3918 u32 rx_buffer_len, i;
9d5c8243 3919
c809d227 3920 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3921 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3922 return -EINVAL;
3923 }
3924
9d5c8243 3925 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3926 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3927 return -EINVAL;
3928 }
3929
3930 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3931 msleep(1);
73cd78f1 3932
9d5c8243
AK		 3933 /* igb_down has a dependency on max_frame_size */
3934 adapter->max_frame_size = max_frame;
559e9c49 3935
9d5c8243
AK		 3936 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3937 * means we reserve 2 more, this pushes us to allocate from the next
3938 * larger slab size.
3939 * i.e. RXBUFFER_2048 --> size-4096 slab
3940 */
3941
7d95b717 3942 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3943 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3944 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3945 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3946 else
4c844851
AD		 3947 rx_buffer_len = IGB_RXBUFFER_128;
3948
3949 if (netif_running(netdev))
3950 igb_down(adapter);
9d5c8243 3951
090b1795 3952 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3953 netdev->mtu, new_mtu);
3954 netdev->mtu = new_mtu;
3955
4c844851 3956 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3957 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 3958
9d5c8243
AK		 3959 if (netif_running(netdev))
3960 igb_up(adapter);
3961 else
3962 igb_reset(adapter);
3963
3964 clear_bit(__IGB_RESETTING, &adapter->state);
3965
3966 return 0;
3967}
3968
3969/**
3970 * igb_update_stats - Update the board statistics counters
3971 * @adapter: board private structure
3972 **/
3973
3974void igb_update_stats(struct igb_adapter *adapter)
3975{
128e45eb 3976 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 3977 struct e1000_hw *hw = &adapter->hw;
3978 struct pci_dev *pdev = adapter->pdev;
43915c7c 3979 u32 rnbc, reg;
9d5c8243 3980 u16 phy_tmp;
3f9c0164
AD		 3981 int i;
3982 u64 bytes, packets;
9d5c8243
AK		 3983
3984#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3985
3986 /*
3987 * Prevent stats update while adapter is being reset, or if the pci
3988 * connection is down.
3989 */
3990 if (adapter->link_speed == 0)
3991 return;
3992 if (pci_channel_offline(pdev))
3993 return;
3994
3f9c0164
AD		 3995 bytes = 0;
3996 packets = 0;
3997 for (i = 0; i < adapter->num_rx_queues; i++) {
3998 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 3999 struct igb_ring *ring = adapter->rx_ring[i];
4000 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4001 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 4002 bytes += ring->rx_stats.bytes;
4003 packets += ring->rx_stats.packets;
3f9c0164
AD		 4004 }
4005
128e45eb
AD		 4006 net_stats->rx_bytes = bytes;
4007 net_stats->rx_packets = packets;
3f9c0164
AD		 4008
4009 bytes = 0;
4010 packets = 0;
4011 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4012 struct igb_ring *ring = adapter->tx_ring[i];
4013 bytes += ring->tx_stats.bytes;
4014 packets += ring->tx_stats.packets;
3f9c0164 4015 }
128e45eb
AD		 4016 net_stats->tx_bytes = bytes;
4017 net_stats->tx_packets = packets;
3f9c0164
AD		 4018
4019 /* read stats registers */
9d5c8243
AK		 4020 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4021 adapter->stats.gprc += rd32(E1000_GPRC);
4022 adapter->stats.gorc += rd32(E1000_GORCL);
4023 rd32(E1000_GORCH); /* clear GORCL */
4024 adapter->stats.bprc += rd32(E1000_BPRC);
4025 adapter->stats.mprc += rd32(E1000_MPRC);
4026 adapter->stats.roc += rd32(E1000_ROC);
4027
4028 adapter->stats.prc64 += rd32(E1000_PRC64);
4029 adapter->stats.prc127 += rd32(E1000_PRC127);
4030 adapter->stats.prc255 += rd32(E1000_PRC255);
4031 adapter->stats.prc511 += rd32(E1000_PRC511);
4032 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4033 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4034 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4035 adapter->stats.sec += rd32(E1000_SEC);
4036
4037 adapter->stats.mpc += rd32(E1000_MPC);
4038 adapter->stats.scc += rd32(E1000_SCC);
4039 adapter->stats.ecol += rd32(E1000_ECOL);
4040 adapter->stats.mcc += rd32(E1000_MCC);
4041 adapter->stats.latecol += rd32(E1000_LATECOL);
4042 adapter->stats.dc += rd32(E1000_DC);
4043 adapter->stats.rlec += rd32(E1000_RLEC);
4044 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4045 adapter->stats.xontxc += rd32(E1000_XONTXC);
4046 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4047 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4048 adapter->stats.fcruc += rd32(E1000_FCRUC);
4049 adapter->stats.gptc += rd32(E1000_GPTC);
4050 adapter->stats.gotc += rd32(E1000_GOTCL);
4051 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 4052 rnbc = rd32(E1000_RNBC);
4053 adapter->stats.rnbc += rnbc;
128e45eb 4054 net_stats->rx_fifo_errors += rnbc;
9d5c8243
AK		 4055 adapter->stats.ruc += rd32(E1000_RUC);
4056 adapter->stats.rfc += rd32(E1000_RFC);
4057 adapter->stats.rjc += rd32(E1000_RJC);
4058 adapter->stats.tor += rd32(E1000_TORH);
4059 adapter->stats.tot += rd32(E1000_TOTH);
4060 adapter->stats.tpr += rd32(E1000_TPR);
4061
4062 adapter->stats.ptc64 += rd32(E1000_PTC64);
4063 adapter->stats.ptc127 += rd32(E1000_PTC127);
4064 adapter->stats.ptc255 += rd32(E1000_PTC255);
4065 adapter->stats.ptc511 += rd32(E1000_PTC511);
4066 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4067 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4068
4069 adapter->stats.mptc += rd32(E1000_MPTC);
4070 adapter->stats.bptc += rd32(E1000_BPTC);
4071
2d0b0f69
NN		 4072 adapter->stats.tpt += rd32(E1000_TPT);
4073 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4074
4075 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4076 /* read internal phy specific stats */
4077 reg = rd32(E1000_CTRL_EXT);
4078 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4079 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4080 adapter->stats.tncrs += rd32(E1000_TNCRS);
4081 }
4082
9d5c8243
AK		 4083 adapter->stats.tsctc += rd32(E1000_TSCTC);
4084 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4085
4086 adapter->stats.iac += rd32(E1000_IAC);
4087 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4088 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4089 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4090 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4091 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4092 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4093 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4094 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4095
4096 /* Fill out the OS statistics structure */
128e45eb
AD		 4097 net_stats->multicast = adapter->stats.mprc;
4098 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4099
4100 /* Rx Errors */
4101
4102 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4103 * our own version based on RUC and ROC */
128e45eb 4104 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4105 adapter->stats.crcerrs + adapter->stats.algnerrc +
4106 adapter->stats.ruc + adapter->stats.roc +
4107 adapter->stats.cexterr;
128e45eb
AD		 4108 net_stats->rx_length_errors = adapter->stats.ruc +
4109 adapter->stats.roc;
4110 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4111 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4112 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4113
4114 /* Tx Errors */
128e45eb
AD		 4115 net_stats->tx_errors = adapter->stats.ecol +
4116 adapter->stats.latecol;
4117 net_stats->tx_aborted_errors = adapter->stats.ecol;
4118 net_stats->tx_window_errors = adapter->stats.latecol;
4119 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4120
4121 /* Tx Dropped needs to be maintained elsewhere */
4122
4123 /* Phy Stats */
4124 if (hw->phy.media_type == e1000_media_type_copper) {
4125 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4126 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4127 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4128 adapter->phy_stats.idle_errors += phy_tmp;
4129 }
4130 }
4131
4132 /* Management Stats */
4133 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4134 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4135 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4136}
4137
9d5c8243
AK		 4138static irqreturn_t igb_msix_other(int irq, void *data)
4139{
047e0030 4140 struct igb_adapter *adapter = data;
9d5c8243 4141 struct e1000_hw *hw = &adapter->hw;
844290e5 4142 u32 icr = rd32(E1000_ICR);
844290e5 4143 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4144
7f081d40
AD		 4145 if (icr & E1000_ICR_DRSTA)
4146 schedule_work(&adapter->reset_task);
4147
047e0030 4148 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4149 /* HW is reporting DMA is out of sync */
4150 adapter->stats.doosync++;
4151 }
eebbbdba 4152
4ae196df
AD		 4153 /* Check for a mailbox event */
4154 if (icr & E1000_ICR_VMMB)
4155 igb_msg_task(adapter);
4156
4157 if (icr & E1000_ICR_LSC) {
4158 hw->mac.get_link_status = 1;
4159 /* guard against interrupt when we're going down */
4160 if (!test_bit(__IGB_DOWN, &adapter->state))
4161 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4162 }
4163
25568a53
AD		 4164 if (adapter->vfs_allocated_count)
4165 wr32(E1000_IMS, E1000_IMS_LSC |
4166 E1000_IMS_VMMB |
4167 E1000_IMS_DOUTSYNC);
4168 else
4169 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4170 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4171
4172 return IRQ_HANDLED;
4173}
4174
047e0030 4175static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4176{
26b39276 4177 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4178 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4179
047e0030
AD		 4180 if (!q_vector->set_itr)
4181 return;
73cd78f1 4182
047e0030
AD		 4183 if (!itr_val)
4184 itr_val = 0x4;
661086df 4185
26b39276
AD		 4186 if (adapter->hw.mac.type == e1000_82575)
4187 itr_val |= itr_val << 16;
661086df 4188 else
047e0030 4189 itr_val |= 0x8000000;
661086df 4190
047e0030
AD		 4191 writel(itr_val, q_vector->itr_register);
4192 q_vector->set_itr = 0;
6eb5a7f1
AD		 4193}
4194
047e0030 4195static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4196{
047e0030 4197 struct igb_q_vector *q_vector = data;
9d5c8243 4198
047e0030
AD		 4199 /* Write the ITR value calculated from the previous interrupt. */
4200 igb_write_itr(q_vector);
9d5c8243 4201
047e0030 4202 napi_schedule(&q_vector->napi);
844290e5 4203
047e0030 4204 return IRQ_HANDLED;
fe4506b6
JC		 4205}
4206
421e02f0 4207#ifdef CONFIG_IGB_DCA
047e0030 4208static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4209{
047e0030 4210 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4211 struct e1000_hw *hw = &adapter->hw;
4212 int cpu = get_cpu();
fe4506b6 4213
047e0030
AD		 4214 if (q_vector->cpu == cpu)
4215 goto out_no_update;
4216
4217 if (q_vector->tx_ring) {
4218 int q = q_vector->tx_ring->reg_idx;
4219 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4220 if (hw->mac.type == e1000_82575) {
4221 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4222 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4223 } else {
047e0030
AD		 4224 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4225 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4226 E1000_DCA_TXCTRL_CPUID_SHIFT;
4227 }
4228 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4229 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4230 }
4231 if (q_vector->rx_ring) {
4232 int q = q_vector->rx_ring->reg_idx;
4233 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4234 if (hw->mac.type == e1000_82575) {
2d064c06 4235 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4236 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4237 } else {
4238 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4239 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4240 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4241 }
fe4506b6
JC		 4242 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4243 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4244 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4245 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4246 }
047e0030
AD		 4247 q_vector->cpu = cpu;
4248out_no_update:
fe4506b6
JC		 4249 put_cpu();
4250}
4251
4252static void igb_setup_dca(struct igb_adapter *adapter)
4253{
7e0e99ef 4254 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4255 int i;
4256
7dfc16fa 4257 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4258 return;
4259
7e0e99ef
AD		 4260 /* Always use CB2 mode, difference is masked in the CB driver. */
4261 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4262
047e0030 4263 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4264 adapter->q_vector[i]->cpu = -1;
4265 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4266 }
4267}
4268
4269static int __igb_notify_dca(struct device *dev, void *data)
4270{
4271 struct net_device *netdev = dev_get_drvdata(dev);
4272 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4273 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4274 struct e1000_hw *hw = &adapter->hw;
4275 unsigned long event = *(unsigned long *)data;
4276
4277 switch (event) {
4278 case DCA_PROVIDER_ADD:
4279 /* if already enabled, don't do it again */
7dfc16fa 4280 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4281 break;
fe4506b6 4282 if (dca_add_requester(dev) == 0) {
bbd98fe4 4283 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4284 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4285 igb_setup_dca(adapter);
4286 break;
4287 }
4288 /* Fall Through since DCA is disabled. */
4289 case DCA_PROVIDER_REMOVE:
7dfc16fa 4290 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4291 /* without this a class_device is left
047e0030 4292 * hanging around in the sysfs model */
fe4506b6 4293 dca_remove_requester(dev);
090b1795 4294 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4295 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4296 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4297 }
4298 break;
4299 }
bbd98fe4 4300
fe4506b6 4301 return 0;
9d5c8243
AK		 4302}
4303
fe4506b6
JC		 4304static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4305 void *p)
4306{
4307 int ret_val;
4308
4309 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4310 __igb_notify_dca);
4311
4312 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4313}
421e02f0 4314#endif /* CONFIG_IGB_DCA */
9d5c8243 4315
4ae196df
AD		 4316static void igb_ping_all_vfs(struct igb_adapter *adapter)
4317{
4318 struct e1000_hw *hw = &adapter->hw;
4319 u32 ping;
4320 int i;
4321
4322 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4323 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4324 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4325 ping |= E1000_VT_MSGTYPE_CTS;
4326 igb_write_mbx(hw, &ping, 1, i);
4327 }
4328}
4329
7d5753f0
AD		 4330static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4331{
4332 struct e1000_hw *hw = &adapter->hw;
4333 u32 vmolr = rd32(E1000_VMOLR(vf));
4334 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4335
4336 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4337 IGB_VF_FLAG_MULTI_PROMISC);
4338 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4339
4340 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4341 vmolr |= E1000_VMOLR_MPME;
4342 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4343 } else {
4344 /*
4345 * if we have hashes and we are clearing a multicast promisc
4346 * flag we need to write the hashes to the MTA as this step
4347 * was previously skipped
4348 */
4349 if (vf_data->num_vf_mc_hashes > 30) {
4350 vmolr |= E1000_VMOLR_MPME;
4351 } else if (vf_data->num_vf_mc_hashes) {
4352 int j;
4353 vmolr |= E1000_VMOLR_ROMPE;
4354 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4355 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4356 }
4357 }
4358
4359 wr32(E1000_VMOLR(vf), vmolr);
4360
4361 /* there are flags left unprocessed, likely not supported */
4362 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4363 return -EINVAL;
4364
4365 return 0;
4366
4367}
4368
4ae196df
AD		 4369static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4370 u32 *msgbuf, u32 vf)
4371{
4372 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4373 u16 *hash_list = (u16 *)&msgbuf[1];
4374 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4375 int i;
4376
7d5753f0 4377 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4378 * to this VF for later use to restore when the PF multi cast
4379 * list changes
4380 */
4381 vf_data->num_vf_mc_hashes = n;
4382
7d5753f0
AD		 4383 /* only up to 30 hash values supported */
4384 if (n > 30)
4385 n = 30;
4386
4387 /* store the hashes for later use */
4ae196df 4388 for (i = 0; i < n; i++)
a419aef8 4389 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4390
4391 /* Flush and reset the mta with the new values */
ff41f8dc 4392 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4393
4394 return 0;
4395}
4396
4397static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4398{
4399 struct e1000_hw *hw = &adapter->hw;
4400 struct vf_data_storage *vf_data;
4401 int i, j;
4402
4403 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4404 u32 vmolr = rd32(E1000_VMOLR(i));
4405 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4406
4ae196df 4407 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4408
4409 if ((vf_data->num_vf_mc_hashes > 30) ||
4410 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4411 vmolr |= E1000_VMOLR_MPME;
4412 } else if (vf_data->num_vf_mc_hashes) {
4413 vmolr |= E1000_VMOLR_ROMPE;
4414 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4415 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4416 }
4417 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4418 }
4419}
4420
4421static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4422{
4423 struct e1000_hw *hw = &adapter->hw;
4424 u32 pool_mask, reg, vid;
4425 int i;
4426
4427 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4428
4429 /* Find the vlan filter for this id */
4430 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4431 reg = rd32(E1000_VLVF(i));
4432
4433 /* remove the vf from the pool */
4434 reg &= ~pool_mask;
4435
4436 /* if pool is empty then remove entry from vfta */
4437 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4438 (reg & E1000_VLVF_VLANID_ENABLE)) {
4439 reg = 0;
4440 vid = reg & E1000_VLVF_VLANID_MASK;
4441 igb_vfta_set(hw, vid, false);
4442 }
4443
4444 wr32(E1000_VLVF(i), reg);
4445 }
ae641bdc
AD		 4446
4447 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4448}
4449
4450static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4451{
4452 struct e1000_hw *hw = &adapter->hw;
4453 u32 reg, i;
4454
51466239
AD		 4455 /* The vlvf table only exists on 82576 hardware and newer */
4456 if (hw->mac.type < e1000_82576)
4457 return -1;
4458
4459 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4460 if (!adapter->vfs_allocated_count)
4461 return -1;
4462
4463 /* Find the vlan filter for this id */
4464 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4465 reg = rd32(E1000_VLVF(i));
4466 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4467 vid == (reg & E1000_VLVF_VLANID_MASK))
4468 break;
4469 }
4470
4471 if (add) {
4472 if (i == E1000_VLVF_ARRAY_SIZE) {
4473 /* Did not find a matching VLAN ID entry that was
4474 * enabled. Search for a free filter entry, i.e.
4475 * one without the enable bit set
4476 */
4477 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4478 reg = rd32(E1000_VLVF(i));
4479 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4480 break;
4481 }
4482 }
4483 if (i < E1000_VLVF_ARRAY_SIZE) {
4484 /* Found an enabled/available entry */
4485 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4486
4487 /* if !enabled we need to set this up in vfta */
4488 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4489 /* add VID to filter table */
4490 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4491 reg |= E1000_VLVF_VLANID_ENABLE;
4492 }
cad6d05f
AD		 4493 reg &= ~E1000_VLVF_VLANID_MASK;
4494 reg |= vid;
4ae196df 4495 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4496
4497 /* do not modify RLPML for PF devices */
4498 if (vf >= adapter->vfs_allocated_count)
4499 return 0;
4500
4501 if (!adapter->vf_data[vf].vlans_enabled) {
4502 u32 size;
4503 reg = rd32(E1000_VMOLR(vf));
4504 size = reg & E1000_VMOLR_RLPML_MASK;
4505 size += 4;
4506 reg &= ~E1000_VMOLR_RLPML_MASK;
4507 reg |= size;
4508 wr32(E1000_VMOLR(vf), reg);
4509 }
ae641bdc 4510
51466239 4511 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4512 return 0;
4513 }
4514 } else {
4515 if (i < E1000_VLVF_ARRAY_SIZE) {
4516 /* remove vf from the pool */
4517 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4518 /* if pool is empty then remove entry from vfta */
4519 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4520 reg = 0;
4521 igb_vfta_set(hw, vid, false);
4522 }
4523 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4524
4525 /* do not modify RLPML for PF devices */
4526 if (vf >= adapter->vfs_allocated_count)
4527 return 0;
4528
4529 adapter->vf_data[vf].vlans_enabled--;
4530 if (!adapter->vf_data[vf].vlans_enabled) {
4531 u32 size;
4532 reg = rd32(E1000_VMOLR(vf));
4533 size = reg & E1000_VMOLR_RLPML_MASK;
4534 size -= 4;
4535 reg &= ~E1000_VMOLR_RLPML_MASK;
4536 reg |= size;
4537 wr32(E1000_VMOLR(vf), reg);
4538 }
4ae196df
AD		 4539 }
4540 }
8151d294
WM		 4541 return 0;
4542}
4543
4544static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4545{
4546 struct e1000_hw *hw = &adapter->hw;
4547
4548 if (vid)
4549 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4550 else
4551 wr32(E1000_VMVIR(vf), 0);
4552}
4553
4554static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4555 int vf, u16 vlan, u8 qos)
4556{
4557 int err = 0;
4558 struct igb_adapter *adapter = netdev_priv(netdev);
4559
4560 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4561 return -EINVAL;
4562 if (vlan || qos) {
4563 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4564 if (err)
4565 goto out;
4566 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4567 igb_set_vmolr(adapter, vf, !vlan);
4568 adapter->vf_data[vf].pf_vlan = vlan;
4569 adapter->vf_data[vf].pf_qos = qos;
4570 dev_info(&adapter->pdev->dev,
4571 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4572 if (test_bit(__IGB_DOWN, &adapter->state)) {
4573 dev_warn(&adapter->pdev->dev,
4574 "The VF VLAN has been set,"
4575 " but the PF device is not up.\n");
4576 dev_warn(&adapter->pdev->dev,
4577 "Bring the PF device up before"
4578 " attempting to use the VF device.\n");
4579 }
4580 } else {
4581 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4582 false, vf);
4583 igb_set_vmvir(adapter, vlan, vf);
4584 igb_set_vmolr(adapter, vf, true);
4585 adapter->vf_data[vf].pf_vlan = 0;
4586 adapter->vf_data[vf].pf_qos = 0;
4587 }
4588out:
4589 return err;
4ae196df
AD		 4590}
4591
4592static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4593{
4594 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4595 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4596
4597 return igb_vlvf_set(adapter, vid, add, vf);
4598}
4599
f2ca0dbe 4600static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4601{
8151d294
WM		 4602 /* clear flags */
4603 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4604 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4605
4606 /* reset offloads to defaults */
8151d294 4607 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4608
4609 /* reset vlans for device */
4610 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4611 if (adapter->vf_data[vf].pf_vlan)
4612 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4613 adapter->vf_data[vf].pf_vlan,
4614 adapter->vf_data[vf].pf_qos);
4615 else
4616 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4617
4618 /* reset multicast table array for vf */
4619 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4620
4621 /* Flush and reset the mta with the new values */
ff41f8dc 4622 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4623}
4624
f2ca0dbe
AD		 4625static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4626{
4627 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4628
4629 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4630 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4631 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4632
4633 /* process remaining reset events */
4634 igb_vf_reset(adapter, vf);
4635}
4636
4637static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4638{
4639 struct e1000_hw *hw = &adapter->hw;
4640 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4641 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4642 u32 reg, msgbuf[3];
4643 u8 *addr = (u8 *)(&msgbuf[1]);
4644
4645 /* process all the same items cleared in a function level reset */
f2ca0dbe 4646 igb_vf_reset(adapter, vf);
4ae196df
AD		 4647
4648 /* set vf mac address */
26ad9178 4649 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4650
4651 /* enable transmit and receive for vf */
4652 reg = rd32(E1000_VFTE);
4653 wr32(E1000_VFTE, reg | (1 << vf));
4654 reg = rd32(E1000_VFRE);
4655 wr32(E1000_VFRE, reg | (1 << vf));
4656
f2ca0dbe 4657 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4658
4659 /* reply to reset with ack and vf mac address */
4660 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4661 memcpy(addr, vf_mac, 6);
4662 igb_write_mbx(hw, msgbuf, 3, vf);
4663}
4664
4665static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4666{
f2ca0dbe
AD		 4667 unsigned char *addr = (char *)&msg[1];
4668 int err = -1;
4ae196df 4669
f2ca0dbe
AD		 4670 if (is_valid_ether_addr(addr))
4671 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4672
f2ca0dbe 4673 return err;
4ae196df
AD		 4674}
4675
4676static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4677{
4678 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4679 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4680 u32 msg = E1000_VT_MSGTYPE_NACK;
4681
4682 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4683 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4684 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4685 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4686 vf_data->last_nack = jiffies;
4ae196df
AD		 4687 }
4688}
4689
f2ca0dbe 4690static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4691{
f2ca0dbe
AD		 4692 struct pci_dev *pdev = adapter->pdev;
4693 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4694 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4695 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4696 s32 retval;
4697
f2ca0dbe 4698 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 4699
fef45f4c
AD		 4700 if (retval) {
4701 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 4702 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 4703 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4704 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4705 return;
4706 goto out;
4707 }
4ae196df
AD		 4708
4709 /* this is a message we already processed, do nothing */
4710 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4711 return;
4ae196df
AD		 4712
4713 /*
4714 * until the vf completes a reset it should not be
4715 * allowed to start any configuration.
4716 */
4717
4718 if (msgbuf[0] == E1000_VF_RESET) {
4719 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4720 return;
4ae196df
AD		 4721 }
4722
f2ca0dbe 4723 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 4724 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4725 return;
4726 retval = -1;
4727 goto out;
4ae196df
AD		 4728 }
4729
4730 switch ((msgbuf[0] & 0xFFFF)) {
4731 case E1000_VF_SET_MAC_ADDR:
4732 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4733 break;
7d5753f0
AD		 4734 case E1000_VF_SET_PROMISC:
4735 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4736 break;
4ae196df
AD		 4737 case E1000_VF_SET_MULTICAST:
4738 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4739 break;
4740 case E1000_VF_SET_LPE:
4741 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4742 break;
4743 case E1000_VF_SET_VLAN:
8151d294
WM		 4744 if (adapter->vf_data[vf].pf_vlan)
4745 retval = -1;
4746 else
4747 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 4748 break;
4749 default:
090b1795 4750 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4751 retval = -1;
4752 break;
4753 }
4754
fef45f4c
AD		 4755 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4756out:
4ae196df
AD		 4757 /* notify the VF of the results of what it sent us */
4758 if (retval)
4759 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4760 else
4761 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4762
4ae196df 4763 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4764}
4ae196df 4765
f2ca0dbe
AD		 4766static void igb_msg_task(struct igb_adapter *adapter)
4767{
4768 struct e1000_hw *hw = &adapter->hw;
4769 u32 vf;
4770
4771 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4772 /* process any reset requests */
4773 if (!igb_check_for_rst(hw, vf))
4774 igb_vf_reset_event(adapter, vf);
4775
4776 /* process any messages pending */
4777 if (!igb_check_for_msg(hw, vf))
4778 igb_rcv_msg_from_vf(adapter, vf);
4779
4780 /* process any acks */
4781 if (!igb_check_for_ack(hw, vf))
4782 igb_rcv_ack_from_vf(adapter, vf);
4783 }
4ae196df
AD		 4784}
4785
68d480c4
AD		 4786/**
4787 * igb_set_uta - Set unicast filter table address
4788 * @adapter: board private structure
4789 *
4790 * The unicast table address is a register array of 32-bit registers.
4791 * The table is meant to be used in a way similar to how the MTA is used
4792 * however due to certain limitations in the hardware it is necessary to
4793 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4794 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4795 **/
4796static void igb_set_uta(struct igb_adapter *adapter)
4797{
4798 struct e1000_hw *hw = &adapter->hw;
4799 int i;
4800
4801 /* The UTA table only exists on 82576 hardware and newer */
4802 if (hw->mac.type < e1000_82576)
4803 return;
4804
4805 /* we only need to do this if VMDq is enabled */
4806 if (!adapter->vfs_allocated_count)
4807 return;
4808
4809 for (i = 0; i < hw->mac.uta_reg_count; i++)
4810 array_wr32(E1000_UTA, i, ~0);
4811}
4812
9d5c8243
AK		 4813/**
4814 * igb_intr_msi - Interrupt Handler
4815 * @irq: interrupt number
4816 * @data: pointer to a network interface device structure
4817 **/
4818static irqreturn_t igb_intr_msi(int irq, void *data)
4819{
047e0030
AD		 4820 struct igb_adapter *adapter = data;
4821 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4822 struct e1000_hw *hw = &adapter->hw;
4823 /* read ICR disables interrupts using IAM */
4824 u32 icr = rd32(E1000_ICR);
4825
047e0030 4826 igb_write_itr(q_vector);
9d5c8243 4827
7f081d40
AD		 4828 if (icr & E1000_ICR_DRSTA)
4829 schedule_work(&adapter->reset_task);
4830
047e0030 4831 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4832 /* HW is reporting DMA is out of sync */
4833 adapter->stats.doosync++;
4834 }
4835
9d5c8243
AK		 4836 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4837 hw->mac.get_link_status = 1;
4838 if (!test_bit(__IGB_DOWN, &adapter->state))
4839 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4840 }
4841
047e0030 4842 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4843
4844 return IRQ_HANDLED;
4845}
4846
4847/**
4a3c6433 4848 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4849 * @irq: interrupt number
4850 * @data: pointer to a network interface device structure
4851 **/
4852static irqreturn_t igb_intr(int irq, void *data)
4853{
047e0030
AD		 4854 struct igb_adapter *adapter = data;
4855 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4856 struct e1000_hw *hw = &adapter->hw;
4857 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4858 * need for the IMC write */
4859 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4860 if (!icr)
4861 return IRQ_NONE; /* Not our interrupt */
4862
047e0030 4863 igb_write_itr(q_vector);
9d5c8243
AK		 4864
4865 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4866 * not set, then the adapter didn't send an interrupt */
4867 if (!(icr & E1000_ICR_INT_ASSERTED))
4868 return IRQ_NONE;
4869
7f081d40
AD		 4870 if (icr & E1000_ICR_DRSTA)
4871 schedule_work(&adapter->reset_task);
4872
047e0030 4873 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4874 /* HW is reporting DMA is out of sync */
4875 adapter->stats.doosync++;
4876 }
4877
9d5c8243
AK		 4878 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4879 hw->mac.get_link_status = 1;
4880 /* guard against interrupt when we're going down */
4881 if (!test_bit(__IGB_DOWN, &adapter->state))
4882 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4883 }
4884
047e0030 4885 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4886
4887 return IRQ_HANDLED;
4888}
4889
047e0030 4890static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4891{
047e0030 4892 struct igb_adapter *adapter = q_vector->adapter;
46544258 4893 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4894
4fc82adf
AD		 4895 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4896 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4897 if (!adapter->msix_entries)
6eb5a7f1 4898 igb_set_itr(adapter);
46544258 4899 else
047e0030 4900 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4901 }
4902
46544258
AD		 4903 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4904 if (adapter->msix_entries)
047e0030 4905 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4906 else
4907 igb_irq_enable(adapter);
4908 }
9d5c8243
AK		 4909}
4910
46544258
AD		 4911/**
4912 * igb_poll - NAPI Rx polling callback
4913 * @napi: napi polling structure
4914 * @budget: count of how many packets we should handle
4915 **/
4916static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4917{
047e0030
AD		 4918 struct igb_q_vector *q_vector = container_of(napi,
4919 struct igb_q_vector,
4920 napi);
4921 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4922
421e02f0 4923#ifdef CONFIG_IGB_DCA
047e0030
AD		 4924 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4925 igb_update_dca(q_vector);
fe4506b6 4926#endif
047e0030
AD		 4927 if (q_vector->tx_ring)
4928 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4929
047e0030
AD		 4930 if (q_vector->rx_ring)
4931 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4932
4933 if (!tx_clean_complete)
4934 work_done = budget;
46544258 4935
9d5c8243 4936 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4937 if (work_done < budget) {
288379f0 4938 napi_complete(napi);
047e0030 4939 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4940 }
4941
46544258 4942 return work_done;
9d5c8243 4943}
6d8126f9 4944
33af6bcc 4945/**
c5b9bd5e 4946 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4947 * @adapter: board private structure
c5b9bd5e
AD		 4948 * @shhwtstamps: timestamp structure to update
4949 * @regval: unsigned 64bit system time value.
4950 *
4951 * We need to convert the system time value stored in the RX/TXSTMP registers
4952 * into a hwtstamp which can be used by the upper level timestamping functions
4953 */
4954static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4955 struct skb_shared_hwtstamps *shhwtstamps,
4956 u64 regval)
4957{
4958 u64 ns;
4959
55cac248
AD		 4960 /*
4961 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4962 * 24 to match clock shift we setup earlier.
4963 */
4964 if (adapter->hw.mac.type == e1000_82580)
4965 regval <<= IGB_82580_TSYNC_SHIFT;
4966
c5b9bd5e
AD		 4967 ns = timecounter_cyc2time(&adapter->clock, regval);
4968 timecompare_update(&adapter->compare, ns);
4969 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4970 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4971 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4972}
4973
4974/**
4975 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4976 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4977 * @skb: packet that was just sent
4978 *
4979 * If we were asked to do hardware stamping and such a time stamp is
4980 * available, then it must have been for this skb here because we only
4981 * allow only one such packet into the queue.
4982 */
c5b9bd5e 4983static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4984{
c5b9bd5e 4985 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4986 union skb_shared_tx *shtx = skb_tx(skb);
4987 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4988 struct skb_shared_hwtstamps shhwtstamps;
4989 u64 regval;
33af6bcc 4990
c5b9bd5e
AD		 4991 /* if skb does not support hw timestamp or TX stamp not valid exit */
4992 if (likely(!shtx->hardware) ||
4993 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4994 return;
4995
4996 regval = rd32(E1000_TXSTMPL);
4997 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4998
4999 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5000 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 5001}
5002
9d5c8243
AK		 5003/**
5004 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5005 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5006 * returns true if ring is completely cleaned
5007 **/
047e0030 5008static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5009{
047e0030
AD		 5010 struct igb_adapter *adapter = q_vector->adapter;
5011 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5012 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5013 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 5014 struct igb_buffer *buffer_info;
5015 struct sk_buff *skb;
0e014cb1 5016 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5017 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5018 unsigned int i, eop, count = 0;
5019 bool cleaned = false;
9d5c8243 5020
9d5c8243 5021 i = tx_ring->next_to_clean;
0e014cb1
AD		 5022 eop = tx_ring->buffer_info[i].next_to_watch;
5023 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5024
5025 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5026 (count < tx_ring->count)) {
5027 for (cleaned = false; !cleaned; count++) {
5028 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5029 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5030 cleaned = (i == eop);
9d5c8243
AK		 5031 skb = buffer_info->skb;
5032
5033 if (skb) {
5034 unsigned int segs, bytecount;
5035 /* gso_segs is currently only valid for tcp */
40e90c26 5036 segs = buffer_info->gso_segs;
9d5c8243
AK		 5037 /* multiply data chunks by size of headers */
5038 bytecount = ((segs - 1) * skb_headlen(skb)) +
5039 skb->len;
5040 total_packets += segs;
5041 total_bytes += bytecount;
33af6bcc 5042
c5b9bd5e 5043 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 5044 }
5045
80785298 5046 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5047 tx_desc->wb.status = 0;
9d5c8243
AK		 5048
5049 i++;
5050 if (i == tx_ring->count)
5051 i = 0;
9d5c8243 5052 }
0e014cb1
AD		 5053 eop = tx_ring->buffer_info[i].next_to_watch;
5054 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5055 }
5056
9d5c8243
AK		 5057 tx_ring->next_to_clean = i;
5058
fc7d345d 5059 if (unlikely(count &&
9d5c8243 5060 netif_carrier_ok(netdev) &&
c493ea45 5061 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5062 /* Make sure that anybody stopping the queue after this
5063 * sees the new next_to_clean.
5064 */
5065 smp_mb();
661086df
PWJ		 5066 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5067 !(test_bit(__IGB_DOWN, &adapter->state))) {
5068 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5069 tx_ring->tx_stats.restart_queue++;
661086df 5070 }
9d5c8243
AK		 5071 }
5072
5073 if (tx_ring->detect_tx_hung) {
5074 /* Detect a transmit hang in hardware, this serializes the
5075 * check with the clearing of time_stamp and movement of i */
5076 tx_ring->detect_tx_hung = false;
5077 if (tx_ring->buffer_info[i].time_stamp &&
5078 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5079 (adapter->tx_timeout_factor * HZ)) &&
5080 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5081
9d5c8243 5082 /* detected Tx unit hang */
80785298 5083 dev_err(&tx_ring->pdev->dev,
9d5c8243 5084 "Detected Tx Unit Hang\n"
2d064c06 5085 " Tx Queue <%d>\n"
9d5c8243
AK		 5086 " TDH <%x>\n"
5087 " TDT <%x>\n"
5088 " next_to_use <%x>\n"
5089 " next_to_clean <%x>\n"
9d5c8243
AK		 5090 "buffer_info[next_to_clean]\n"
5091 " time_stamp <%lx>\n"
0e014cb1 5092 " next_to_watch <%x>\n"
9d5c8243
AK		 5093 " jiffies <%lx>\n"
5094 " desc.status <%x>\n",
2d064c06 5095 tx_ring->queue_index,
fce99e34
AD		 5096 readl(tx_ring->head),
5097 readl(tx_ring->tail),
9d5c8243
AK		 5098 tx_ring->next_to_use,
5099 tx_ring->next_to_clean,
f7ba205e 5100 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5101 eop,
9d5c8243 5102 jiffies,
0e014cb1 5103 eop_desc->wb.status);
661086df 5104 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5105 }
5106 }
5107 tx_ring->total_bytes += total_bytes;
5108 tx_ring->total_packets += total_packets;
e21ed353
AD		 5109 tx_ring->tx_stats.bytes += total_bytes;
5110 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5111 return (count < tx_ring->count);
9d5c8243
AK		 5112}
5113
9d5c8243
AK		 5114/**
5115 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5116 * @q_vector: structure containing interrupt and ring information
5117 * @skb: packet to send up
5118 * @vlan_tag: vlan tag for packet
9d5c8243 5119 **/
047e0030
AD		 5120static void igb_receive_skb(struct igb_q_vector *q_vector,
5121 struct sk_buff *skb,
5122 u16 vlan_tag)
5123{
5124 struct igb_adapter *adapter = q_vector->adapter;
5125
5126 if (vlan_tag)
5127 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5128 vlan_tag, skb);
182ff8df 5129 else
047e0030 5130 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5131}
5132
04a5fcaa 5133static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5134 u32 status_err, struct sk_buff *skb)
5135{
5136 skb->ip_summed = CHECKSUM_NONE;
5137
5138 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5139 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5140 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5141 return;
85ad76b2 5142
9d5c8243
AK		 5143 /* TCP/UDP checksum error bit is set */
5144 if (status_err &
5145 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5146 /*
5147 * work around errata with sctp packets where the TCPE aka
5148 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5149 * packets, (aka let the stack check the crc32c)
5150 */
85ad76b2
AD		 5151 if ((skb->len == 60) &&
5152 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5153 ring->rx_stats.csum_err++;
85ad76b2 5154
9d5c8243 5155 /* let the stack verify checksum errors */
9d5c8243
AK		 5156 return;
5157 }
5158 /* It must be a TCP or UDP packet with a valid checksum */
5159 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5160 skb->ip_summed = CHECKSUM_UNNECESSARY;
5161
85ad76b2 5162 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5163}
5164
c5b9bd5e
AD		 5165static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5166 struct sk_buff *skb)
5167{
5168 struct igb_adapter *adapter = q_vector->adapter;
5169 struct e1000_hw *hw = &adapter->hw;
5170 u64 regval;
5171
5172 /*
5173 * If this bit is set, then the RX registers contain the time stamp. No
5174 * other packet will be time stamped until we read these registers, so
5175 * read the registers to make them available again. Because only one
5176 * packet can be time stamped at a time, we know that the register
5177 * values must belong to this one here and therefore we don't need to
5178 * compare any of the additional attributes stored for it.
5179 *
5180 * If nothing went wrong, then it should have a skb_shared_tx that we
5181 * can turn into a skb_shared_hwtstamps.
5182 */
5183 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5184 return;
5185 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5186 return;
5187
5188 regval = rd32(E1000_RXSTMPL);
5189 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5190
5191 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5192}
4c844851 5193static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5194 union e1000_adv_rx_desc *rx_desc)
5195{
5196 /* HW will not DMA in data larger than the given buffer, even if it
5197 * parses the (NFS, of course) header to be larger. In that case, it
5198 * fills the header buffer and spills the rest into the page.
5199 */
5200 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5201 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5202 if (hlen > rx_ring->rx_buffer_len)
5203 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5204 return hlen;
5205}
5206
047e0030
AD		 5207static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5208 int *work_done, int budget)
9d5c8243 5209{
047e0030 5210 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5211 struct net_device *netdev = rx_ring->netdev;
80785298 5212 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 5213 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5214 struct igb_buffer *buffer_info , *next_buffer;
5215 struct sk_buff *skb;
9d5c8243
AK		 5216 bool cleaned = false;
5217 int cleaned_count = 0;
d1eff350 5218 int current_node = numa_node_id();
9d5c8243 5219 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5220 unsigned int i;
2d94d8ab
AD		 5221 u32 staterr;
5222 u16 length;
047e0030 5223 u16 vlan_tag;
9d5c8243
AK		 5224
5225 i = rx_ring->next_to_clean;
69d3ca53 5226 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5227 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5228 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5229
5230 while (staterr & E1000_RXD_STAT_DD) {
5231 if (*work_done >= budget)
5232 break;
5233 (*work_done)++;
9d5c8243 5234
69d3ca53
AD		 5235 skb = buffer_info->skb;
5236 prefetch(skb->data - NET_IP_ALIGN);
5237 buffer_info->skb = NULL;
5238
5239 i++;
5240 if (i == rx_ring->count)
5241 i = 0;
42d0781a 5242
69d3ca53
AD		 5243 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5244 prefetch(next_rxd);
5245 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5246
5247 length = le16_to_cpu(rx_desc->wb.upper.length);
5248 cleaned = true;
5249 cleaned_count++;
5250
2d94d8ab 5251 if (buffer_info->dma) {
bf36c1a0 5252 pci_unmap_single(pdev, buffer_info->dma,
4c844851 5253 rx_ring->rx_buffer_len,
bf36c1a0 5254 PCI_DMA_FROMDEVICE);
91615f76 5255 buffer_info->dma = 0;
4c844851 5256 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5257 skb_put(skb, length);
5258 goto send_up;
5259 }
4c844851 5260 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5261 }
5262
5263 if (length) {
9d5c8243 5264 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 5265 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 5266 buffer_info->page_dma = 0;
bf36c1a0
AD		 5267
5268 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5269 buffer_info->page,
5270 buffer_info->page_offset,
5271 length);
5272
d1eff350
AD		 5273 if ((page_count(buffer_info->page) != 1) ||
5274 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5275 buffer_info->page = NULL;
5276 else
5277 get_page(buffer_info->page);
9d5c8243
AK		 5278
5279 skb->len += length;
5280 skb->data_len += length;
bf36c1a0 5281 skb->truesize += length;
9d5c8243 5282 }
9d5c8243 5283
bf36c1a0 5284 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5285 buffer_info->skb = next_buffer->skb;
5286 buffer_info->dma = next_buffer->dma;
5287 next_buffer->skb = skb;
5288 next_buffer->dma = 0;
bf36c1a0
AD		 5289 goto next_desc;
5290 }
69d3ca53 5291send_up:
9d5c8243
AK		 5292 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5293 dev_kfree_skb_irq(skb);
5294 goto next_desc;
5295 }
9d5c8243 5296
c5b9bd5e 5297 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5298 total_bytes += skb->len;
5299 total_packets++;
5300
85ad76b2 5301 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5302
5303 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5304 skb_record_rx_queue(skb, rx_ring->queue_index);
5305
5306 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5307 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5308
047e0030 5309 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5310
9d5c8243
AK		 5311next_desc:
5312 rx_desc->wb.upper.status_error = 0;
5313
5314 /* return some buffers to hardware, one at a time is too slow */
5315 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5316 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5317 cleaned_count = 0;
5318 }
5319
5320 /* use prefetched values */
5321 rx_desc = next_rxd;
5322 buffer_info = next_buffer;
9d5c8243
AK		 5323 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5324 }
bf36c1a0 5325
9d5c8243 5326 rx_ring->next_to_clean = i;
c493ea45 5327 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5328
5329 if (cleaned_count)
3b644cf6 5330 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5331
5332 rx_ring->total_packets += total_packets;
5333 rx_ring->total_bytes += total_bytes;
5334 rx_ring->rx_stats.packets += total_packets;
5335 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5336 return cleaned;
5337}
5338
9d5c8243
AK		 5339/**
5340 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5341 * @adapter: address of board private structure
5342 **/
d7ee5b3a 5343void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5344{
e694e964 5345 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5346 union e1000_adv_rx_desc *rx_desc;
5347 struct igb_buffer *buffer_info;
5348 struct sk_buff *skb;
5349 unsigned int i;
db761762 5350 int bufsz;
9d5c8243
AK		 5351
5352 i = rx_ring->next_to_use;
5353 buffer_info = &rx_ring->buffer_info[i];
5354
4c844851 5355 bufsz = rx_ring->rx_buffer_len;
db761762 5356
9d5c8243
AK		 5357 while (cleaned_count--) {
5358 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5359
6ec43fe6 5360 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5361 if (!buffer_info->page) {
42d0781a 5362 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5363 if (!buffer_info->page) {
04a5fcaa 5364 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5365 goto no_buffers;
5366 }
5367 buffer_info->page_offset = 0;
5368 } else {
5369 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5370 }
5371 buffer_info->page_dma =
80785298 5372 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5373 buffer_info->page_offset,
5374 PAGE_SIZE / 2,
9d5c8243 5375 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5376 if (pci_dma_mapping_error(rx_ring->pdev,
5377 buffer_info->page_dma)) {
5378 buffer_info->page_dma = 0;
5379 rx_ring->rx_stats.alloc_failed++;
5380 goto no_buffers;
5381 }
9d5c8243
AK		 5382 }
5383
42d0781a
AD		 5384 skb = buffer_info->skb;
5385 if (!skb) {
89d71a66 5386 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5387 if (!skb) {
04a5fcaa 5388 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5389 goto no_buffers;
5390 }
5391
9d5c8243 5392 buffer_info->skb = skb;
42d0781a
AD		 5393 }
5394 if (!buffer_info->dma) {
80785298
AD		 5395 buffer_info->dma = pci_map_single(rx_ring->pdev,
5396 skb->data,
9d5c8243
AK		 5397 bufsz,
5398 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5399 if (pci_dma_mapping_error(rx_ring->pdev,
5400 buffer_info->dma)) {
5401 buffer_info->dma = 0;
5402 rx_ring->rx_stats.alloc_failed++;
5403 goto no_buffers;
5404 }
9d5c8243
AK		 5405 }
5406 /* Refresh the desc even if buffer_addrs didn't change because
5407 * each write-back erases this info. */
6ec43fe6 5408 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5409 rx_desc->read.pkt_addr =
5410 cpu_to_le64(buffer_info->page_dma);
5411 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5412 } else {
42d0781a 5413 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5414 rx_desc->read.hdr_addr = 0;
5415 }
5416
5417 i++;
5418 if (i == rx_ring->count)
5419 i = 0;
5420 buffer_info = &rx_ring->buffer_info[i];
5421 }
5422
5423no_buffers:
5424 if (rx_ring->next_to_use != i) {
5425 rx_ring->next_to_use = i;
5426 if (i == 0)
5427 i = (rx_ring->count - 1);
5428 else
5429 i--;
5430
5431 /* Force memory writes to complete before letting h/w
5432 * know there are new descriptors to fetch. (Only
5433 * applicable for weak-ordered memory model archs,
5434 * such as IA-64). */
5435 wmb();
fce99e34 5436 writel(i, rx_ring->tail);
9d5c8243
AK		 5437 }
5438}
5439
5440/**
5441 * igb_mii_ioctl -
5442 * @netdev:
5443 * @ifreq:
5444 * @cmd:
5445 **/
5446static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5447{
5448 struct igb_adapter *adapter = netdev_priv(netdev);
5449 struct mii_ioctl_data *data = if_mii(ifr);
5450
5451 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5452 return -EOPNOTSUPP;
5453
5454 switch (cmd) {
5455 case SIOCGMIIPHY:
5456 data->phy_id = adapter->hw.phy.addr;
5457 break;
5458 case SIOCGMIIREG:
f5f4cf08
AD		 5459 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5460 &data->val_out))
9d5c8243
AK		 5461 return -EIO;
5462 break;
5463 case SIOCSMIIREG:
5464 default:
5465 return -EOPNOTSUPP;
5466 }
5467 return 0;
5468}
5469
c6cb090b
PO		 5470/**
5471 * igb_hwtstamp_ioctl - control hardware time stamping
5472 * @netdev:
5473 * @ifreq:
5474 * @cmd:
5475 *
33af6bcc
PO		 5476 * Outgoing time stamping can be enabled and disabled. Play nice and
5477 * disable it when requested, although it shouldn't case any overhead
5478 * when no packet needs it. At most one packet in the queue may be
5479 * marked for time stamping, otherwise it would be impossible to tell
5480 * for sure to which packet the hardware time stamp belongs.
5481 *
5482 * Incoming time stamping has to be configured via the hardware
5483 * filters. Not all combinations are supported, in particular event
5484 * type has to be specified. Matching the kind of event packet is
5485 * not supported, with the exception of "all V2 events regardless of
5486 * level 2 or 4".
5487 *
c6cb090b
PO		 5488 **/
5489static int igb_hwtstamp_ioctl(struct net_device *netdev,
5490 struct ifreq *ifr, int cmd)
5491{
33af6bcc
PO		 5492 struct igb_adapter *adapter = netdev_priv(netdev);
5493 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5494 struct hwtstamp_config config;
c5b9bd5e
AD		 5495 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5496 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5497 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5498 bool is_l4 = false;
5499 bool is_l2 = false;
33af6bcc 5500 u32 regval;
c6cb090b
PO		 5501
5502 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5503 return -EFAULT;
5504
5505 /* reserved for future extensions */
5506 if (config.flags)
5507 return -EINVAL;
5508
33af6bcc
PO		 5509 switch (config.tx_type) {
5510 case HWTSTAMP_TX_OFF:
c5b9bd5e 5511 tsync_tx_ctl = 0;
33af6bcc 5512 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5513 break;
5514 default:
5515 return -ERANGE;
5516 }
5517
5518 switch (config.rx_filter) {
5519 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5520 tsync_rx_ctl = 0;
33af6bcc
PO		 5521 break;
5522 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5523 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5524 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5525 case HWTSTAMP_FILTER_ALL:
5526 /*
5527 * register TSYNCRXCFG must be set, therefore it is not
5528 * possible to time stamp both Sync and Delay_Req messages
5529 * => fall back to time stamping all packets
5530 */
c5b9bd5e 5531 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5532 config.rx_filter = HWTSTAMP_FILTER_ALL;
5533 break;
5534 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5535 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5536 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5537 is_l4 = true;
33af6bcc
PO		 5538 break;
5539 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5540 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5541 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5542 is_l4 = true;
33af6bcc
PO		 5543 break;
5544 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5545 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5546 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5547 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5548 is_l2 = true;
5549 is_l4 = true;
33af6bcc
PO		 5550 config.rx_filter = HWTSTAMP_FILTER_SOME;
5551 break;
5552 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5553 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5554 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5555 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5556 is_l2 = true;
5557 is_l4 = true;
33af6bcc
PO		 5558 config.rx_filter = HWTSTAMP_FILTER_SOME;
5559 break;
5560 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5561 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5562 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5563 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5564 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5565 is_l2 = true;
33af6bcc
PO		 5566 break;
5567 default:
5568 return -ERANGE;
5569 }
5570
c5b9bd5e
AD		 5571 if (hw->mac.type == e1000_82575) {
5572 if (tsync_rx_ctl | tsync_tx_ctl)
5573 return -EINVAL;
5574 return 0;
5575 }
5576
33af6bcc
PO		 5577 /* enable/disable TX */
5578 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5579 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5580 regval |= tsync_tx_ctl;
33af6bcc
PO		 5581 wr32(E1000_TSYNCTXCTL, regval);
5582
c5b9bd5e 5583 /* enable/disable RX */
33af6bcc 5584 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5585 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5586 regval |= tsync_rx_ctl;
33af6bcc 5587 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5588
c5b9bd5e
AD		 5589 /* define which PTP packets are time stamped */
5590 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5591
c5b9bd5e
AD		 5592 /* define ethertype filter for timestamped packets */
5593 if (is_l2)
5594 wr32(E1000_ETQF(3),
5595 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5596 E1000_ETQF_1588 | /* enable timestamping */
5597 ETH_P_1588)); /* 1588 eth protocol type */
5598 else
5599 wr32(E1000_ETQF(3), 0);
5600
5601#define PTP_PORT 319
5602 /* L4 Queue Filter[3]: filter by destination port and protocol */
5603 if (is_l4) {
5604 u32 ftqf = (IPPROTO_UDP /* UDP */
5605 | E1000_FTQF_VF_BP /* VF not compared */
5606 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5607 | E1000_FTQF_MASK); /* mask all inputs */
5608 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5609
5610 wr32(E1000_IMIR(3), htons(PTP_PORT));
5611 wr32(E1000_IMIREXT(3),
5612 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5613 if (hw->mac.type == e1000_82576) {
5614 /* enable source port check */
5615 wr32(E1000_SPQF(3), htons(PTP_PORT));
5616 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5617 }
5618 wr32(E1000_FTQF(3), ftqf);
5619 } else {
5620 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5621 }
33af6bcc
PO		 5622 wrfl();
5623
5624 adapter->hwtstamp_config = config;
5625
5626 /* clear TX/RX time stamp registers, just to be sure */
5627 regval = rd32(E1000_TXSTMPH);
5628 regval = rd32(E1000_RXSTMPH);
c6cb090b 5629
33af6bcc
PO		 5630 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5631 -EFAULT : 0;
c6cb090b
PO		 5632}
5633
9d5c8243
AK		 5634/**
5635 * igb_ioctl -
5636 * @netdev:
5637 * @ifreq:
5638 * @cmd:
5639 **/
5640static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5641{
5642 switch (cmd) {
5643 case SIOCGMIIPHY:
5644 case SIOCGMIIREG:
5645 case SIOCSMIIREG:
5646 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5647 case SIOCSHWTSTAMP:
5648 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5649 default:
5650 return -EOPNOTSUPP;
5651 }
5652}
5653
009bc06e
AD		 5654s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5655{
5656 struct igb_adapter *adapter = hw->back;
5657 u16 cap_offset;
5658
5659 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5660 if (!cap_offset)
5661 return -E1000_ERR_CONFIG;
5662
5663 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5664
5665 return 0;
5666}
5667
5668s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5669{
5670 struct igb_adapter *adapter = hw->back;
5671 u16 cap_offset;
5672
5673 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5674 if (!cap_offset)
5675 return -E1000_ERR_CONFIG;
5676
5677 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5678
5679 return 0;
5680}
5681
9d5c8243
AK		 5682static void igb_vlan_rx_register(struct net_device *netdev,
5683 struct vlan_group *grp)
5684{
5685 struct igb_adapter *adapter = netdev_priv(netdev);
5686 struct e1000_hw *hw = &adapter->hw;
5687 u32 ctrl, rctl;
5688
5689 igb_irq_disable(adapter);
5690 adapter->vlgrp = grp;
5691
5692 if (grp) {
5693 /* enable VLAN tag insert/strip */
5694 ctrl = rd32(E1000_CTRL);
5695 ctrl |= E1000_CTRL_VME;
5696 wr32(E1000_CTRL, ctrl);
5697
51466239 5698 /* Disable CFI check */
9d5c8243 5699 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5700 rctl &= ~E1000_RCTL_CFIEN;
5701 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5702 } else {
5703 /* disable VLAN tag insert/strip */
5704 ctrl = rd32(E1000_CTRL);
5705 ctrl &= ~E1000_CTRL_VME;
5706 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5707 }
5708
e1739522
AD		 5709 igb_rlpml_set(adapter);
5710
9d5c8243
AK		 5711 if (!test_bit(__IGB_DOWN, &adapter->state))
5712 igb_irq_enable(adapter);
5713}
5714
5715static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5716{
5717 struct igb_adapter *adapter = netdev_priv(netdev);
5718 struct e1000_hw *hw = &adapter->hw;
4ae196df 5719 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5720
51466239
AD		 5721 /* attempt to add filter to vlvf array */
5722 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5723
51466239
AD		 5724 /* add the filter since PF can receive vlans w/o entry in vlvf */
5725 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5726}
5727
5728static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5729{
5730 struct igb_adapter *adapter = netdev_priv(netdev);
5731 struct e1000_hw *hw = &adapter->hw;
4ae196df 5732 int pf_id = adapter->vfs_allocated_count;
51466239 5733 s32 err;
9d5c8243
AK		 5734
5735 igb_irq_disable(adapter);
5736 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5737
5738 if (!test_bit(__IGB_DOWN, &adapter->state))
5739 igb_irq_enable(adapter);
5740
51466239
AD		 5741 /* remove vlan from VLVF table array */
5742 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5743
51466239
AD		 5744 /* if vid was not present in VLVF just remove it from table */
5745 if (err)
4ae196df 5746 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5747}
5748
5749static void igb_restore_vlan(struct igb_adapter *adapter)
5750{
5751 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5752
5753 if (adapter->vlgrp) {
5754 u16 vid;
5755 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5756 if (!vlan_group_get_device(adapter->vlgrp, vid))
5757 continue;
5758 igb_vlan_rx_add_vid(adapter->netdev, vid);
5759 }
5760 }
5761}
5762
5763int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5764{
090b1795 5765 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5766 struct e1000_mac_info *mac = &adapter->hw.mac;
5767
5768 mac->autoneg = 0;
5769
9d5c8243
AK		 5770 switch (spddplx) {
5771 case SPEED_10 + DUPLEX_HALF:
5772 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5773 break;
5774 case SPEED_10 + DUPLEX_FULL:
5775 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5776 break;
5777 case SPEED_100 + DUPLEX_HALF:
5778 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5779 break;
5780 case SPEED_100 + DUPLEX_FULL:
5781 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5782 break;
5783 case SPEED_1000 + DUPLEX_FULL:
5784 mac->autoneg = 1;
5785 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5786 break;
5787 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5788 default:
090b1795 5789 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5790 return -EINVAL;
5791 }
5792 return 0;
5793}
5794
3fe7c4c9 5795static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5796{
5797 struct net_device *netdev = pci_get_drvdata(pdev);
5798 struct igb_adapter *adapter = netdev_priv(netdev);
5799 struct e1000_hw *hw = &adapter->hw;
2d064c06 5800 u32 ctrl, rctl, status;
9d5c8243
AK		 5801 u32 wufc = adapter->wol;
5802#ifdef CONFIG_PM
5803 int retval = 0;
5804#endif
5805
5806 netif_device_detach(netdev);
5807
a88f10ec
AD		 5808 if (netif_running(netdev))
5809 igb_close(netdev);
5810
047e0030 5811 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5812
5813#ifdef CONFIG_PM
5814 retval = pci_save_state(pdev);
5815 if (retval)
5816 return retval;
5817#endif
5818
5819 status = rd32(E1000_STATUS);
5820 if (status & E1000_STATUS_LU)
5821 wufc &= ~E1000_WUFC_LNKC;
5822
5823 if (wufc) {
5824 igb_setup_rctl(adapter);
ff41f8dc 5825 igb_set_rx_mode(netdev);
9d5c8243
AK		 5826
5827 /* turn on all-multi mode if wake on multicast is enabled */
5828 if (wufc & E1000_WUFC_MC) {
5829 rctl = rd32(E1000_RCTL);
5830 rctl |= E1000_RCTL_MPE;
5831 wr32(E1000_RCTL, rctl);
5832 }
5833
5834 ctrl = rd32(E1000_CTRL);
5835 /* advertise wake from D3Cold */
5836 #define E1000_CTRL_ADVD3WUC 0x00100000
5837 /* phy power management enable */
5838 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5839 ctrl |= E1000_CTRL_ADVD3WUC;
5840 wr32(E1000_CTRL, ctrl);
5841
9d5c8243 5842 /* Allow time for pending master requests to run */
330a6d6a 5843 igb_disable_pcie_master(hw);
9d5c8243
AK		 5844
5845 wr32(E1000_WUC, E1000_WUC_PME_EN);
5846 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5847 } else {
5848 wr32(E1000_WUC, 0);
5849 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5850 }
5851
3fe7c4c9
RW		 5852 *enable_wake = wufc || adapter->en_mng_pt;
5853 if (!*enable_wake)
88a268c1
NN		 5854 igb_power_down_link(adapter);
5855 else
5856 igb_power_up_link(adapter);
9d5c8243
AK		 5857
5858 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5859 * would have already happened in close and is redundant. */
5860 igb_release_hw_control(adapter);
5861
5862 pci_disable_device(pdev);
5863
9d5c8243
AK		 5864 return 0;
5865}
5866
5867#ifdef CONFIG_PM
3fe7c4c9
RW		 5868static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5869{
5870 int retval;
5871 bool wake;
5872
5873 retval = __igb_shutdown(pdev, &wake);
5874 if (retval)
5875 return retval;
5876
5877 if (wake) {
5878 pci_prepare_to_sleep(pdev);
5879 } else {
5880 pci_wake_from_d3(pdev, false);
5881 pci_set_power_state(pdev, PCI_D3hot);
5882 }
5883
5884 return 0;
5885}
5886
9d5c8243
AK		 5887static int igb_resume(struct pci_dev *pdev)
5888{
5889 struct net_device *netdev = pci_get_drvdata(pdev);
5890 struct igb_adapter *adapter = netdev_priv(netdev);
5891 struct e1000_hw *hw = &adapter->hw;
5892 u32 err;
5893
5894 pci_set_power_state(pdev, PCI_D0);
5895 pci_restore_state(pdev);
b94f2d77 5896 pci_save_state(pdev);
42bfd33a 5897
aed5dec3 5898 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5899 if (err) {
5900 dev_err(&pdev->dev,
5901 "igb: Cannot enable PCI device from suspend\n");
5902 return err;
5903 }
5904 pci_set_master(pdev);
5905
5906 pci_enable_wake(pdev, PCI_D3hot, 0);
5907 pci_enable_wake(pdev, PCI_D3cold, 0);
5908
047e0030 5909 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5910 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5911 return -ENOMEM;
9d5c8243
AK		 5912 }
5913
9d5c8243 5914 igb_reset(adapter);
a8564f03
AD		 5915
5916 /* let the f/w know that the h/w is now under the control of the
5917 * driver. */
5918 igb_get_hw_control(adapter);
5919
9d5c8243
AK		 5920 wr32(E1000_WUS, ~0);
5921
a88f10ec
AD		 5922 if (netif_running(netdev)) {
5923 err = igb_open(netdev);
5924 if (err)
5925 return err;
5926 }
9d5c8243
AK		 5927
5928 netif_device_attach(netdev);
5929
9d5c8243
AK		 5930 return 0;
5931}
5932#endif
5933
5934static void igb_shutdown(struct pci_dev *pdev)
5935{
3fe7c4c9
RW		 5936 bool wake;
5937
5938 __igb_shutdown(pdev, &wake);
5939
5940 if (system_state == SYSTEM_POWER_OFF) {
5941 pci_wake_from_d3(pdev, wake);
5942 pci_set_power_state(pdev, PCI_D3hot);
5943 }
9d5c8243
AK		 5944}
5945
5946#ifdef CONFIG_NET_POLL_CONTROLLER
5947/*
5948 * Polling 'interrupt' - used by things like netconsole to send skbs
5949 * without having to re-enable interrupts. It's not called while
5950 * the interrupt routine is executing.
5951 */
5952static void igb_netpoll(struct net_device *netdev)
5953{
5954 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5955 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5956 int i;
9d5c8243 5957
eebbbdba 5958 if (!adapter->msix_entries) {
047e0030 5959 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5960 igb_irq_disable(adapter);
047e0030 5961 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5962 return;
5963 }
9d5c8243 5964
047e0030
AD		 5965 for (i = 0; i < adapter->num_q_vectors; i++) {
5966 struct igb_q_vector *q_vector = adapter->q_vector[i];
5967 wr32(E1000_EIMC, q_vector->eims_value);
5968 napi_schedule(&q_vector->napi);
eebbbdba 5969 }
9d5c8243
AK		 5970}
5971#endif /* CONFIG_NET_POLL_CONTROLLER */
5972
5973/**
5974 * igb_io_error_detected - called when PCI error is detected
5975 * @pdev: Pointer to PCI device
5976 * @state: The current pci connection state
5977 *
5978 * This function is called after a PCI bus error affecting
5979 * this device has been detected.
5980 */
5981static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5982 pci_channel_state_t state)
5983{
5984 struct net_device *netdev = pci_get_drvdata(pdev);
5985 struct igb_adapter *adapter = netdev_priv(netdev);
5986
5987 netif_device_detach(netdev);
5988
59ed6eec
AD		 5989 if (state == pci_channel_io_perm_failure)
5990 return PCI_ERS_RESULT_DISCONNECT;
5991
9d5c8243
AK		 5992 if (netif_running(netdev))
5993 igb_down(adapter);
5994 pci_disable_device(pdev);
5995
5996 /* Request a slot slot reset. */
5997 return PCI_ERS_RESULT_NEED_RESET;
5998}
5999
6000/**
6001 * igb_io_slot_reset - called after the pci bus has been reset.
6002 * @pdev: Pointer to PCI device
6003 *
6004 * Restart the card from scratch, as if from a cold-boot. Implementation
6005 * resembles the first-half of the igb_resume routine.
6006 */
6007static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6008{
6009 struct net_device *netdev = pci_get_drvdata(pdev);
6010 struct igb_adapter *adapter = netdev_priv(netdev);
6011 struct e1000_hw *hw = &adapter->hw;
40a914fa 6012 pci_ers_result_t result;
42bfd33a 6013 int err;
9d5c8243 6014
aed5dec3 6015 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6016 dev_err(&pdev->dev,
6017 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6018 result = PCI_ERS_RESULT_DISCONNECT;
6019 } else {
6020 pci_set_master(pdev);
6021 pci_restore_state(pdev);
b94f2d77 6022 pci_save_state(pdev);
9d5c8243 6023
40a914fa
AD		 6024 pci_enable_wake(pdev, PCI_D3hot, 0);
6025 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6026
40a914fa
AD		 6027 igb_reset(adapter);
6028 wr32(E1000_WUS, ~0);
6029 result = PCI_ERS_RESULT_RECOVERED;
6030 }
9d5c8243 6031
ea943d41
JK		 6032 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6033 if (err) {
6034 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6035 "failed 0x%0x\n", err);
6036 /* non-fatal, continue */
6037 }
40a914fa
AD		 6038
6039 return result;
9d5c8243
AK		 6040}
6041
6042/**
6043 * igb_io_resume - called when traffic can start flowing again.
6044 * @pdev: Pointer to PCI device
6045 *
6046 * This callback is called when the error recovery driver tells us that
6047 * its OK to resume normal operation. Implementation resembles the
6048 * second-half of the igb_resume routine.
6049 */
6050static void igb_io_resume(struct pci_dev *pdev)
6051{
6052 struct net_device *netdev = pci_get_drvdata(pdev);
6053 struct igb_adapter *adapter = netdev_priv(netdev);
6054
9d5c8243
AK		 6055 if (netif_running(netdev)) {
6056 if (igb_up(adapter)) {
6057 dev_err(&pdev->dev, "igb_up failed after reset\n");
6058 return;
6059 }
6060 }
6061
6062 netif_device_attach(netdev);
6063
6064 /* let the f/w know that the h/w is now under the control of the
6065 * driver. */
6066 igb_get_hw_control(adapter);
9d5c8243
AK		 6067}
6068
26ad9178
AD		 6069static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6070 u8 qsel)
6071{
6072 u32 rar_low, rar_high;
6073 struct e1000_hw *hw = &adapter->hw;
6074
6075 /* HW expects these in little endian so we reverse the byte order
6076 * from network order (big endian) to little endian
6077 */
6078 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6079 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6080 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6081
6082 /* Indicate to hardware the Address is Valid. */
6083 rar_high |= E1000_RAH_AV;
6084
6085 if (hw->mac.type == e1000_82575)
6086 rar_high |= E1000_RAH_POOL_1 * qsel;
6087 else
6088 rar_high |= E1000_RAH_POOL_1 << qsel;
6089
6090 wr32(E1000_RAL(index), rar_low);
6091 wrfl();
6092 wr32(E1000_RAH(index), rar_high);
6093 wrfl();
6094}
6095
4ae196df
AD		 6096static int igb_set_vf_mac(struct igb_adapter *adapter,
6097 int vf, unsigned char *mac_addr)
6098{
6099 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6100 /* VF MAC addresses start at end of receive addresses and moves
6101 * torwards the first, as a result a collision should not be possible */
6102 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6103
37680117 6104 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6105
26ad9178 6106 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6107
6108 return 0;
6109}
6110
8151d294
WM		 6111static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6112{
6113 struct igb_adapter *adapter = netdev_priv(netdev);
6114 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6115 return -EINVAL;
6116 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6117 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6118 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6119 " change effective.");
6120 if (test_bit(__IGB_DOWN, &adapter->state)) {
6121 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6122 " but the PF device is not up.\n");
6123 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6124 " attempting to use the VF device.\n");
6125 }
6126 return igb_set_vf_mac(adapter, vf, mac);
6127}
6128
6129static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6130{
6131 return -EOPNOTSUPP;
6132}
6133
6134static int igb_ndo_get_vf_config(struct net_device *netdev,
6135 int vf, struct ifla_vf_info *ivi)
6136{
6137 struct igb_adapter *adapter = netdev_priv(netdev);
6138 if (vf >= adapter->vfs_allocated_count)
6139 return -EINVAL;
6140 ivi->vf = vf;
6141 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6142 ivi->tx_rate = 0;
6143 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6144 ivi->qos = adapter->vf_data[vf].pf_qos;
6145 return 0;
6146}
6147
4ae196df
AD		 6148static void igb_vmm_control(struct igb_adapter *adapter)
6149{
6150 struct e1000_hw *hw = &adapter->hw;
10d8e907 6151 u32 reg;
4ae196df 6152
d4960307
AD		 6153 /* replication is not supported for 82575 */
6154 if (hw->mac.type == e1000_82575)
4ae196df
AD		 6155 return;
6156
10d8e907
AD		 6157 /* enable replication vlan tag stripping */
6158 reg = rd32(E1000_RPLOLR);
6159 reg |= E1000_RPLOLR_STRVLAN;
6160 wr32(E1000_RPLOLR, reg);
6161
6162 /* notify HW that the MAC is adding vlan tags */
6163 reg = rd32(E1000_DTXCTL);
6164 reg |= E1000_DTXCTL_VLAN_ADDED;
6165 wr32(E1000_DTXCTL, reg);
6166
d4960307
AD		 6167 if (adapter->vfs_allocated_count) {
6168 igb_vmdq_set_loopback_pf(hw, true);
6169 igb_vmdq_set_replication_pf(hw, true);
6170 } else {
6171 igb_vmdq_set_loopback_pf(hw, false);
6172 igb_vmdq_set_replication_pf(hw, false);
6173 }
4ae196df
AD		 6174}
6175
9d5c8243 6176/* igb_main.c */
Clone of davem's net-next-2.6 tree