]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/sfc/efx.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sfc: Implement TSO for TCP/IPv6
[net-next-2.6.git] / drivers / net / sfc / efx.c
CommitLineData
8ceee660
BH		 1/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2008 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/module.h>
12#include <linux/pci.h>
13#include <linux/netdevice.h>
14#include <linux/etherdevice.h>
15#include <linux/delay.h>
16#include <linux/notifier.h>
17#include <linux/ip.h>
18#include <linux/tcp.h>
19#include <linux/in.h>
20#include <linux/crc32.h>
21#include <linux/ethtool.h>
aa6ef27e 22#include <linux/topology.h>
8ceee660 23#include "net_driver.h"
8ceee660
BH		 24#include "efx.h"
25#include "mdio_10g.h"
744093c9 26#include "nic.h"
8ceee660 27
8880f4ec
BH		 28#include "mcdi.h"
29
c459302d
BH		 30/**************************************************************************
31 *
32 * Type name strings
33 *
34 **************************************************************************
35 */
36
37/* Loopback mode names (see LOOPBACK_MODE()) */
38const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
39const char *efx_loopback_mode_names[] = {
40 [LOOPBACK_NONE] = "NONE",
e58f69f4 41 [LOOPBACK_DATA] = "DATAPATH",
c459302d
BH		 42 [LOOPBACK_GMAC] = "GMAC",
43 [LOOPBACK_XGMII] = "XGMII",
44 [LOOPBACK_XGXS] = "XGXS",
45 [LOOPBACK_XAUI] = "XAUI",
e58f69f4
BH		 46 [LOOPBACK_GMII] = "GMII",
47 [LOOPBACK_SGMII] = "SGMII",
48 [LOOPBACK_XGBR] = "XGBR",
49 [LOOPBACK_XFI] = "XFI",
50 [LOOPBACK_XAUI_FAR] = "XAUI_FAR",
51 [LOOPBACK_GMII_FAR] = "GMII_FAR",
52 [LOOPBACK_SGMII_FAR] = "SGMII_FAR",
53 [LOOPBACK_XFI_FAR] = "XFI_FAR",
c459302d
BH		 54 [LOOPBACK_GPHY] = "GPHY",
55 [LOOPBACK_PHYXS] = "PHYXS",
56 [LOOPBACK_PCS] = "PCS",
57 [LOOPBACK_PMAPMD] = "PMA/PMD",
e58f69f4
BH		 58 [LOOPBACK_XPORT] = "XPORT",
59 [LOOPBACK_XGMII_WS] = "XGMII_WS",
60 [LOOPBACK_XAUI_WS] = "XAUI_WS",
61 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",
62 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
63 [LOOPBACK_GMII_WS] = "GMII_WS",
64 [LOOPBACK_XFI_WS] = "XFI_WS",
65 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",
66 [LOOPBACK_PHYXS_WS] = "PHYXS_WS",
c459302d
BH		 67};
68
69/* Interrupt mode names (see INT_MODE())) */
70const unsigned int efx_interrupt_mode_max = EFX_INT_MODE_MAX;
71const char *efx_interrupt_mode_names[] = {
72 [EFX_INT_MODE_MSIX] = "MSI-X",
73 [EFX_INT_MODE_MSI] = "MSI",
74 [EFX_INT_MODE_LEGACY] = "legacy",
75};
76
77const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
78const char *efx_reset_type_names[] = {
79 [RESET_TYPE_INVISIBLE] = "INVISIBLE",
80 [RESET_TYPE_ALL] = "ALL",
81 [RESET_TYPE_WORLD] = "WORLD",
82 [RESET_TYPE_DISABLE] = "DISABLE",
83 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",
84 [RESET_TYPE_INT_ERROR] = "INT_ERROR",
85 [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY",
86 [RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",
87 [RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",
88 [RESET_TYPE_TX_SKIP] = "TX_SKIP",
8880f4ec 89 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE",
c459302d
BH		 90};
91
8ceee660
BH		 92#define EFX_MAX_MTU (9 * 1024)
93
94/* RX slow fill workqueue. If memory allocation fails in the fast path,
95 * a work item is pushed onto this work queue to retry the allocation later,
96 * to avoid the NIC being starved of RX buffers. Since this is a per cpu
97 * workqueue, there is nothing to be gained in making it per NIC
98 */
99static struct workqueue_struct *refill_workqueue;
100
1ab00629
SH		 101/* Reset workqueue. If any NIC has a hardware failure then a reset will be
102 * queued onto this work queue. This is not a per-nic work queue, because
103 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
104 */
105static struct workqueue_struct *reset_workqueue;
106
8ceee660
BH		 107/**************************************************************************
108 *
109 * Configurable values
110 *
111 *************************************************************************/
112
8ceee660
BH		 113/*
114 * Use separate channels for TX and RX events
115 *
28b581ab
NT		 116 * Set this to 1 to use separate channels for TX and RX. It allows us
117 * to control interrupt affinity separately for TX and RX.
8ceee660 118 *
28b581ab 119 * This is only used in MSI-X interrupt mode
8ceee660 120 */
28b581ab
NT		 121static unsigned int separate_tx_channels;
122module_param(separate_tx_channels, uint, 0644);
123MODULE_PARM_DESC(separate_tx_channels,
124 "Use separate channels for TX and RX");
8ceee660
BH		 125
126/* This is the weight assigned to each of the (per-channel) virtual
127 * NAPI devices.
128 */
129static int napi_weight = 64;
130
131/* This is the time (in jiffies) between invocations of the hardware
132 * monitor, which checks for known hardware bugs and resets the
133 * hardware and driver as necessary.
134 */
135unsigned int efx_monitor_interval = 1 * HZ;
136
8ceee660
BH		 137/* This controls whether or not the driver will initialise devices
138 * with invalid MAC addresses stored in the EEPROM or flash. If true,
139 * such devices will be initialised with a random locally-generated
140 * MAC address. This allows for loading the sfc_mtd driver to
141 * reprogram the flash, even if the flash contents (including the MAC
142 * address) have previously been erased.
143 */
144static unsigned int allow_bad_hwaddr;
145
146/* Initial interrupt moderation settings. They can be modified after
147 * module load with ethtool.
148 *
149 * The default for RX should strike a balance between increasing the
150 * round-trip latency and reducing overhead.
151 */
152static unsigned int rx_irq_mod_usec = 60;
153
154/* Initial interrupt moderation settings. They can be modified after
155 * module load with ethtool.
156 *
157 * This default is chosen to ensure that a 10G link does not go idle
158 * while a TX queue is stopped after it has become full. A queue is
159 * restarted when it drops below half full. The time this takes (assuming
160 * worst case 3 descriptors per packet and 1024 descriptors) is
161 * 512 / 3 * 1.2 = 205 usec.
162 */
163static unsigned int tx_irq_mod_usec = 150;
164
165/* This is the first interrupt mode to try out of:
166 * 0 => MSI-X
167 * 1 => MSI
168 * 2 => legacy
169 */
170static unsigned int interrupt_mode;
171
172/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
173 * i.e. the number of CPUs among which we may distribute simultaneous
174 * interrupt handling.
175 *
176 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
177 * The default (0) means to assign an interrupt to each package (level II cache)
178 */
179static unsigned int rss_cpus;
180module_param(rss_cpus, uint, 0444);
181MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
182
84ae48fe
BH		 183static int phy_flash_cfg;
184module_param(phy_flash_cfg, int, 0644);
185MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
186
6fb70fd1
BH		 187static unsigned irq_adapt_low_thresh = 10000;
188module_param(irq_adapt_low_thresh, uint, 0644);
189MODULE_PARM_DESC(irq_adapt_low_thresh,
190 "Threshold score for reducing IRQ moderation");
191
192static unsigned irq_adapt_high_thresh = 20000;
193module_param(irq_adapt_high_thresh, uint, 0644);
194MODULE_PARM_DESC(irq_adapt_high_thresh,
195 "Threshold score for increasing IRQ moderation");
196
8ceee660
BH		 197/**************************************************************************
198 *
199 * Utility functions and prototypes
200 *
201 *************************************************************************/
202static void efx_remove_channel(struct efx_channel *channel);
203static void efx_remove_port(struct efx_nic *efx);
204static void efx_fini_napi(struct efx_nic *efx);
205static void efx_fini_channels(struct efx_nic *efx);
206
207#define EFX_ASSERT_RESET_SERIALISED(efx) \
208 do { \
332c1ce9
BH		 209 if ((efx->state == STATE_RUNNING) || \
210 (efx->state == STATE_DISABLED)) \
8ceee660
BH		 211 ASSERT_RTNL(); \
212 } while (0)
213
214/**************************************************************************
215 *
216 * Event queue processing
217 *
218 *************************************************************************/
219
220/* Process channel's event queue
221 *
222 * This function is responsible for processing the event queue of a
223 * single channel. The caller must guarantee that this function will
224 * never be concurrently called more than once on the same channel,
225 * though different channels may be being processed concurrently.
226 */
4d566063 227static int efx_process_channel(struct efx_channel *channel, int rx_quota)
8ceee660 228{
42cbe2d7
BH		 229 struct efx_nic *efx = channel->efx;
230 int rx_packets;
8ceee660 231
42cbe2d7 232 if (unlikely(efx->reset_pending != RESET_TYPE_NONE ||
8ceee660 233 !channel->enabled))
42cbe2d7 234 return 0;
8ceee660 235
152b6a62 236 rx_packets = efx_nic_process_eventq(channel, rx_quota);
42cbe2d7
BH		 237 if (rx_packets == 0)
238 return 0;
8ceee660
BH		 239
240 /* Deliver last RX packet. */
241 if (channel->rx_pkt) {
242 __efx_rx_packet(channel, channel->rx_pkt,
243 channel->rx_pkt_csummed);
244 channel->rx_pkt = NULL;
245 }
246
8ceee660
BH		 247 efx_rx_strategy(channel);
248
42cbe2d7 249 efx_fast_push_rx_descriptors(&efx->rx_queue[channel->channel]);
8ceee660 250
42cbe2d7 251 return rx_packets;
8ceee660
BH		 252}
253
254/* Mark channel as finished processing
255 *
256 * Note that since we will not receive further interrupts for this
257 * channel before we finish processing and call the eventq_read_ack()
258 * method, there is no need to use the interrupt hold-off timers.
259 */
260static inline void efx_channel_processed(struct efx_channel *channel)
261{
5b9e207c
BH		 262 /* The interrupt handler for this channel may set work_pending
263 * as soon as we acknowledge the events we've seen. Make sure
264 * it's cleared before then. */
dc8cfa55 265 channel->work_pending = false;
5b9e207c
BH		 266 smp_wmb();
267
152b6a62 268 efx_nic_eventq_read_ack(channel);
8ceee660
BH		 269}
270
271/* NAPI poll handler
272 *
273 * NAPI guarantees serialisation of polls of the same device, which
274 * provides the guarantee required by efx_process_channel().
275 */
276static int efx_poll(struct napi_struct *napi, int budget)
277{
278 struct efx_channel *channel =
279 container_of(napi, struct efx_channel, napi_str);
8ceee660
BH		 280 int rx_packets;
281
282 EFX_TRACE(channel->efx, "channel %d NAPI poll executing on CPU %d\n",
283 channel->channel, raw_smp_processor_id());
284
42cbe2d7 285 rx_packets = efx_process_channel(channel, budget);
8ceee660
BH		 286
287 if (rx_packets < budget) {
6fb70fd1
BH		 288 struct efx_nic *efx = channel->efx;
289
290 if (channel->used_flags & EFX_USED_BY_RX &&
291 efx->irq_rx_adaptive &&
292 unlikely(++channel->irq_count == 1000)) {
6fb70fd1
BH		 293 if (unlikely(channel->irq_mod_score <
294 irq_adapt_low_thresh)) {
0d86ebd8
BH		 295 if (channel->irq_moderation > 1) {
296 channel->irq_moderation -= 1;
ef2b90ee 297 efx->type->push_irq_moderation(channel);
0d86ebd8 298 }
6fb70fd1
BH		 299 } else if (unlikely(channel->irq_mod_score >
300 irq_adapt_high_thresh)) {
0d86ebd8
BH		 301 if (channel->irq_moderation <
302 efx->irq_rx_moderation) {
303 channel->irq_moderation += 1;
ef2b90ee 304 efx->type->push_irq_moderation(channel);
0d86ebd8 305 }
6fb70fd1 306 }
6fb70fd1
BH		 307 channel->irq_count = 0;
308 channel->irq_mod_score = 0;
309 }
310
8ceee660 311 /* There is no race here; although napi_disable() will
288379f0 312 * only wait for napi_complete(), this isn't a problem
8ceee660
BH		 313 * since efx_channel_processed() will have no effect if
314 * interrupts have already been disabled.
315 */
288379f0 316 napi_complete(napi);
8ceee660
BH		 317 efx_channel_processed(channel);
318 }
319
320 return rx_packets;
321}
322
323/* Process the eventq of the specified channel immediately on this CPU
324 *
325 * Disable hardware generated interrupts, wait for any existing
326 * processing to finish, then directly poll (and ack ) the eventq.
327 * Finally reenable NAPI and interrupts.
328 *
329 * Since we are touching interrupts the caller should hold the suspend lock
330 */
331void efx_process_channel_now(struct efx_channel *channel)
332{
333 struct efx_nic *efx = channel->efx;
334
335 BUG_ON(!channel->used_flags);
336 BUG_ON(!channel->enabled);
337
338 /* Disable interrupts and wait for ISRs to complete */
152b6a62 339 efx_nic_disable_interrupts(efx);
8ceee660
BH		 340 if (efx->legacy_irq)
341 synchronize_irq(efx->legacy_irq);
64ee3120 342 if (channel->irq)
8ceee660
BH		 343 synchronize_irq(channel->irq);
344
345 /* Wait for any NAPI processing to complete */
346 napi_disable(&channel->napi_str);
347
348 /* Poll the channel */
3ffeabdd 349 efx_process_channel(channel, EFX_EVQ_SIZE);
8ceee660
BH		 350
351 /* Ack the eventq. This may cause an interrupt to be generated
352 * when they are reenabled */
353 efx_channel_processed(channel);
354
355 napi_enable(&channel->napi_str);
152b6a62 356 efx_nic_enable_interrupts(efx);
8ceee660
BH		 357}
358
359/* Create event queue
360 * Event queue memory allocations are done only once. If the channel
361 * is reset, the memory buffer will be reused; this guards against
362 * errors during channel reset and also simplifies interrupt handling.
363 */
364static int efx_probe_eventq(struct efx_channel *channel)
365{
366 EFX_LOG(channel->efx, "chan %d create event queue\n", channel->channel);
367
152b6a62 368 return efx_nic_probe_eventq(channel);
8ceee660
BH		 369}
370
371/* Prepare channel's event queue */
bc3c90a2 372static void efx_init_eventq(struct efx_channel *channel)
8ceee660
BH		 373{
374 EFX_LOG(channel->efx, "chan %d init event queue\n", channel->channel);
375
376 channel->eventq_read_ptr = 0;
377
152b6a62 378 efx_nic_init_eventq(channel);
8ceee660
BH		 379}
380
381static void efx_fini_eventq(struct efx_channel *channel)
382{
383 EFX_LOG(channel->efx, "chan %d fini event queue\n", channel->channel);
384
152b6a62 385 efx_nic_fini_eventq(channel);
8ceee660
BH		 386}
387
388static void efx_remove_eventq(struct efx_channel *channel)
389{
390 EFX_LOG(channel->efx, "chan %d remove event queue\n", channel->channel);
391
152b6a62 392 efx_nic_remove_eventq(channel);
8ceee660
BH		 393}
394
395/**************************************************************************
396 *
397 * Channel handling
398 *
399 *************************************************************************/
400
8ceee660
BH		 401static int efx_probe_channel(struct efx_channel *channel)
402{
403 struct efx_tx_queue *tx_queue;
404 struct efx_rx_queue *rx_queue;
405 int rc;
406
407 EFX_LOG(channel->efx, "creating channel %d\n", channel->channel);
408
409 rc = efx_probe_eventq(channel);
410 if (rc)
411 goto fail1;
412
413 efx_for_each_channel_tx_queue(tx_queue, channel) {
414 rc = efx_probe_tx_queue(tx_queue);
415 if (rc)
416 goto fail2;
417 }
418
419 efx_for_each_channel_rx_queue(rx_queue, channel) {
420 rc = efx_probe_rx_queue(rx_queue);
421 if (rc)
422 goto fail3;
423 }
424
425 channel->n_rx_frm_trunc = 0;
426
427 return 0;
428
429 fail3:
430 efx_for_each_channel_rx_queue(rx_queue, channel)
431 efx_remove_rx_queue(rx_queue);
432 fail2:
433 efx_for_each_channel_tx_queue(tx_queue, channel)
434 efx_remove_tx_queue(tx_queue);
435 fail1:
436 return rc;
437}
438
439
56536e9c
BH		 440static void efx_set_channel_names(struct efx_nic *efx)
441{
442 struct efx_channel *channel;
443 const char *type = "";
444 int number;
445
446 efx_for_each_channel(channel, efx) {
447 number = channel->channel;
448 if (efx->n_channels > efx->n_rx_queues) {
449 if (channel->channel < efx->n_rx_queues) {
450 type = "-rx";
451 } else {
452 type = "-tx";
453 number -= efx->n_rx_queues;
454 }
455 }
456 snprintf(channel->name, sizeof(channel->name),
457 "%s%s-%d", efx->name, type, number);
458 }
459}
460
8ceee660
BH		 461/* Channels are shutdown and reinitialised whilst the NIC is running
462 * to propagate configuration changes (mtu, checksum offload), or
463 * to clear hardware error conditions
464 */
bc3c90a2 465static void efx_init_channels(struct efx_nic *efx)
8ceee660
BH		 466{
467 struct efx_tx_queue *tx_queue;
468 struct efx_rx_queue *rx_queue;
469 struct efx_channel *channel;
8ceee660 470
f7f13b0b
BH		 471 /* Calculate the rx buffer allocation parameters required to
472 * support the current MTU, including padding for header
473 * alignment and overruns.
474 */
475 efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
476 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
477 efx->type->rx_buffer_padding);
478 efx->rx_buffer_order = get_order(efx->rx_buffer_len);
8ceee660
BH		 479
480 /* Initialise the channels */
481 efx_for_each_channel(channel, efx) {
482 EFX_LOG(channel->efx, "init chan %d\n", channel->channel);
483
bc3c90a2 484 efx_init_eventq(channel);
8ceee660 485
bc3c90a2
BH		 486 efx_for_each_channel_tx_queue(tx_queue, channel)
487 efx_init_tx_queue(tx_queue);
8ceee660
BH		 488
489 /* The rx buffer allocation strategy is MTU dependent */
490 efx_rx_strategy(channel);
491
bc3c90a2
BH		 492 efx_for_each_channel_rx_queue(rx_queue, channel)
493 efx_init_rx_queue(rx_queue);
8ceee660
BH		 494
495 WARN_ON(channel->rx_pkt != NULL);
496 efx_rx_strategy(channel);
497 }
8ceee660
BH		 498}
499
500/* This enables event queue processing and packet transmission.
501 *
502 * Note that this function is not allowed to fail, since that would
503 * introduce too much complexity into the suspend/resume path.
504 */
505static void efx_start_channel(struct efx_channel *channel)
506{
507 struct efx_rx_queue *rx_queue;
508
509 EFX_LOG(channel->efx, "starting chan %d\n", channel->channel);
510
5b9e207c
BH		 511 /* The interrupt handler for this channel may set work_pending
512 * as soon as we enable it. Make sure it's cleared before
513 * then. Similarly, make sure it sees the enabled flag set. */
dc8cfa55
BH		 514 channel->work_pending = false;
515 channel->enabled = true;
5b9e207c 516 smp_wmb();
8ceee660
BH		 517
518 napi_enable(&channel->napi_str);
519
520 /* Load up RX descriptors */
521 efx_for_each_channel_rx_queue(rx_queue, channel)
522 efx_fast_push_rx_descriptors(rx_queue);
523}
524
525/* This disables event queue processing and packet transmission.
526 * This function does not guarantee that all queue processing
527 * (e.g. RX refill) is complete.
528 */
529static void efx_stop_channel(struct efx_channel *channel)
530{
531 struct efx_rx_queue *rx_queue;
532
533 if (!channel->enabled)
534 return;
535
536 EFX_LOG(channel->efx, "stop chan %d\n", channel->channel);
537
dc8cfa55 538 channel->enabled = false;
8ceee660
BH		 539 napi_disable(&channel->napi_str);
540
541 /* Ensure that any worker threads have exited or will be no-ops */
542 efx_for_each_channel_rx_queue(rx_queue, channel) {
543 spin_lock_bh(&rx_queue->add_lock);
544 spin_unlock_bh(&rx_queue->add_lock);
545 }
546}
547
548static void efx_fini_channels(struct efx_nic *efx)
549{
550 struct efx_channel *channel;
551 struct efx_tx_queue *tx_queue;
552 struct efx_rx_queue *rx_queue;
6bc5d3a9 553 int rc;
8ceee660
BH		 554
555 EFX_ASSERT_RESET_SERIALISED(efx);
556 BUG_ON(efx->port_enabled);
557
152b6a62 558 rc = efx_nic_flush_queues(efx);
6bc5d3a9
BH		 559 if (rc)
560 EFX_ERR(efx, "failed to flush queues\n");
561 else
562 EFX_LOG(efx, "successfully flushed all queues\n");
563
8ceee660
BH		 564 efx_for_each_channel(channel, efx) {
565 EFX_LOG(channel->efx, "shut down chan %d\n", channel->channel);
566
567 efx_for_each_channel_rx_queue(rx_queue, channel)
568 efx_fini_rx_queue(rx_queue);
569 efx_for_each_channel_tx_queue(tx_queue, channel)
570 efx_fini_tx_queue(tx_queue);
8ceee660
BH		 571 efx_fini_eventq(channel);
572 }
573}
574
575static void efx_remove_channel(struct efx_channel *channel)
576{
577 struct efx_tx_queue *tx_queue;
578 struct efx_rx_queue *rx_queue;
579
580 EFX_LOG(channel->efx, "destroy chan %d\n", channel->channel);
581
582 efx_for_each_channel_rx_queue(rx_queue, channel)
583 efx_remove_rx_queue(rx_queue);
584 efx_for_each_channel_tx_queue(tx_queue, channel)
585 efx_remove_tx_queue(tx_queue);
586 efx_remove_eventq(channel);
587
588 channel->used_flags = 0;
589}
590
591void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue, int delay)
592{
593 queue_delayed_work(refill_workqueue, &rx_queue->work, delay);
594}
595
596/**************************************************************************
597 *
598 * Port handling
599 *
600 **************************************************************************/
601
602/* This ensures that the kernel is kept informed (via
603 * netif_carrier_on/off) of the link status, and also maintains the
604 * link status's stop on the port's TX queue.
605 */
fdaa9aed 606void efx_link_status_changed(struct efx_nic *efx)
8ceee660 607{
eb50c0d6
BH		 608 struct efx_link_state *link_state = &efx->link_state;
609
8ceee660
BH		 610 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
611 * that no events are triggered between unregister_netdev() and the
612 * driver unloading. A more general condition is that NETDEV_CHANGE
613 * can only be generated between NETDEV_UP and NETDEV_DOWN */
614 if (!netif_running(efx->net_dev))
615 return;
616
8c8661e4
BH		 617 if (efx->port_inhibited) {
618 netif_carrier_off(efx->net_dev);
619 return;
620 }
621
eb50c0d6 622 if (link_state->up != netif_carrier_ok(efx->net_dev)) {
8ceee660
BH		 623 efx->n_link_state_changes++;
624
eb50c0d6 625 if (link_state->up)
8ceee660
BH		 626 netif_carrier_on(efx->net_dev);
627 else
628 netif_carrier_off(efx->net_dev);
629 }
630
631 /* Status message for kernel log */
eb50c0d6 632 if (link_state->up) {
f31a45d2 633 EFX_INFO(efx, "link up at %uMbps %s-duplex (MTU %d)%s\n",
eb50c0d6 634 link_state->speed, link_state->fd ? "full" : "half",
8ceee660
BH		 635 efx->net_dev->mtu,
636 (efx->promiscuous ? " [PROMISC]" : ""));
637 } else {
638 EFX_INFO(efx, "link down\n");
639 }
640
641}
642
d3245b28
BH		 643void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
644{
645 efx->link_advertising = advertising;
646 if (advertising) {
647 if (advertising & ADVERTISED_Pause)
648 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
649 else
650 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
651 if (advertising & ADVERTISED_Asym_Pause)
652 efx->wanted_fc ^= EFX_FC_TX;
653 }
654}
655
656void efx_link_set_wanted_fc(struct efx_nic *efx, enum efx_fc_type wanted_fc)
657{
658 efx->wanted_fc = wanted_fc;
659 if (efx->link_advertising) {
660 if (wanted_fc & EFX_FC_RX)
661 efx->link_advertising |= (ADVERTISED_Pause |
662 ADVERTISED_Asym_Pause);
663 else
664 efx->link_advertising &= ~(ADVERTISED_Pause |
665 ADVERTISED_Asym_Pause);
666 if (wanted_fc & EFX_FC_TX)
667 efx->link_advertising ^= ADVERTISED_Asym_Pause;
668 }
669}
670
115122af
BH		 671static void efx_fini_port(struct efx_nic *efx);
672
d3245b28
BH		 673/* Push loopback/power/transmit disable settings to the PHY, and reconfigure
674 * the MAC appropriately. All other PHY configuration changes are pushed
675 * through phy_op->set_settings(), and pushed asynchronously to the MAC
676 * through efx_monitor().
677 *
678 * Callers must hold the mac_lock
679 */
680int __efx_reconfigure_port(struct efx_nic *efx)
8ceee660 681{
d3245b28
BH		 682 enum efx_phy_mode phy_mode;
683 int rc;
8ceee660 684
d3245b28 685 WARN_ON(!mutex_is_locked(&efx->mac_lock));
8ceee660 686
a816f75a
BH		 687 /* Serialise the promiscuous flag with efx_set_multicast_list. */
688 if (efx_dev_registered(efx)) {
689 netif_addr_lock_bh(efx->net_dev);
690 netif_addr_unlock_bh(efx->net_dev);
691 }
692
d3245b28
BH		 693 /* Disable PHY transmit in mac level loopbacks */
694 phy_mode = efx->phy_mode;
177dfcd8
BH		 695 if (LOOPBACK_INTERNAL(efx))
696 efx->phy_mode |= PHY_MODE_TX_DISABLED;
697 else
698 efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
177dfcd8 699
d3245b28 700 rc = efx->type->reconfigure_port(efx);
8ceee660 701
d3245b28
BH		 702 if (rc)
703 efx->phy_mode = phy_mode;
177dfcd8 704
d3245b28 705 return rc;
8ceee660
BH		 706}
707
708/* Reinitialise the MAC to pick up new PHY settings, even if the port is
709 * disabled. */
d3245b28 710int efx_reconfigure_port(struct efx_nic *efx)
8ceee660 711{
d3245b28
BH		 712 int rc;
713
8ceee660
BH		 714 EFX_ASSERT_RESET_SERIALISED(efx);
715
716 mutex_lock(&efx->mac_lock);
d3245b28 717 rc = __efx_reconfigure_port(efx);
8ceee660 718 mutex_unlock(&efx->mac_lock);
d3245b28
BH		 719
720 return rc;
8ceee660
BH		 721}
722
8be4f3e6
BH		 723/* Asynchronous work item for changing MAC promiscuity and multicast
724 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
725 * MAC directly. */
766ca0fa
BH		 726static void efx_mac_work(struct work_struct *data)
727{
728 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
729
730 mutex_lock(&efx->mac_lock);
8be4f3e6 731 if (efx->port_enabled) {
ef2b90ee 732 efx->type->push_multicast_hash(efx);
8be4f3e6
BH		 733 efx->mac_op->reconfigure(efx);
734 }
766ca0fa
BH		 735 mutex_unlock(&efx->mac_lock);
736}
737
8ceee660
BH		 738static int efx_probe_port(struct efx_nic *efx)
739{
740 int rc;
741
742 EFX_LOG(efx, "create port\n");
743
ef2b90ee
BH		 744 /* Connect up MAC/PHY operations table */
745 rc = efx->type->probe_port(efx);
8ceee660
BH		 746 if (rc)
747 goto err;
748
84ae48fe
BH		 749 if (phy_flash_cfg)
750 efx->phy_mode = PHY_MODE_SPECIAL;
751
8ceee660
BH		 752 /* Sanity check MAC address */
753 if (is_valid_ether_addr(efx->mac_address)) {
754 memcpy(efx->net_dev->dev_addr, efx->mac_address, ETH_ALEN);
755 } else {
e174961c
JB		 756 EFX_ERR(efx, "invalid MAC address %pM\n",
757 efx->mac_address);
8ceee660
BH		 758 if (!allow_bad_hwaddr) {
759 rc = -EINVAL;
760 goto err;
761 }
762 random_ether_addr(efx->net_dev->dev_addr);
e174961c
JB		 763 EFX_INFO(efx, "using locally-generated MAC %pM\n",
764 efx->net_dev->dev_addr);
8ceee660
BH		 765 }
766
767 return 0;
768
769 err:
770 efx_remove_port(efx);
771 return rc;
772}
773
774static int efx_init_port(struct efx_nic *efx)
775{
776 int rc;
777
778 EFX_LOG(efx, "init port\n");
779
1dfc5cea
BH		 780 mutex_lock(&efx->mac_lock);
781
177dfcd8 782 rc = efx->phy_op->init(efx);
8ceee660 783 if (rc)
1dfc5cea 784 goto fail1;
8ceee660 785
dc8cfa55 786 efx->port_initialized = true;
1dfc5cea 787
d3245b28
BH		 788 /* Reconfigure the MAC before creating dma queues (required for
789 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
790 efx->mac_op->reconfigure(efx);
791
792 /* Ensure the PHY advertises the correct flow control settings */
793 rc = efx->phy_op->reconfigure(efx);
794 if (rc)
795 goto fail2;
796
1dfc5cea 797 mutex_unlock(&efx->mac_lock);
8ceee660 798 return 0;
177dfcd8 799
1dfc5cea 800fail2:
177dfcd8 801 efx->phy_op->fini(efx);
1dfc5cea
BH		 802fail1:
803 mutex_unlock(&efx->mac_lock);
177dfcd8 804 return rc;
8ceee660
BH		 805}
806
8ceee660
BH		 807static void efx_start_port(struct efx_nic *efx)
808{
809 EFX_LOG(efx, "start port\n");
810 BUG_ON(efx->port_enabled);
811
812 mutex_lock(&efx->mac_lock);
dc8cfa55 813 efx->port_enabled = true;
8be4f3e6
BH		 814
815 /* efx_mac_work() might have been scheduled after efx_stop_port(),
816 * and then cancelled by efx_flush_all() */
ef2b90ee 817 efx->type->push_multicast_hash(efx);
8be4f3e6
BH		 818 efx->mac_op->reconfigure(efx);
819
8ceee660
BH		 820 mutex_unlock(&efx->mac_lock);
821}
822
fdaa9aed 823/* Prevent efx_mac_work() and efx_monitor() from working */
8ceee660
BH		 824static void efx_stop_port(struct efx_nic *efx)
825{
826 EFX_LOG(efx, "stop port\n");
827
828 mutex_lock(&efx->mac_lock);
dc8cfa55 829 efx->port_enabled = false;
8ceee660
BH		 830 mutex_unlock(&efx->mac_lock);
831
832 /* Serialise against efx_set_multicast_list() */
55668611 833 if (efx_dev_registered(efx)) {
b9e40857
DM		 834 netif_addr_lock_bh(efx->net_dev);
835 netif_addr_unlock_bh(efx->net_dev);
8ceee660
BH		 836 }
837}
838
839static void efx_fini_port(struct efx_nic *efx)
840{
841 EFX_LOG(efx, "shut down port\n");
842
843 if (!efx->port_initialized)
844 return;
845
177dfcd8 846 efx->phy_op->fini(efx);
dc8cfa55 847 efx->port_initialized = false;
8ceee660 848
eb50c0d6 849 efx->link_state.up = false;
8ceee660
BH		 850 efx_link_status_changed(efx);
851}
852
853static void efx_remove_port(struct efx_nic *efx)
854{
855 EFX_LOG(efx, "destroying port\n");
856
ef2b90ee 857 efx->type->remove_port(efx);
8ceee660
BH		 858}
859
860/**************************************************************************
861 *
862 * NIC handling
863 *
864 **************************************************************************/
865
866/* This configures the PCI device to enable I/O and DMA. */
867static int efx_init_io(struct efx_nic *efx)
868{
869 struct pci_dev *pci_dev = efx->pci_dev;
870 dma_addr_t dma_mask = efx->type->max_dma_mask;
871 int rc;
872
873 EFX_LOG(efx, "initialising I/O\n");
874
875 rc = pci_enable_device(pci_dev);
876 if (rc) {
877 EFX_ERR(efx, "failed to enable PCI device\n");
878 goto fail1;
879 }
880
881 pci_set_master(pci_dev);
882
883 /* Set the PCI DMA mask. Try all possibilities from our
884 * genuine mask down to 32 bits, because some architectures
885 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
886 * masks event though they reject 46 bit masks.
887 */
888 while (dma_mask > 0x7fffffffUL) {
889 if (pci_dma_supported(pci_dev, dma_mask) &&
890 ((rc = pci_set_dma_mask(pci_dev, dma_mask)) == 0))
891 break;
892 dma_mask >>= 1;
893 }
894 if (rc) {
895 EFX_ERR(efx, "could not find a suitable DMA mask\n");
896 goto fail2;
897 }
898 EFX_LOG(efx, "using DMA mask %llx\n", (unsigned long long) dma_mask);
899 rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);
900 if (rc) {
901 /* pci_set_consistent_dma_mask() is not *allowed* to
902 * fail with a mask that pci_set_dma_mask() accepted,
903 * but just in case...
904 */
905 EFX_ERR(efx, "failed to set consistent DMA mask\n");
906 goto fail2;
907 }
908
dc803df8
BH		 909 efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);
910 rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");
8ceee660
BH		 911 if (rc) {
912 EFX_ERR(efx, "request for memory BAR failed\n");
913 rc = -EIO;
914 goto fail3;
915 }
916 efx->membase = ioremap_nocache(efx->membase_phys,
917 efx->type->mem_map_size);
918 if (!efx->membase) {
dc803df8 919 EFX_ERR(efx, "could not map memory BAR at %llx+%x\n",
086ea356 920 (unsigned long long)efx->membase_phys,
8ceee660
BH		 921 efx->type->mem_map_size);
922 rc = -ENOMEM;
923 goto fail4;
924 }
dc803df8
BH		 925 EFX_LOG(efx, "memory BAR at %llx+%x (virtual %p)\n",
926 (unsigned long long)efx->membase_phys,
086ea356 927 efx->type->mem_map_size, efx->membase);
8ceee660
BH		 928
929 return 0;
930
931 fail4:
dc803df8 932 pci_release_region(efx->pci_dev, EFX_MEM_BAR);
8ceee660 933 fail3:
2c118e0f 934 efx->membase_phys = 0;
8ceee660
BH		 935 fail2:
936 pci_disable_device(efx->pci_dev);
937 fail1:
938 return rc;
939}
940
941static void efx_fini_io(struct efx_nic *efx)
942{
943 EFX_LOG(efx, "shutting down I/O\n");
944
945 if (efx->membase) {
946 iounmap(efx->membase);
947 efx->membase = NULL;
948 }
949
950 if (efx->membase_phys) {
dc803df8 951 pci_release_region(efx->pci_dev, EFX_MEM_BAR);
2c118e0f 952 efx->membase_phys = 0;
8ceee660
BH		 953 }
954
955 pci_disable_device(efx->pci_dev);
956}
957
46123d04
BH		 958/* Get number of RX queues wanted. Return number of online CPU
959 * packages in the expectation that an IRQ balancer will spread
960 * interrupts across them. */
961static int efx_wanted_rx_queues(void)
962{
2f8975fb 963 cpumask_var_t core_mask;
46123d04
BH		 964 int count;
965 int cpu;
966
79f55997 967 if (unlikely(!zalloc_cpumask_var(&core_mask, GFP_KERNEL))) {
2f8975fb 968 printk(KERN_WARNING
3977d033 969 "sfc: RSS disabled due to allocation failure\n");
2f8975fb
RR		 970 return 1;
971 }
972
46123d04
BH		 973 count = 0;
974 for_each_online_cpu(cpu) {
2f8975fb 975 if (!cpumask_test_cpu(cpu, core_mask)) {
46123d04 976 ++count;
2f8975fb 977 cpumask_or(core_mask, core_mask,
fbd59a8d 978 topology_core_cpumask(cpu));
46123d04
BH		 979 }
980 }
981
2f8975fb 982 free_cpumask_var(core_mask);
46123d04
BH		 983 return count;
984}
985
986/* Probe the number and type of interrupts we are able to obtain, and
987 * the resulting numbers of channels and RX queues.
988 */
8ceee660
BH		 989static void efx_probe_interrupts(struct efx_nic *efx)
990{
46123d04
BH		 991 int max_channels =
992 min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
8ceee660
BH		 993 int rc, i;
994
995 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
46123d04
BH		 996 struct msix_entry xentries[EFX_MAX_CHANNELS];
997 int wanted_ints;
28b581ab 998 int rx_queues;
aa6ef27e 999
46123d04
BH		 1000 /* We want one RX queue and interrupt per CPU package
1001 * (or as specified by the rss_cpus module parameter).
1002 * We will need one channel per interrupt.
1003 */
28b581ab
NT		 1004 rx_queues = rss_cpus ? rss_cpus : efx_wanted_rx_queues();
1005 wanted_ints = rx_queues + (separate_tx_channels ? 1 : 0);
1006 wanted_ints = min(wanted_ints, max_channels);
8ceee660 1007
28b581ab 1008 for (i = 0; i < wanted_ints; i++)
8ceee660 1009 xentries[i].entry = i;
28b581ab 1010 rc = pci_enable_msix(efx->pci_dev, xentries, wanted_ints);
8ceee660 1011 if (rc > 0) {
28b581ab
NT		 1012 EFX_ERR(efx, "WARNING: Insufficient MSI-X vectors"
1013 " available (%d < %d).\n", rc, wanted_ints);
1014 EFX_ERR(efx, "WARNING: Performance may be reduced.\n");
1015 EFX_BUG_ON_PARANOID(rc >= wanted_ints);
1016 wanted_ints = rc;
8ceee660 1017 rc = pci_enable_msix(efx->pci_dev, xentries,
28b581ab 1018 wanted_ints);
8ceee660
BH		 1019 }
1020
1021 if (rc == 0) {
28b581ab
NT		 1022 efx->n_rx_queues = min(rx_queues, wanted_ints);
1023 efx->n_channels = wanted_ints;
1024 for (i = 0; i < wanted_ints; i++)
8ceee660 1025 efx->channel[i].irq = xentries[i].vector;
8ceee660
BH		 1026 } else {
1027 /* Fall back to single channel MSI */
1028 efx->interrupt_mode = EFX_INT_MODE_MSI;
1029 EFX_ERR(efx, "could not enable MSI-X\n");
1030 }
1031 }
1032
1033 /* Try single interrupt MSI */
1034 if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
8831da7b 1035 efx->n_rx_queues = 1;
28b581ab 1036 efx->n_channels = 1;
8ceee660
BH		 1037 rc = pci_enable_msi(efx->pci_dev);
1038 if (rc == 0) {
1039 efx->channel[0].irq = efx->pci_dev->irq;
8ceee660
BH		 1040 } else {
1041 EFX_ERR(efx, "could not enable MSI\n");
1042 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1043 }
1044 }
1045
1046 /* Assume legacy interrupts */
1047 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
8831da7b 1048 efx->n_rx_queues = 1;
28b581ab 1049 efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
8ceee660
BH		 1050 efx->legacy_irq = efx->pci_dev->irq;
1051 }
1052}
1053
1054static void efx_remove_interrupts(struct efx_nic *efx)
1055{
1056 struct efx_channel *channel;
1057
1058 /* Remove MSI/MSI-X interrupts */
64ee3120 1059 efx_for_each_channel(channel, efx)
8ceee660
BH		 1060 channel->irq = 0;
1061 pci_disable_msi(efx->pci_dev);
1062 pci_disable_msix(efx->pci_dev);
1063
1064 /* Remove legacy interrupt */
1065 efx->legacy_irq = 0;
1066}
1067
8831da7b 1068static void efx_set_channels(struct efx_nic *efx)
8ceee660
BH		 1069{
1070 struct efx_tx_queue *tx_queue;
1071 struct efx_rx_queue *rx_queue;
8ceee660 1072
60ac1065 1073 efx_for_each_tx_queue(tx_queue, efx) {
28b581ab
NT		 1074 if (separate_tx_channels)
1075 tx_queue->channel = &efx->channel[efx->n_channels-1];
60ac1065
BH		 1076 else
1077 tx_queue->channel = &efx->channel[0];
1078 tx_queue->channel->used_flags |= EFX_USED_BY_TX;
1079 }
8ceee660 1080
8831da7b
BH		 1081 efx_for_each_rx_queue(rx_queue, efx) {
1082 rx_queue->channel = &efx->channel[rx_queue->queue];
1083 rx_queue->channel->used_flags |= EFX_USED_BY_RX;
8ceee660
BH		 1084 }
1085}
1086
1087static int efx_probe_nic(struct efx_nic *efx)
1088{
1089 int rc;
1090
1091 EFX_LOG(efx, "creating NIC\n");
1092
1093 /* Carry out hardware-type specific initialisation */
ef2b90ee 1094 rc = efx->type->probe(efx);
8ceee660
BH		 1095 if (rc)
1096 return rc;
1097
1098 /* Determine the number of channels and RX queues by trying to hook
1099 * in MSI-X interrupts. */
1100 efx_probe_interrupts(efx);
1101
8831da7b 1102 efx_set_channels(efx);
8ceee660
BH		 1103
1104 /* Initialise the interrupt moderation settings */
6fb70fd1 1105 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);
8ceee660
BH		 1106
1107 return 0;
1108}
1109
1110static void efx_remove_nic(struct efx_nic *efx)
1111{
1112 EFX_LOG(efx, "destroying NIC\n");
1113
1114 efx_remove_interrupts(efx);
ef2b90ee 1115 efx->type->remove(efx);
8ceee660
BH		 1116}
1117
1118/**************************************************************************
1119 *
1120 * NIC startup/shutdown
1121 *
1122 *************************************************************************/
1123
1124static int efx_probe_all(struct efx_nic *efx)
1125{
1126 struct efx_channel *channel;
1127 int rc;
1128
1129 /* Create NIC */
1130 rc = efx_probe_nic(efx);
1131 if (rc) {
1132 EFX_ERR(efx, "failed to create NIC\n");
1133 goto fail1;
1134 }
1135
1136 /* Create port */
1137 rc = efx_probe_port(efx);
1138 if (rc) {
1139 EFX_ERR(efx, "failed to create port\n");
1140 goto fail2;
1141 }
1142
1143 /* Create channels */
1144 efx_for_each_channel(channel, efx) {
1145 rc = efx_probe_channel(channel);
1146 if (rc) {
1147 EFX_ERR(efx, "failed to create channel %d\n",
1148 channel->channel);
1149 goto fail3;
1150 }
1151 }
56536e9c 1152 efx_set_channel_names(efx);
8ceee660
BH		 1153
1154 return 0;
1155
1156 fail3:
1157 efx_for_each_channel(channel, efx)
1158 efx_remove_channel(channel);
1159 efx_remove_port(efx);
1160 fail2:
1161 efx_remove_nic(efx);
1162 fail1:
1163 return rc;
1164}
1165
1166/* Called after previous invocation(s) of efx_stop_all, restarts the
1167 * port, kernel transmit queue, NAPI processing and hardware interrupts,
1168 * and ensures that the port is scheduled to be reconfigured.
1169 * This function is safe to call multiple times when the NIC is in any
1170 * state. */
1171static void efx_start_all(struct efx_nic *efx)
1172{
1173 struct efx_channel *channel;
1174
1175 EFX_ASSERT_RESET_SERIALISED(efx);
1176
1177 /* Check that it is appropriate to restart the interface. All
1178 * of these flags are safe to read under just the rtnl lock */
1179 if (efx->port_enabled)
1180 return;
1181 if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))
1182 return;
55668611 1183 if (efx_dev_registered(efx) && !netif_running(efx->net_dev))
8ceee660
BH		 1184 return;
1185
1186 /* Mark the port as enabled so port reconfigurations can start, then
1187 * restart the transmit interface early so the watchdog timer stops */
1188 efx_start_port(efx);
dacccc74
SH		 1189 if (efx_dev_registered(efx))
1190 efx_wake_queue(efx);
8ceee660
BH		 1191
1192 efx_for_each_channel(channel, efx)
1193 efx_start_channel(channel);
1194
152b6a62 1195 efx_nic_enable_interrupts(efx);
8ceee660 1196
8880f4ec
BH		 1197 /* Switch to event based MCDI completions after enabling interrupts.
1198 * If a reset has been scheduled, then we need to stay in polled mode.
1199 * Rather than serialising efx_mcdi_mode_event() [which sleeps] and
1200 * reset_pending [modified from an atomic context], we instead guarantee
1201 * that efx_mcdi_mode_poll() isn't reverted erroneously */
1202 efx_mcdi_mode_event(efx);
1203 if (efx->reset_pending != RESET_TYPE_NONE)
1204 efx_mcdi_mode_poll(efx);
1205
78c1f0a0
SH		 1206 /* Start the hardware monitor if there is one. Otherwise (we're link
1207 * event driven), we have to poll the PHY because after an event queue
1208 * flush, we could have a missed a link state change */
1209 if (efx->type->monitor != NULL) {
8ceee660
BH		 1210 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1211 efx_monitor_interval);
78c1f0a0
SH		 1212 } else {
1213 mutex_lock(&efx->mac_lock);
1214 if (efx->phy_op->poll(efx))
1215 efx_link_status_changed(efx);
1216 mutex_unlock(&efx->mac_lock);
1217 }
55edc6e6 1218
ef2b90ee 1219 efx->type->start_stats(efx);
8ceee660
BH		 1220}
1221
1222/* Flush all delayed work. Should only be called when no more delayed work
1223 * will be scheduled. This doesn't flush pending online resets (efx_reset),
1224 * since we're holding the rtnl_lock at this point. */
1225static void efx_flush_all(struct efx_nic *efx)
1226{
1227 struct efx_rx_queue *rx_queue;
1228
1229 /* Make sure the hardware monitor is stopped */
1230 cancel_delayed_work_sync(&efx->monitor_work);
1231
1232 /* Ensure that all RX slow refills are complete. */
b3475645 1233 efx_for_each_rx_queue(rx_queue, efx)
8ceee660 1234 cancel_delayed_work_sync(&rx_queue->work);
8ceee660
BH		 1235
1236 /* Stop scheduled port reconfigurations */
766ca0fa 1237 cancel_work_sync(&efx->mac_work);
8ceee660
BH		 1238}
1239
1240/* Quiesce hardware and software without bringing the link down.
1241 * Safe to call multiple times, when the nic and interface is in any
1242 * state. The caller is guaranteed to subsequently be in a position
1243 * to modify any hardware and software state they see fit without
1244 * taking locks. */
1245static void efx_stop_all(struct efx_nic *efx)
1246{
1247 struct efx_channel *channel;
1248
1249 EFX_ASSERT_RESET_SERIALISED(efx);
1250
1251 /* port_enabled can be read safely under the rtnl lock */
1252 if (!efx->port_enabled)
1253 return;
1254
ef2b90ee 1255 efx->type->stop_stats(efx);
55edc6e6 1256
8880f4ec
BH		 1257 /* Switch to MCDI polling on Siena before disabling interrupts */
1258 efx_mcdi_mode_poll(efx);
1259
8ceee660 1260 /* Disable interrupts and wait for ISR to complete */
152b6a62 1261 efx_nic_disable_interrupts(efx);
8ceee660
BH		 1262 if (efx->legacy_irq)
1263 synchronize_irq(efx->legacy_irq);
64ee3120 1264 efx_for_each_channel(channel, efx) {
8ceee660
BH		 1265 if (channel->irq)
1266 synchronize_irq(channel->irq);
b3475645 1267 }
8ceee660
BH		 1268
1269 /* Stop all NAPI processing and synchronous rx refills */
1270 efx_for_each_channel(channel, efx)
1271 efx_stop_channel(channel);
1272
1273 /* Stop all asynchronous port reconfigurations. Since all
1274 * event processing has already been stopped, there is no
1275 * window to loose phy events */
1276 efx_stop_port(efx);
1277
fdaa9aed 1278 /* Flush efx_mac_work(), refill_workqueue, monitor_work */
8ceee660
BH		 1279 efx_flush_all(efx);
1280
8ceee660
BH		 1281 /* Stop the kernel transmit interface late, so the watchdog
1282 * timer isn't ticking over the flush */
55668611 1283 if (efx_dev_registered(efx)) {
dacccc74 1284 efx_stop_queue(efx);
8ceee660
BH		 1285 netif_tx_lock_bh(efx->net_dev);
1286 netif_tx_unlock_bh(efx->net_dev);
1287 }
1288}
1289
1290static void efx_remove_all(struct efx_nic *efx)
1291{
1292 struct efx_channel *channel;
1293
1294 efx_for_each_channel(channel, efx)
1295 efx_remove_channel(channel);
1296 efx_remove_port(efx);
1297 efx_remove_nic(efx);
1298}
1299
8ceee660
BH		 1300/**************************************************************************
1301 *
1302 * Interrupt moderation
1303 *
1304 **************************************************************************/
1305
0d86ebd8
BH		 1306static unsigned irq_mod_ticks(int usecs, int resolution)
1307{
1308 if (usecs <= 0)
1309 return 0; /* cannot receive interrupts ahead of time :-) */
1310 if (usecs < resolution)
1311 return 1; /* never round down to 0 */
1312 return usecs / resolution;
1313}
1314
8ceee660 1315/* Set interrupt moderation parameters */
6fb70fd1
BH		 1316void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
1317 bool rx_adaptive)
8ceee660
BH		 1318{
1319 struct efx_tx_queue *tx_queue;
1320 struct efx_rx_queue *rx_queue;
152b6a62
BH		 1321 unsigned tx_ticks = irq_mod_ticks(tx_usecs, EFX_IRQ_MOD_RESOLUTION);
1322 unsigned rx_ticks = irq_mod_ticks(rx_usecs, EFX_IRQ_MOD_RESOLUTION);
8ceee660
BH		 1323
1324 EFX_ASSERT_RESET_SERIALISED(efx);
1325
1326 efx_for_each_tx_queue(tx_queue, efx)
0d86ebd8 1327 tx_queue->channel->irq_moderation = tx_ticks;
8ceee660 1328
6fb70fd1 1329 efx->irq_rx_adaptive = rx_adaptive;
0d86ebd8 1330 efx->irq_rx_moderation = rx_ticks;
8ceee660 1331 efx_for_each_rx_queue(rx_queue, efx)
0d86ebd8 1332 rx_queue->channel->irq_moderation = rx_ticks;
8ceee660
BH		 1333}
1334
1335/**************************************************************************
1336 *
1337 * Hardware monitor
1338 *
1339 **************************************************************************/
1340
1341/* Run periodically off the general workqueue. Serialised against
1342 * efx_reconfigure_port via the mac_lock */
1343static void efx_monitor(struct work_struct *data)
1344{
1345 struct efx_nic *efx = container_of(data, struct efx_nic,
1346 monitor_work.work);
8ceee660
BH		 1347
1348 EFX_TRACE(efx, "hardware monitor executing on CPU %d\n",
1349 raw_smp_processor_id());
ef2b90ee 1350 BUG_ON(efx->type->monitor == NULL);
8ceee660 1351
8ceee660
BH		 1352 /* If the mac_lock is already held then it is likely a port
1353 * reconfiguration is already in place, which will likely do
1354 * most of the work of check_hw() anyway. */
766ca0fa
BH		 1355 if (!mutex_trylock(&efx->mac_lock))
1356 goto out_requeue;
1357 if (!efx->port_enabled)
1358 goto out_unlock;
ef2b90ee 1359 efx->type->monitor(efx);
8ceee660 1360
766ca0fa 1361out_unlock:
8ceee660 1362 mutex_unlock(&efx->mac_lock);
766ca0fa 1363out_requeue:
8ceee660
BH		 1364 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1365 efx_monitor_interval);
1366}
1367
1368/**************************************************************************
1369 *
1370 * ioctls
1371 *
1372 *************************************************************************/
1373
1374/* Net device ioctl
1375 * Context: process, rtnl_lock() held.
1376 */
1377static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
1378{
767e468c 1379 struct efx_nic *efx = netdev_priv(net_dev);
68e7f45e 1380 struct mii_ioctl_data *data = if_mii(ifr);
8ceee660
BH		 1381
1382 EFX_ASSERT_RESET_SERIALISED(efx);
1383
68e7f45e
BH		 1384 /* Convert phy_id from older PRTAD/DEVAD format */
1385 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
1386 (data->phy_id & 0xfc00) == 0x0400)
1387 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
1388
1389 return mdio_mii_ioctl(&efx->mdio, data, cmd);
8ceee660
BH		 1390}
1391
1392/**************************************************************************
1393 *
1394 * NAPI interface
1395 *
1396 **************************************************************************/
1397
1398static int efx_init_napi(struct efx_nic *efx)
1399{
1400 struct efx_channel *channel;
8ceee660
BH		 1401
1402 efx_for_each_channel(channel, efx) {
1403 channel->napi_dev = efx->net_dev;
718cff1e
BH		 1404 netif_napi_add(channel->napi_dev, &channel->napi_str,
1405 efx_poll, napi_weight);
8ceee660
BH		 1406 }
1407 return 0;
8ceee660
BH		 1408}
1409
1410static void efx_fini_napi(struct efx_nic *efx)
1411{
1412 struct efx_channel *channel;
1413
1414 efx_for_each_channel(channel, efx) {
718cff1e
BH		 1415 if (channel->napi_dev)
1416 netif_napi_del(&channel->napi_str);
8ceee660
BH		 1417 channel->napi_dev = NULL;
1418 }
1419}
1420
1421/**************************************************************************
1422 *
1423 * Kernel netpoll interface
1424 *
1425 *************************************************************************/
1426
1427#ifdef CONFIG_NET_POLL_CONTROLLER
1428
1429/* Although in the common case interrupts will be disabled, this is not
1430 * guaranteed. However, all our work happens inside the NAPI callback,
1431 * so no locking is required.
1432 */
1433static void efx_netpoll(struct net_device *net_dev)
1434{
767e468c 1435 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1436 struct efx_channel *channel;
1437
64ee3120 1438 efx_for_each_channel(channel, efx)
8ceee660
BH		 1439 efx_schedule_channel(channel);
1440}
1441
1442#endif
1443
1444/**************************************************************************
1445 *
1446 * Kernel net device interface
1447 *
1448 *************************************************************************/
1449
1450/* Context: process, rtnl_lock() held. */
1451static int efx_net_open(struct net_device *net_dev)
1452{
767e468c 1453 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1454 EFX_ASSERT_RESET_SERIALISED(efx);
1455
1456 EFX_LOG(efx, "opening device %s on CPU %d\n", net_dev->name,
1457 raw_smp_processor_id());
1458
f4bd954e
BH		 1459 if (efx->state == STATE_DISABLED)
1460 return -EIO;
f8b87c17
BH		 1461 if (efx->phy_mode & PHY_MODE_SPECIAL)
1462 return -EBUSY;
8880f4ec
BH		 1463 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
1464 return -EIO;
f8b87c17 1465
78c1f0a0
SH		 1466 /* Notify the kernel of the link state polled during driver load,
1467 * before the monitor starts running */
1468 efx_link_status_changed(efx);
1469
8ceee660
BH		 1470 efx_start_all(efx);
1471 return 0;
1472}
1473
1474/* Context: process, rtnl_lock() held.
1475 * Note that the kernel will ignore our return code; this method
1476 * should really be a void.
1477 */
1478static int efx_net_stop(struct net_device *net_dev)
1479{
767e468c 1480 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1481
1482 EFX_LOG(efx, "closing %s on CPU %d\n", net_dev->name,
1483 raw_smp_processor_id());
1484
f4bd954e
BH		 1485 if (efx->state != STATE_DISABLED) {
1486 /* Stop the device and flush all the channels */
1487 efx_stop_all(efx);
1488 efx_fini_channels(efx);
1489 efx_init_channels(efx);
1490 }
8ceee660
BH		 1491
1492 return 0;
1493}
1494
5b9e207c 1495/* Context: process, dev_base_lock or RTNL held, non-blocking. */
8ceee660
BH		 1496static struct net_device_stats *efx_net_stats(struct net_device *net_dev)
1497{
767e468c 1498 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1499 struct efx_mac_stats *mac_stats = &efx->mac_stats;
1500 struct net_device_stats *stats = &net_dev->stats;
1501
55edc6e6 1502 spin_lock_bh(&efx->stats_lock);
ef2b90ee 1503 efx->type->update_stats(efx);
55edc6e6 1504 spin_unlock_bh(&efx->stats_lock);
8ceee660
BH		 1505
1506 stats->rx_packets = mac_stats->rx_packets;
1507 stats->tx_packets = mac_stats->tx_packets;
1508 stats->rx_bytes = mac_stats->rx_bytes;
1509 stats->tx_bytes = mac_stats->tx_bytes;
1510 stats->multicast = mac_stats->rx_multicast;
1511 stats->collisions = mac_stats->tx_collision;
1512 stats->rx_length_errors = (mac_stats->rx_gtjumbo +
1513 mac_stats->rx_length_error);
1514 stats->rx_over_errors = efx->n_rx_nodesc_drop_cnt;
1515 stats->rx_crc_errors = mac_stats->rx_bad;
1516 stats->rx_frame_errors = mac_stats->rx_align_error;
1517 stats->rx_fifo_errors = mac_stats->rx_overflow;
1518 stats->rx_missed_errors = mac_stats->rx_missed;
1519 stats->tx_window_errors = mac_stats->tx_late_collision;
1520
1521 stats->rx_errors = (stats->rx_length_errors +
1522 stats->rx_over_errors +
1523 stats->rx_crc_errors +
1524 stats->rx_frame_errors +
1525 stats->rx_fifo_errors +
1526 stats->rx_missed_errors +
1527 mac_stats->rx_symbol_error);
1528 stats->tx_errors = (stats->tx_window_errors +
1529 mac_stats->tx_bad);
1530
1531 return stats;
1532}
1533
1534/* Context: netif_tx_lock held, BHs disabled. */
1535static void efx_watchdog(struct net_device *net_dev)
1536{
767e468c 1537 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660 1538
739bb23d
BH		 1539 EFX_ERR(efx, "TX stuck with stop_count=%d port_enabled=%d:"
1540 " resetting channels\n",
1541 atomic_read(&efx->netif_stop_count), efx->port_enabled);
8ceee660 1542
739bb23d 1543 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
8ceee660
BH		 1544}
1545
1546
1547/* Context: process, rtnl_lock() held. */
1548static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
1549{
767e468c 1550 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1551 int rc = 0;
1552
1553 EFX_ASSERT_RESET_SERIALISED(efx);
1554
1555 if (new_mtu > EFX_MAX_MTU)
1556 return -EINVAL;
1557
1558 efx_stop_all(efx);
1559
1560 EFX_LOG(efx, "changing MTU to %d\n", new_mtu);
1561
1562 efx_fini_channels(efx);
d3245b28
BH		 1563
1564 mutex_lock(&efx->mac_lock);
1565 /* Reconfigure the MAC before enabling the dma queues so that
1566 * the RX buffers don't overflow */
8ceee660 1567 net_dev->mtu = new_mtu;
d3245b28
BH		 1568 efx->mac_op->reconfigure(efx);
1569 mutex_unlock(&efx->mac_lock);
1570
bc3c90a2 1571 efx_init_channels(efx);
8ceee660
BH		 1572
1573 efx_start_all(efx);
1574 return rc;
8ceee660
BH		 1575}
1576
1577static int efx_set_mac_address(struct net_device *net_dev, void *data)
1578{
767e468c 1579 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1580 struct sockaddr *addr = data;
1581 char *new_addr = addr->sa_data;
1582
1583 EFX_ASSERT_RESET_SERIALISED(efx);
1584
1585 if (!is_valid_ether_addr(new_addr)) {
e174961c
JB		 1586 EFX_ERR(efx, "invalid ethernet MAC address requested: %pM\n",
1587 new_addr);
8ceee660
BH		 1588 return -EINVAL;
1589 }
1590
1591 memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);
1592
1593 /* Reconfigure the MAC */
d3245b28
BH		 1594 mutex_lock(&efx->mac_lock);
1595 efx->mac_op->reconfigure(efx);
1596 mutex_unlock(&efx->mac_lock);
8ceee660
BH		 1597
1598 return 0;
1599}
1600
a816f75a 1601/* Context: netif_addr_lock held, BHs disabled. */
8ceee660
BH		 1602static void efx_set_multicast_list(struct net_device *net_dev)
1603{
767e468c 1604 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 1605 struct dev_mc_list *mc_list = net_dev->mc_list;
1606 union efx_multicast_hash *mc_hash = &efx->multicast_hash;
8ceee660
BH		 1607 u32 crc;
1608 int bit;
1609 int i;
1610
8be4f3e6 1611 efx->promiscuous = !!(net_dev->flags & IFF_PROMISC);
8ceee660
BH		 1612
1613 /* Build multicast hash table */
8be4f3e6 1614 if (efx->promiscuous || (net_dev->flags & IFF_ALLMULTI)) {
8ceee660
BH		 1615 memset(mc_hash, 0xff, sizeof(*mc_hash));
1616 } else {
1617 memset(mc_hash, 0x00, sizeof(*mc_hash));
1618 for (i = 0; i < net_dev->mc_count; i++) {
1619 crc = ether_crc_le(ETH_ALEN, mc_list->dmi_addr);
1620 bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
1621 set_bit_le(bit, mc_hash->byte);
1622 mc_list = mc_list->next;
1623 }
8ceee660 1624
8be4f3e6
BH		 1625 /* Broadcast packets go through the multicast hash filter.
1626 * ether_crc_le() of the broadcast address is 0xbe2612ff
1627 * so we always add bit 0xff to the mask.
1628 */
1629 set_bit_le(0xff, mc_hash->byte);
1630 }
a816f75a 1631
8be4f3e6
BH		 1632 if (efx->port_enabled)
1633 queue_work(efx->workqueue, &efx->mac_work);
1634 /* Otherwise efx_start_port() will do this */
8ceee660
BH		 1635}
1636
c3ecb9f3
SH		 1637static const struct net_device_ops efx_netdev_ops = {
1638 .ndo_open = efx_net_open,
1639 .ndo_stop = efx_net_stop,
1640 .ndo_get_stats = efx_net_stats,
1641 .ndo_tx_timeout = efx_watchdog,
1642 .ndo_start_xmit = efx_hard_start_xmit,
1643 .ndo_validate_addr = eth_validate_addr,
1644 .ndo_do_ioctl = efx_ioctl,
1645 .ndo_change_mtu = efx_change_mtu,
1646 .ndo_set_mac_address = efx_set_mac_address,
1647 .ndo_set_multicast_list = efx_set_multicast_list,
1648#ifdef CONFIG_NET_POLL_CONTROLLER
1649 .ndo_poll_controller = efx_netpoll,
1650#endif
1651};
1652
7dde596e
BH		 1653static void efx_update_name(struct efx_nic *efx)
1654{
1655 strcpy(efx->name, efx->net_dev->name);
1656 efx_mtd_rename(efx);
1657 efx_set_channel_names(efx);
1658}
1659
8ceee660
BH		 1660static int efx_netdev_event(struct notifier_block *this,
1661 unsigned long event, void *ptr)
1662{
d3208b5e 1663 struct net_device *net_dev = ptr;
8ceee660 1664
7dde596e
BH		 1665 if (net_dev->netdev_ops == &efx_netdev_ops &&
1666 event == NETDEV_CHANGENAME)
1667 efx_update_name(netdev_priv(net_dev));
8ceee660
BH		 1668
1669 return NOTIFY_DONE;
1670}
1671
1672static struct notifier_block efx_netdev_notifier = {
1673 .notifier_call = efx_netdev_event,
1674};
1675
06d5e193
BH		 1676static ssize_t
1677show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
1678{
1679 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
1680 return sprintf(buf, "%d\n", efx->phy_type);
1681}
1682static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);
1683
8ceee660
BH		 1684static int efx_register_netdev(struct efx_nic *efx)
1685{
1686 struct net_device *net_dev = efx->net_dev;
1687 int rc;
1688
1689 net_dev->watchdog_timeo = 5 * HZ;
1690 net_dev->irq = efx->pci_dev->irq;
c3ecb9f3 1691 net_dev->netdev_ops = &efx_netdev_ops;
8ceee660
BH		 1692 SET_NETDEV_DEV(net_dev, &efx->pci_dev->dev);
1693 SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);
1694
8ceee660 1695 /* Clear MAC statistics */
177dfcd8 1696 efx->mac_op->update_stats(efx);
8ceee660
BH		 1697 memset(&efx->mac_stats, 0, sizeof(efx->mac_stats));
1698
7dde596e 1699 rtnl_lock();
aed0628d
BH		 1700
1701 rc = dev_alloc_name(net_dev, net_dev->name);
1702 if (rc < 0)
1703 goto fail_locked;
7dde596e 1704 efx_update_name(efx);
aed0628d
BH		 1705
1706 rc = register_netdevice(net_dev);
1707 if (rc)
1708 goto fail_locked;
1709
1710 /* Always start with carrier off; PHY events will detect the link */
1711 netif_carrier_off(efx->net_dev);
1712
7dde596e 1713 rtnl_unlock();
8ceee660 1714
06d5e193
BH		 1715 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
1716 if (rc) {
1717 EFX_ERR(efx, "failed to init net dev attributes\n");
1718 goto fail_registered;
1719 }
1720
8ceee660 1721 return 0;
06d5e193 1722
aed0628d
BH		 1723fail_locked:
1724 rtnl_unlock();
1725 EFX_ERR(efx, "could not register net dev\n");
1726 return rc;
1727
06d5e193
BH		 1728fail_registered:
1729 unregister_netdev(net_dev);
1730 return rc;
8ceee660
BH		 1731}
1732
1733static void efx_unregister_netdev(struct efx_nic *efx)
1734{
1735 struct efx_tx_queue *tx_queue;
1736
1737 if (!efx->net_dev)
1738 return;
1739
767e468c 1740 BUG_ON(netdev_priv(efx->net_dev) != efx);
8ceee660
BH		 1741
1742 /* Free up any skbs still remaining. This has to happen before
1743 * we try to unregister the netdev as running their destructors
1744 * may be needed to get the device ref. count to 0. */
1745 efx_for_each_tx_queue(tx_queue, efx)
1746 efx_release_tx_buffers(tx_queue);
1747
55668611 1748 if (efx_dev_registered(efx)) {
8ceee660 1749 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
06d5e193 1750 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
8ceee660
BH		 1751 unregister_netdev(efx->net_dev);
1752 }
1753}
1754
1755/**************************************************************************
1756 *
1757 * Device reset and suspend
1758 *
1759 **************************************************************************/
1760
2467ca46
BH		 1761/* Tears down the entire software state and most of the hardware state
1762 * before reset. */
d3245b28 1763void efx_reset_down(struct efx_nic *efx, enum reset_type method)
8ceee660 1764{
8ceee660
BH		 1765 EFX_ASSERT_RESET_SERIALISED(efx);
1766
2467ca46
BH		 1767 efx_stop_all(efx);
1768 mutex_lock(&efx->mac_lock);
f4150724 1769 mutex_lock(&efx->spi_lock);
2467ca46 1770
8ceee660 1771 efx_fini_channels(efx);
4b988280
SH		 1772 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
1773 efx->phy_op->fini(efx);
ef2b90ee 1774 efx->type->fini(efx);
8ceee660
BH		 1775}
1776
2467ca46
BH		 1777/* This function will always ensure that the locks acquired in
1778 * efx_reset_down() are released. A failure return code indicates
1779 * that we were unable to reinitialise the hardware, and the
1780 * driver should be disabled. If ok is false, then the rx and tx
1781 * engines are not restarted, pending a RESET_DISABLE. */
d3245b28 1782int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
8ceee660
BH		 1783{
1784 int rc;
1785
2467ca46 1786 EFX_ASSERT_RESET_SERIALISED(efx);
8ceee660 1787
ef2b90ee 1788 rc = efx->type->init(efx);
8ceee660 1789 if (rc) {
2467ca46 1790 EFX_ERR(efx, "failed to initialise NIC\n");
eb9f6744 1791 goto fail;
8ceee660
BH		 1792 }
1793
eb9f6744
BH		 1794 if (!ok)
1795 goto fail;
1796
4b988280 1797 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
eb9f6744
BH		 1798 rc = efx->phy_op->init(efx);
1799 if (rc)
1800 goto fail;
1801 if (efx->phy_op->reconfigure(efx))
1802 EFX_ERR(efx, "could not restore PHY settings\n");
4b988280
SH		 1803 }
1804
eb9f6744 1805 efx->mac_op->reconfigure(efx);
8ceee660 1806
eb9f6744
BH		 1807 efx_init_channels(efx);
1808
1809 mutex_unlock(&efx->spi_lock);
1810 mutex_unlock(&efx->mac_lock);
1811
1812 efx_start_all(efx);
1813
1814 return 0;
1815
1816fail:
1817 efx->port_initialized = false;
2467ca46 1818
f4150724 1819 mutex_unlock(&efx->spi_lock);
2467ca46
BH		 1820 mutex_unlock(&efx->mac_lock);
1821
8ceee660
BH		 1822 return rc;
1823}
1824
eb9f6744
BH		 1825/* Reset the NIC using the specified method. Note that the reset may
1826 * fail, in which case the card will be left in an unusable state.
8ceee660 1827 *
eb9f6744 1828 * Caller must hold the rtnl_lock.
8ceee660 1829 */
eb9f6744 1830int efx_reset(struct efx_nic *efx, enum reset_type method)
8ceee660 1831{
eb9f6744
BH		 1832 int rc, rc2;
1833 bool disabled;
8ceee660 1834
c459302d 1835 EFX_INFO(efx, "resetting (%s)\n", RESET_TYPE(method));
8ceee660 1836
d3245b28 1837 efx_reset_down(efx, method);
8ceee660 1838
ef2b90ee 1839 rc = efx->type->reset(efx, method);
8ceee660
BH		 1840 if (rc) {
1841 EFX_ERR(efx, "failed to reset hardware\n");
eb9f6744 1842 goto out;
8ceee660
BH		 1843 }
1844
1845 /* Allow resets to be rescheduled. */
1846 efx->reset_pending = RESET_TYPE_NONE;
1847
1848 /* Reinitialise bus-mastering, which may have been turned off before
1849 * the reset was scheduled. This is still appropriate, even in the
1850 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
1851 * can respond to requests. */
1852 pci_set_master(efx->pci_dev);
1853
eb9f6744 1854out:
8ceee660 1855 /* Leave device stopped if necessary */
eb9f6744
BH		 1856 disabled = rc || method == RESET_TYPE_DISABLE;
1857 rc2 = efx_reset_up(efx, method, !disabled);
1858 if (rc2) {
1859 disabled = true;
1860 if (!rc)
1861 rc = rc2;
8ceee660
BH		 1862 }
1863
eb9f6744 1864 if (disabled) {
f4bd954e
BH		 1865 EFX_ERR(efx, "has been disabled\n");
1866 efx->state = STATE_DISABLED;
f4bd954e
BH		 1867 } else {
1868 EFX_LOG(efx, "reset complete\n");
1869 }
8ceee660
BH		 1870 return rc;
1871}
1872
1873/* The worker thread exists so that code that cannot sleep can
1874 * schedule a reset for later.
1875 */
1876static void efx_reset_work(struct work_struct *data)
1877{
eb9f6744 1878 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
8ceee660 1879
eb9f6744
BH		 1880 /* If we're not RUNNING then don't reset. Leave the reset_pending
1881 * flag set so that efx_pci_probe_main will be retried */
1882 if (efx->state != STATE_RUNNING) {
1883 EFX_INFO(efx, "scheduled reset quenched. NIC not RUNNING\n");
1884 return;
1885 }
1886
1887 rtnl_lock();
1888 if (efx_reset(efx, efx->reset_pending))
1889 dev_close(efx->net_dev);
1890 rtnl_unlock();
8ceee660
BH		 1891}
1892
1893void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
1894{
1895 enum reset_type method;
1896
1897 if (efx->reset_pending != RESET_TYPE_NONE) {
1898 EFX_INFO(efx, "quenching already scheduled reset\n");
1899 return;
1900 }
1901
1902 switch (type) {
1903 case RESET_TYPE_INVISIBLE:
1904 case RESET_TYPE_ALL:
1905 case RESET_TYPE_WORLD:
1906 case RESET_TYPE_DISABLE:
1907 method = type;
1908 break;
1909 case RESET_TYPE_RX_RECOVERY:
1910 case RESET_TYPE_RX_DESC_FETCH:
1911 case RESET_TYPE_TX_DESC_FETCH:
1912 case RESET_TYPE_TX_SKIP:
1913 method = RESET_TYPE_INVISIBLE;
1914 break;
8880f4ec 1915 case RESET_TYPE_MC_FAILURE:
8ceee660
BH		 1916 default:
1917 method = RESET_TYPE_ALL;
1918 break;
1919 }
1920
1921 if (method != type)
c459302d
BH		 1922 EFX_LOG(efx, "scheduling %s reset for %s\n",
1923 RESET_TYPE(method), RESET_TYPE(type));
8ceee660 1924 else
c459302d 1925 EFX_LOG(efx, "scheduling %s reset\n", RESET_TYPE(method));
8ceee660
BH		 1926
1927 efx->reset_pending = method;
1928
8880f4ec
BH		 1929 /* efx_process_channel() will no longer read events once a
1930 * reset is scheduled. So switch back to poll'd MCDI completions. */
1931 efx_mcdi_mode_poll(efx);
1932
1ab00629 1933 queue_work(reset_workqueue, &efx->reset_work);
8ceee660
BH		 1934}
1935
1936/**************************************************************************
1937 *
1938 * List of NICs we support
1939 *
1940 **************************************************************************/
1941
1942/* PCI device ID table */
1943static struct pci_device_id efx_pci_table[] __devinitdata = {
1944 {PCI_DEVICE(EFX_VENDID_SFC, FALCON_A_P_DEVID),
daeda630 1945 .driver_data = (unsigned long) &falcon_a1_nic_type},
8ceee660 1946 {PCI_DEVICE(EFX_VENDID_SFC, FALCON_B_P_DEVID),
daeda630 1947 .driver_data = (unsigned long) &falcon_b0_nic_type},
8880f4ec
BH		 1948 {PCI_DEVICE(EFX_VENDID_SFC, BETHPAGE_A_P_DEVID),
1949 .driver_data = (unsigned long) &siena_a0_nic_type},
1950 {PCI_DEVICE(EFX_VENDID_SFC, SIENA_A_P_DEVID),
1951 .driver_data = (unsigned long) &siena_a0_nic_type},
8ceee660
BH		 1952 {0} /* end of list */
1953};
1954
1955/**************************************************************************
1956 *
3759433d 1957 * Dummy PHY/MAC operations
8ceee660 1958 *
01aad7b6 1959 * Can be used for some unimplemented operations
8ceee660
BH		 1960 * Needed so all function pointers are valid and do not have to be tested
1961 * before use
1962 *
1963 **************************************************************************/
1964int efx_port_dummy_op_int(struct efx_nic *efx)
1965{
1966 return 0;
1967}
1968void efx_port_dummy_op_void(struct efx_nic *efx) {}
398468ed
BH		 1969void efx_port_dummy_op_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
1970{
1971}
fdaa9aed
SH		 1972bool efx_port_dummy_op_poll(struct efx_nic *efx)
1973{
1974 return false;
1975}
8ceee660
BH		 1976
1977static struct efx_phy_operations efx_dummy_phy_operations = {
1978 .init = efx_port_dummy_op_int,
d3245b28 1979 .reconfigure = efx_port_dummy_op_int,
fdaa9aed 1980 .poll = efx_port_dummy_op_poll,
8ceee660 1981 .fini = efx_port_dummy_op_void,
8ceee660
BH		 1982};
1983
8ceee660
BH		 1984/**************************************************************************
1985 *
1986 * Data housekeeping
1987 *
1988 **************************************************************************/
1989
1990/* This zeroes out and then fills in the invariants in a struct
1991 * efx_nic (including all sub-structures).
1992 */
1993static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
1994 struct pci_dev *pci_dev, struct net_device *net_dev)
1995{
1996 struct efx_channel *channel;
1997 struct efx_tx_queue *tx_queue;
1998 struct efx_rx_queue *rx_queue;
1ab00629 1999 int i;
8ceee660
BH		 2000
2001 /* Initialise common structures */
2002 memset(efx, 0, sizeof(*efx));
2003 spin_lock_init(&efx->biu_lock);
ab867461 2004 mutex_init(&efx->mdio_lock);
f4150724 2005 mutex_init(&efx->spi_lock);
76884835
BH		 2006#ifdef CONFIG_SFC_MTD
2007 INIT_LIST_HEAD(&efx->mtd_list);
2008#endif
8ceee660
BH		 2009 INIT_WORK(&efx->reset_work, efx_reset_work);
2010 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
2011 efx->pci_dev = pci_dev;
2012 efx->state = STATE_INIT;
2013 efx->reset_pending = RESET_TYPE_NONE;
2014 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
8ceee660
BH		 2015
2016 efx->net_dev = net_dev;
dc8cfa55 2017 efx->rx_checksum_enabled = true;
8ceee660
BH		 2018 spin_lock_init(&efx->netif_stop_lock);
2019 spin_lock_init(&efx->stats_lock);
2020 mutex_init(&efx->mac_lock);
b895d73e 2021 efx->mac_op = type->default_mac_ops;
8ceee660 2022 efx->phy_op = &efx_dummy_phy_operations;
68e7f45e 2023 efx->mdio.dev = net_dev;
766ca0fa 2024 INIT_WORK(&efx->mac_work, efx_mac_work);
8ceee660
BH		 2025 atomic_set(&efx->netif_stop_count, 1);
2026
2027 for (i = 0; i < EFX_MAX_CHANNELS; i++) {
2028 channel = &efx->channel[i];
2029 channel->efx = efx;
2030 channel->channel = i;
dc8cfa55 2031 channel->work_pending = false;
8ceee660 2032 }
60ac1065 2033 for (i = 0; i < EFX_TX_QUEUE_COUNT; i++) {
8ceee660
BH		 2034 tx_queue = &efx->tx_queue[i];
2035 tx_queue->efx = efx;
2036 tx_queue->queue = i;
2037 tx_queue->buffer = NULL;
2038 tx_queue->channel = &efx->channel[0]; /* for safety */
b9b39b62 2039 tx_queue->tso_headers_free = NULL;
8ceee660
BH		 2040 }
2041 for (i = 0; i < EFX_MAX_RX_QUEUES; i++) {
2042 rx_queue = &efx->rx_queue[i];
2043 rx_queue->efx = efx;
2044 rx_queue->queue = i;
2045 rx_queue->channel = &efx->channel[0]; /* for safety */
2046 rx_queue->buffer = NULL;
2047 spin_lock_init(&rx_queue->add_lock);
2048 INIT_DELAYED_WORK(&rx_queue->work, efx_rx_work);
2049 }
2050
2051 efx->type = type;
2052
8ceee660 2053 /* As close as we can get to guaranteeing that we don't overflow */
3ffeabdd
BH		 2054 BUILD_BUG_ON(EFX_EVQ_SIZE < EFX_TXQ_SIZE + EFX_RXQ_SIZE);
2055
8ceee660
BH		 2056 EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);
2057
2058 /* Higher numbered interrupt modes are less capable! */
2059 efx->interrupt_mode = max(efx->type->max_interrupt_mode,
2060 interrupt_mode);
2061
6977dc63
BH		 2062 /* Would be good to use the net_dev name, but we're too early */
2063 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
2064 pci_name(pci_dev));
2065 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
1ab00629
SH		 2066 if (!efx->workqueue)
2067 return -ENOMEM;
8d9853d9 2068
8ceee660 2069 return 0;
8ceee660
BH		 2070}
2071
2072static void efx_fini_struct(struct efx_nic *efx)
2073{
2074 if (efx->workqueue) {
2075 destroy_workqueue(efx->workqueue);
2076 efx->workqueue = NULL;
2077 }
2078}
2079
2080/**************************************************************************
2081 *
2082 * PCI interface
2083 *
2084 **************************************************************************/
2085
2086/* Main body of final NIC shutdown code
2087 * This is called only at module unload (or hotplug removal).
2088 */
2089static void efx_pci_remove_main(struct efx_nic *efx)
2090{
152b6a62 2091 efx_nic_fini_interrupt(efx);
8ceee660
BH		 2092 efx_fini_channels(efx);
2093 efx_fini_port(efx);
ef2b90ee 2094 efx->type->fini(efx);
8ceee660
BH		 2095 efx_fini_napi(efx);
2096 efx_remove_all(efx);
2097}
2098
2099/* Final NIC shutdown
2100 * This is called only at module unload (or hotplug removal).
2101 */
2102static void efx_pci_remove(struct pci_dev *pci_dev)
2103{
2104 struct efx_nic *efx;
2105
2106 efx = pci_get_drvdata(pci_dev);
2107 if (!efx)
2108 return;
2109
2110 /* Mark the NIC as fini, then stop the interface */
2111 rtnl_lock();
2112 efx->state = STATE_FINI;
2113 dev_close(efx->net_dev);
2114
2115 /* Allow any queued efx_resets() to complete */
2116 rtnl_unlock();
2117
8ceee660
BH		 2118 efx_unregister_netdev(efx);
2119
7dde596e
BH		 2120 efx_mtd_remove(efx);
2121
8ceee660
BH		 2122 /* Wait for any scheduled resets to complete. No more will be
2123 * scheduled from this point because efx_stop_all() has been
2124 * called, we are no longer registered with driverlink, and
2125 * the net_device's have been removed. */
1ab00629 2126 cancel_work_sync(&efx->reset_work);
8ceee660
BH		 2127
2128 efx_pci_remove_main(efx);
2129
8ceee660
BH		 2130 efx_fini_io(efx);
2131 EFX_LOG(efx, "shutdown successful\n");
2132
2133 pci_set_drvdata(pci_dev, NULL);
2134 efx_fini_struct(efx);
2135 free_netdev(efx->net_dev);
2136};
2137
2138/* Main body of NIC initialisation
2139 * This is called at module load (or hotplug insertion, theoretically).
2140 */
2141static int efx_pci_probe_main(struct efx_nic *efx)
2142{
2143 int rc;
2144
2145 /* Do start-of-day initialisation */
2146 rc = efx_probe_all(efx);
2147 if (rc)
2148 goto fail1;
2149
2150 rc = efx_init_napi(efx);
2151 if (rc)
2152 goto fail2;
2153
ef2b90ee 2154 rc = efx->type->init(efx);
8ceee660
BH		 2155 if (rc) {
2156 EFX_ERR(efx, "failed to initialise NIC\n");
278c0621 2157 goto fail3;
8ceee660
BH		 2158 }
2159
2160 rc = efx_init_port(efx);
2161 if (rc) {
2162 EFX_ERR(efx, "failed to initialise port\n");
278c0621 2163 goto fail4;
8ceee660
BH		 2164 }
2165
bc3c90a2 2166 efx_init_channels(efx);
8ceee660 2167
152b6a62 2168 rc = efx_nic_init_interrupt(efx);
8ceee660 2169 if (rc)
278c0621 2170 goto fail5;
8ceee660
BH		 2171
2172 return 0;
2173
278c0621 2174 fail5:
bc3c90a2 2175 efx_fini_channels(efx);
8ceee660 2176 efx_fini_port(efx);
8ceee660 2177 fail4:
ef2b90ee 2178 efx->type->fini(efx);
8ceee660
BH		 2179 fail3:
2180 efx_fini_napi(efx);
2181 fail2:
2182 efx_remove_all(efx);
2183 fail1:
2184 return rc;
2185}
2186
2187/* NIC initialisation
2188 *
2189 * This is called at module load (or hotplug insertion,
2190 * theoretically). It sets up PCI mappings, tests and resets the NIC,
2191 * sets up and registers the network devices with the kernel and hooks
2192 * the interrupt service routine. It does not prepare the device for
2193 * transmission; this is left to the first time one of the network
2194 * interfaces is brought up (i.e. efx_net_open).
2195 */
2196static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
2197 const struct pci_device_id *entry)
2198{
2199 struct efx_nic_type *type = (struct efx_nic_type *) entry->driver_data;
2200 struct net_device *net_dev;
2201 struct efx_nic *efx;
2202 int i, rc;
2203
2204 /* Allocate and initialise a struct net_device and struct efx_nic */
2205 net_dev = alloc_etherdev(sizeof(*efx));
2206 if (!net_dev)
2207 return -ENOMEM;
c383b537 2208 net_dev->features |= (type->offload_features | NETIF_F_SG |
97bc5415
BH		 2209 NETIF_F_HIGHDMA | NETIF_F_TSO |
2210 NETIF_F_GRO);
738a8f4b
BH		 2211 if (type->offload_features & NETIF_F_V6_CSUM)
2212 net_dev->features |= NETIF_F_TSO6;
28506563
BH		 2213 /* Mask for features that also apply to VLAN devices */
2214 net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
740847da 2215 NETIF_F_HIGHDMA | NETIF_F_TSO);
767e468c 2216 efx = netdev_priv(net_dev);
8ceee660
BH		 2217 pci_set_drvdata(pci_dev, efx);
2218 rc = efx_init_struct(efx, type, pci_dev, net_dev);
2219 if (rc)
2220 goto fail1;
2221
2222 EFX_INFO(efx, "Solarflare Communications NIC detected\n");
2223
2224 /* Set up basic I/O (BAR mappings etc) */
2225 rc = efx_init_io(efx);
2226 if (rc)
2227 goto fail2;
2228
2229 /* No serialisation is required with the reset path because
2230 * we're in STATE_INIT. */
2231 for (i = 0; i < 5; i++) {
2232 rc = efx_pci_probe_main(efx);
8ceee660
BH		 2233
2234 /* Serialise against efx_reset(). No more resets will be
2235 * scheduled since efx_stop_all() has been called, and we
2236 * have not and never have been registered with either
2237 * the rtnetlink or driverlink layers. */
1ab00629 2238 cancel_work_sync(&efx->reset_work);
8ceee660 2239
fa402b2e
SH		 2240 if (rc == 0) {
2241 if (efx->reset_pending != RESET_TYPE_NONE) {
2242 /* If there was a scheduled reset during
2243 * probe, the NIC is probably hosed anyway */
2244 efx_pci_remove_main(efx);
2245 rc = -EIO;
2246 } else {
2247 break;
2248 }
2249 }
2250
8ceee660
BH		 2251 /* Retry if a recoverably reset event has been scheduled */
2252 if ((efx->reset_pending != RESET_TYPE_INVISIBLE) &&
2253 (efx->reset_pending != RESET_TYPE_ALL))
2254 goto fail3;
2255
2256 efx->reset_pending = RESET_TYPE_NONE;
2257 }
2258
2259 if (rc) {
2260 EFX_ERR(efx, "Could not reset NIC\n");
2261 goto fail4;
2262 }
2263
55edc6e6
BH		 2264 /* Switch to the running state before we expose the device to the OS,
2265 * so that dev_open()|efx_start_all() will actually start the device */
8ceee660 2266 efx->state = STATE_RUNNING;
7dde596e 2267
8ceee660
BH		 2268 rc = efx_register_netdev(efx);
2269 if (rc)
2270 goto fail5;
2271
2272 EFX_LOG(efx, "initialisation successful\n");
a5211bb5
BH		 2273
2274 rtnl_lock();
2275 efx_mtd_probe(efx); /* allowed to fail */
2276 rtnl_unlock();
8ceee660
BH		 2277 return 0;
2278
2279 fail5:
2280 efx_pci_remove_main(efx);
2281 fail4:
2282 fail3:
2283 efx_fini_io(efx);
2284 fail2:
2285 efx_fini_struct(efx);
2286 fail1:
2287 EFX_LOG(efx, "initialisation failed. rc=%d\n", rc);
2288 free_netdev(net_dev);
2289 return rc;
2290}
2291
89c758fa
BH		 2292static int efx_pm_freeze(struct device *dev)
2293{
2294 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2295
2296 efx->state = STATE_FINI;
2297
2298 netif_device_detach(efx->net_dev);
2299
2300 efx_stop_all(efx);
2301 efx_fini_channels(efx);
2302
2303 return 0;
2304}
2305
2306static int efx_pm_thaw(struct device *dev)
2307{
2308 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2309
2310 efx->state = STATE_INIT;
2311
2312 efx_init_channels(efx);
2313
2314 mutex_lock(&efx->mac_lock);
2315 efx->phy_op->reconfigure(efx);
2316 mutex_unlock(&efx->mac_lock);
2317
2318 efx_start_all(efx);
2319
2320 netif_device_attach(efx->net_dev);
2321
2322 efx->state = STATE_RUNNING;
2323
2324 efx->type->resume_wol(efx);
2325
2326 return 0;
2327}
2328
2329static int efx_pm_poweroff(struct device *dev)
2330{
2331 struct pci_dev *pci_dev = to_pci_dev(dev);
2332 struct efx_nic *efx = pci_get_drvdata(pci_dev);
2333
2334 efx->type->fini(efx);
2335
2336 efx->reset_pending = RESET_TYPE_NONE;
2337
2338 pci_save_state(pci_dev);
2339 return pci_set_power_state(pci_dev, PCI_D3hot);
2340}
2341
2342/* Used for both resume and restore */
2343static int efx_pm_resume(struct device *dev)
2344{
2345 struct pci_dev *pci_dev = to_pci_dev(dev);
2346 struct efx_nic *efx = pci_get_drvdata(pci_dev);
2347 int rc;
2348
2349 rc = pci_set_power_state(pci_dev, PCI_D0);
2350 if (rc)
2351 return rc;
2352 pci_restore_state(pci_dev);
2353 rc = pci_enable_device(pci_dev);
2354 if (rc)
2355 return rc;
2356 pci_set_master(efx->pci_dev);
2357 rc = efx->type->reset(efx, RESET_TYPE_ALL);
2358 if (rc)
2359 return rc;
2360 rc = efx->type->init(efx);
2361 if (rc)
2362 return rc;
2363 efx_pm_thaw(dev);
2364 return 0;
2365}
2366
2367static int efx_pm_suspend(struct device *dev)
2368{
2369 int rc;
2370
2371 efx_pm_freeze(dev);
2372 rc = efx_pm_poweroff(dev);
2373 if (rc)
2374 efx_pm_resume(dev);
2375 return rc;
2376}
2377
2378static struct dev_pm_ops efx_pm_ops = {
2379 .suspend = efx_pm_suspend,
2380 .resume = efx_pm_resume,
2381 .freeze = efx_pm_freeze,
2382 .thaw = efx_pm_thaw,
2383 .poweroff = efx_pm_poweroff,
2384 .restore = efx_pm_resume,
2385};
2386
8ceee660
BH		 2387static struct pci_driver efx_pci_driver = {
2388 .name = EFX_DRIVER_NAME,
2389 .id_table = efx_pci_table,
2390 .probe = efx_pci_probe,
2391 .remove = efx_pci_remove,
89c758fa 2392 .driver.pm = &efx_pm_ops,
8ceee660
BH		 2393};
2394
2395/**************************************************************************
2396 *
2397 * Kernel module interface
2398 *
2399 *************************************************************************/
2400
2401module_param(interrupt_mode, uint, 0444);
2402MODULE_PARM_DESC(interrupt_mode,
2403 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
2404
2405static int __init efx_init_module(void)
2406{
2407 int rc;
2408
2409 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
2410
2411 rc = register_netdevice_notifier(&efx_netdev_notifier);
2412 if (rc)
2413 goto err_notifier;
2414
2415 refill_workqueue = create_workqueue("sfc_refill");
2416 if (!refill_workqueue) {
2417 rc = -ENOMEM;
2418 goto err_refill;
2419 }
1ab00629
SH		 2420 reset_workqueue = create_singlethread_workqueue("sfc_reset");
2421 if (!reset_workqueue) {
2422 rc = -ENOMEM;
2423 goto err_reset;
2424 }
8ceee660
BH		 2425
2426 rc = pci_register_driver(&efx_pci_driver);
2427 if (rc < 0)
2428 goto err_pci;
2429
2430 return 0;
2431
2432 err_pci:
1ab00629
SH		 2433 destroy_workqueue(reset_workqueue);
2434 err_reset:
8ceee660
BH		 2435 destroy_workqueue(refill_workqueue);
2436 err_refill:
2437 unregister_netdevice_notifier(&efx_netdev_notifier);
2438 err_notifier:
2439 return rc;
2440}
2441
2442static void __exit efx_exit_module(void)
2443{
2444 printk(KERN_INFO "Solarflare NET driver unloading\n");
2445
2446 pci_unregister_driver(&efx_pci_driver);
1ab00629 2447 destroy_workqueue(reset_workqueue);
8ceee660
BH		 2448 destroy_workqueue(refill_workqueue);
2449 unregister_netdevice_notifier(&efx_netdev_notifier);
2450
2451}
2452
2453module_init(efx_init_module);
2454module_exit(efx_exit_module);
2455
2456MODULE_AUTHOR("Michael Brown <mbrown@fensystems.co.uk> and "
2457 "Solarflare Communications");
2458MODULE_DESCRIPTION("Solarflare Communications network driver");
2459MODULE_LICENSE("GPL");
2460MODULE_DEVICE_TABLE(pci, efx_pci_table);
Clone of davem's net-next-2.6 tree