]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/cxgb4/cxgb4_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/linville/wirel...
[net-next-2.6.git] / drivers / net / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if_vlan.h>
45#include <linux/init.h>
46#include <linux/log2.h>
47#include <linux/mdio.h>
48#include <linux/module.h>
49#include <linux/moduleparam.h>
50#include <linux/mutex.h>
51#include <linux/netdevice.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54#include <linux/rtnetlink.h>
55#include <linux/sched.h>
56#include <linux/seq_file.h>
57#include <linux/sockios.h>
58#include <linux/vmalloc.h>
59#include <linux/workqueue.h>
60#include <net/neighbour.h>
61#include <net/netevent.h>
62#include <asm/uaccess.h>
63
64#include "cxgb4.h"
65#include "t4_regs.h"
66#include "t4_msg.h"
67#include "t4fw_api.h"
68#include "l2t.h"
69
99e6d065 70#define DRV_VERSION "1.3.0-ko"
b8ff05a9
DM		 71#define DRV_DESC "Chelsio T4 Network Driver"
72
73/*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78#define MAX_SGE_TIMERVAL 200U
79
7ee9ff94
CL		 80#ifdef CONFIG_PCI_IOV
81/*
82 * Virtual Function provisioning constants. We need two extra Ingress Queues
83 * with Interrupt capability to serve as the VF's Firmware Event Queue and
84 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85 * Lists associated with them). For each Ethernet/Control Egress Queue and
86 * for each Free List, we need an Egress Context.
87 */
88enum {
89 VFRES_NPORTS = 1, /* # of "ports" per VF */
90 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
91
92 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
93 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
94 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
96 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
97 VFRES_TC = 0, /* PCI-E traffic class */
98 VFRES_NEXACTF = 16, /* # of exact MPS filters */
99
100 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102};
103
104/*
105 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
106 * static and likely not to be useful in the long run. We really need to
107 * implement some form of persistent configuration which the firmware
108 * controls.
109 */
110static unsigned int pfvfres_pmask(struct adapter *adapter,
111 unsigned int pf, unsigned int vf)
112{
113 unsigned int portn, portvec;
114
115 /*
116 * Give PF's access to all of the ports.
117 */
118 if (vf == 0)
119 return FW_PFVF_CMD_PMASK_MASK;
120
121 /*
122 * For VFs, we'll assign them access to the ports based purely on the
123 * PF. We assign active ports in order, wrapping around if there are
124 * fewer active ports than PFs: e.g. active port[pf % nports].
125 * Unfortunately the adapter's port_info structs haven't been
126 * initialized yet so we have to compute this.
127 */
128 if (adapter->params.nports == 0)
129 return 0;
130
131 portn = pf % adapter->params.nports;
132 portvec = adapter->params.portvec;
133 for (;;) {
134 /*
135 * Isolate the lowest set bit in the port vector. If we're at
136 * the port number that we want, return that as the pmask.
137 * otherwise mask that bit out of the port vector and
138 * decrement our port number ...
139 */
140 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141 if (portn == 0)
142 return pmask;
143 portn--;
144 portvec &= ~pmask;
145 }
146 /*NOTREACHED*/
147}
148#endif
149
b8ff05a9
DM		 150enum {
151 MEMWIN0_APERTURE = 65536,
152 MEMWIN0_BASE = 0x30000,
153 MEMWIN1_APERTURE = 32768,
154 MEMWIN1_BASE = 0x28000,
155 MEMWIN2_APERTURE = 2048,
156 MEMWIN2_BASE = 0x1b800,
157};
158
159enum {
160 MAX_TXQ_ENTRIES = 16384,
161 MAX_CTRL_TXQ_ENTRIES = 1024,
162 MAX_RSPQ_ENTRIES = 16384,
163 MAX_RX_BUFFERS = 16384,
164 MIN_TXQ_ENTRIES = 32,
165 MIN_CTRL_TXQ_ENTRIES = 32,
166 MIN_RSPQ_ENTRIES = 128,
167 MIN_FL_ENTRIES = 16
168};
169
170#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
060e0c75 174#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 175
176static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 177 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 178 CH_DEVICE(0x4001, -1),
179 CH_DEVICE(0x4002, -1),
180 CH_DEVICE(0x4003, -1),
181 CH_DEVICE(0x4004, -1),
182 CH_DEVICE(0x4005, -1),
183 CH_DEVICE(0x4006, -1),
184 CH_DEVICE(0x4007, -1),
185 CH_DEVICE(0x4008, -1),
186 CH_DEVICE(0x4009, -1),
187 CH_DEVICE(0x400a, -1),
188 CH_DEVICE(0x4401, 4),
189 CH_DEVICE(0x4402, 4),
190 CH_DEVICE(0x4403, 4),
191 CH_DEVICE(0x4404, 4),
192 CH_DEVICE(0x4405, 4),
193 CH_DEVICE(0x4406, 4),
194 CH_DEVICE(0x4407, 4),
195 CH_DEVICE(0x4408, 4),
196 CH_DEVICE(0x4409, 4),
197 CH_DEVICE(0x440a, 4),
b8ff05a9
DM		 198 { 0, }
199};
200
201#define FW_FNAME "cxgb4/t4fw.bin"
202
203MODULE_DESCRIPTION(DRV_DESC);
204MODULE_AUTHOR("Chelsio Communications");
205MODULE_LICENSE("Dual BSD/GPL");
206MODULE_VERSION(DRV_VERSION);
207MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
208MODULE_FIRMWARE(FW_FNAME);
209
210static int dflt_msg_enable = DFLT_MSG_ENABLE;
211
212module_param(dflt_msg_enable, int, 0644);
213MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
214
215/*
216 * The driver uses the best interrupt scheme available on a platform in the
217 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
218 * of these schemes the driver may consider as follows:
219 *
220 * msi = 2: choose from among all three options
221 * msi = 1: only consider MSI and INTx interrupts
222 * msi = 0: force INTx interrupts
223 */
224static int msi = 2;
225
226module_param(msi, int, 0644);
227MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
228
229/*
230 * Queue interrupt hold-off timer values. Queues default to the first of these
231 * upon creation.
232 */
233static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
234
235module_param_array(intr_holdoff, uint, NULL, 0644);
236MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
237 "0..4 in microseconds");
238
239static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
240
241module_param_array(intr_cnt, uint, NULL, 0644);
242MODULE_PARM_DESC(intr_cnt,
243 "thresholds 1..3 for queue interrupt packet counters");
244
245static int vf_acls;
246
247#ifdef CONFIG_PCI_IOV
248module_param(vf_acls, bool, 0644);
249MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
250
251static unsigned int num_vf[4];
252
253module_param_array(num_vf, uint, NULL, 0644);
254MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
255#endif
256
257static struct dentry *cxgb4_debugfs_root;
258
259static LIST_HEAD(adapter_list);
260static DEFINE_MUTEX(uld_mutex);
261static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
262static const char *uld_str[] = { "RDMA", "iSCSI" };
263
264static void link_report(struct net_device *dev)
265{
266 if (!netif_carrier_ok(dev))
267 netdev_info(dev, "link down\n");
268 else {
269 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
270
271 const char *s = "10Mbps";
272 const struct port_info *p = netdev_priv(dev);
273
274 switch (p->link_cfg.speed) {
275 case SPEED_10000:
276 s = "10Gbps";
277 break;
278 case SPEED_1000:
279 s = "1000Mbps";
280 break;
281 case SPEED_100:
282 s = "100Mbps";
283 break;
284 }
285
286 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
287 fc[p->link_cfg.fc]);
288 }
289}
290
291void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
292{
293 struct net_device *dev = adapter->port[port_id];
294
295 /* Skip changes from disabled ports. */
296 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
297 if (link_stat)
298 netif_carrier_on(dev);
299 else
300 netif_carrier_off(dev);
301
302 link_report(dev);
303 }
304}
305
306void t4_os_portmod_changed(const struct adapter *adap, int port_id)
307{
308 static const char *mod_str[] = {
a0881cab 309 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 310 };
311
312 const struct net_device *dev = adap->port[port_id];
313 const struct port_info *pi = netdev_priv(dev);
314
315 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
316 netdev_info(dev, "port module unplugged\n");
a0881cab 317 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 318 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
319}
320
321/*
322 * Configure the exact and hash address filters to handle a port's multicast
323 * and secondary unicast MAC addresses.
324 */
325static int set_addr_filters(const struct net_device *dev, bool sleep)
326{
327 u64 mhash = 0;
328 u64 uhash = 0;
329 bool free = true;
330 u16 filt_idx[7];
331 const u8 *addr[7];
332 int ret, naddr = 0;
b8ff05a9
DM		 333 const struct netdev_hw_addr *ha;
334 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 335 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 336 const struct port_info *pi = netdev_priv(dev);
060e0c75 337 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 338
339 /* first do the secondary unicast addresses */
340 netdev_for_each_uc_addr(ha, dev) {
341 addr[naddr++] = ha->addr;
342 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 343 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 344 naddr, addr, filt_idx, &uhash, sleep);
345 if (ret < 0)
346 return ret;
347
348 free = false;
349 naddr = 0;
350 }
351 }
352
353 /* next set up the multicast addresses */
4a35ecf8
DM		 354 netdev_for_each_mc_addr(ha, dev) {
355 addr[naddr++] = ha->addr;
356 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 357 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 358 naddr, addr, filt_idx, &mhash, sleep);
359 if (ret < 0)
360 return ret;
361
362 free = false;
363 naddr = 0;
364 }
365 }
366
060e0c75 367 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 368 uhash | mhash, sleep);
369}
370
371/*
372 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
373 * If @mtu is -1 it is left unchanged.
374 */
375static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
376{
377 int ret;
378 struct port_info *pi = netdev_priv(dev);
379
380 ret = set_addr_filters(dev, sleep_ok);
381 if (ret == 0)
060e0c75 382 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 383 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 384 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 385 sleep_ok);
386 return ret;
387}
388
389/**
390 * link_start - enable a port
391 * @dev: the port to enable
392 *
393 * Performs the MAC and PHY actions needed to enable a port.
394 */
395static int link_start(struct net_device *dev)
396{
397 int ret;
398 struct port_info *pi = netdev_priv(dev);
060e0c75 399 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 400
401 /*
402 * We do not set address filters and promiscuity here, the stack does
403 * that step explicitly.
404 */
060e0c75 405 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
19ecae2c 406 !!(dev->features & NETIF_F_HW_VLAN_RX), true);
b8ff05a9 407 if (ret == 0) {
060e0c75 408 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 409 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 410 true);
b8ff05a9
DM		 411 if (ret >= 0) {
412 pi->xact_addr_filt = ret;
413 ret = 0;
414 }
415 }
416 if (ret == 0)
060e0c75
DM		 417 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
418 &pi->link_cfg);
b8ff05a9 419 if (ret == 0)
060e0c75 420 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 421 return ret;
422}
423
424/*
425 * Response queue handler for the FW event queue.
426 */
427static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
428 const struct pkt_gl *gl)
429{
430 u8 opcode = ((const struct rss_header *)rsp)->opcode;
431
432 rsp++; /* skip RSS header */
433 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
434 const struct cpl_sge_egr_update *p = (void *)rsp;
435 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 436 struct sge_txq *txq;
b8ff05a9 437
e46dab4d 438 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 439 txq->restarts++;
e46dab4d 440 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 441 struct sge_eth_txq *eq;
442
443 eq = container_of(txq, struct sge_eth_txq, q);
444 netif_tx_wake_queue(eq->txq);
445 } else {
446 struct sge_ofld_txq *oq;
447
448 oq = container_of(txq, struct sge_ofld_txq, q);
449 tasklet_schedule(&oq->qresume_tsk);
450 }
451 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
452 const struct cpl_fw6_msg *p = (void *)rsp;
453
454 if (p->type == 0)
455 t4_handle_fw_rpl(q->adap, p->data);
456 } else if (opcode == CPL_L2T_WRITE_RPL) {
457 const struct cpl_l2t_write_rpl *p = (void *)rsp;
458
459 do_l2t_write_rpl(q->adap, p);
460 } else
461 dev_err(q->adap->pdev_dev,
462 "unexpected CPL %#x on FW event queue\n", opcode);
463 return 0;
464}
465
466/**
467 * uldrx_handler - response queue handler for ULD queues
468 * @q: the response queue that received the packet
469 * @rsp: the response queue descriptor holding the offload message
470 * @gl: the gather list of packet fragments
471 *
472 * Deliver an ingress offload packet to a ULD. All processing is done by
473 * the ULD, we just maintain statistics.
474 */
475static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
476 const struct pkt_gl *gl)
477{
478 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
479
480 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
481 rxq->stats.nomem++;
482 return -1;
483 }
484 if (gl == NULL)
485 rxq->stats.imm++;
486 else if (gl == CXGB4_MSG_AN)
487 rxq->stats.an++;
488 else
489 rxq->stats.pkts++;
490 return 0;
491}
492
493static void disable_msi(struct adapter *adapter)
494{
495 if (adapter->flags & USING_MSIX) {
496 pci_disable_msix(adapter->pdev);
497 adapter->flags &= ~USING_MSIX;
498 } else if (adapter->flags & USING_MSI) {
499 pci_disable_msi(adapter->pdev);
500 adapter->flags &= ~USING_MSI;
501 }
502}
503
504/*
505 * Interrupt handler for non-data events used with MSI-X.
506 */
507static irqreturn_t t4_nondata_intr(int irq, void *cookie)
508{
509 struct adapter *adap = cookie;
510
511 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
512 if (v & PFSW) {
513 adap->swintr = 1;
514 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
515 }
516 t4_slow_intr_handler(adap);
517 return IRQ_HANDLED;
518}
519
520/*
521 * Name the MSI-X interrupts.
522 */
523static void name_msix_vecs(struct adapter *adap)
524{
525 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
526
527 /* non-data interrupts */
528 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
529 adap->msix_info[0].desc[n] = 0;
530
531 /* FW events */
532 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
533 adap->msix_info[1].desc[n] = 0;
534
535 /* Ethernet queues */
536 for_each_port(adap, j) {
537 struct net_device *d = adap->port[j];
538 const struct port_info *pi = netdev_priv(d);
539
540 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
541 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
542 d->name, i);
543 adap->msix_info[msi_idx].desc[n] = 0;
544 }
545 }
546
547 /* offload queues */
548 for_each_ofldrxq(&adap->sge, i) {
549 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
550 adap->name, i);
551 adap->msix_info[msi_idx++].desc[n] = 0;
552 }
553 for_each_rdmarxq(&adap->sge, i) {
554 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
555 adap->name, i);
556 adap->msix_info[msi_idx++].desc[n] = 0;
557 }
558}
559
560static int request_msix_queue_irqs(struct adapter *adap)
561{
562 struct sge *s = &adap->sge;
563 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
564
565 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
566 adap->msix_info[1].desc, &s->fw_evtq);
567 if (err)
568 return err;
569
570 for_each_ethrxq(s, ethqidx) {
571 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
572 adap->msix_info[msi].desc,
573 &s->ethrxq[ethqidx].rspq);
574 if (err)
575 goto unwind;
576 msi++;
577 }
578 for_each_ofldrxq(s, ofldqidx) {
579 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
580 adap->msix_info[msi].desc,
581 &s->ofldrxq[ofldqidx].rspq);
582 if (err)
583 goto unwind;
584 msi++;
585 }
586 for_each_rdmarxq(s, rdmaqidx) {
587 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
588 adap->msix_info[msi].desc,
589 &s->rdmarxq[rdmaqidx].rspq);
590 if (err)
591 goto unwind;
592 msi++;
593 }
594 return 0;
595
596unwind:
597 while (--rdmaqidx >= 0)
598 free_irq(adap->msix_info[--msi].vec,
599 &s->rdmarxq[rdmaqidx].rspq);
600 while (--ofldqidx >= 0)
601 free_irq(adap->msix_info[--msi].vec,
602 &s->ofldrxq[ofldqidx].rspq);
603 while (--ethqidx >= 0)
604 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
605 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
606 return err;
607}
608
609static void free_msix_queue_irqs(struct adapter *adap)
610{
611 int i, msi = 2;
612 struct sge *s = &adap->sge;
613
614 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
615 for_each_ethrxq(s, i)
616 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
617 for_each_ofldrxq(s, i)
618 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
619 for_each_rdmarxq(s, i)
620 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
621}
622
671b0060
DM		 623/**
624 * write_rss - write the RSS table for a given port
625 * @pi: the port
626 * @queues: array of queue indices for RSS
627 *
628 * Sets up the portion of the HW RSS table for the port's VI to distribute
629 * packets to the Rx queues in @queues.
630 */
631static int write_rss(const struct port_info *pi, const u16 *queues)
632{
633 u16 *rss;
634 int i, err;
635 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
636
637 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
638 if (!rss)
639 return -ENOMEM;
640
641 /* map the queue indices to queue ids */
642 for (i = 0; i < pi->rss_size; i++, queues++)
643 rss[i] = q[*queues].rspq.abs_id;
644
060e0c75
DM		 645 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
646 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 647 kfree(rss);
648 return err;
649}
650
b8ff05a9
DM		 651/**
652 * setup_rss - configure RSS
653 * @adap: the adapter
654 *
671b0060 655 * Sets up RSS for each port.
b8ff05a9
DM		 656 */
657static int setup_rss(struct adapter *adap)
658{
671b0060 659 int i, err;
b8ff05a9
DM		 660
661 for_each_port(adap, i) {
662 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 663
671b0060 664 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 665 if (err)
666 return err;
667 }
668 return 0;
669}
670
e46dab4d
DM		 671/*
672 * Return the channel of the ingress queue with the given qid.
673 */
674static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
675{
676 qid -= p->ingr_start;
677 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
678}
679
b8ff05a9
DM		 680/*
681 * Wait until all NAPI handlers are descheduled.
682 */
683static void quiesce_rx(struct adapter *adap)
684{
685 int i;
686
687 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
688 struct sge_rspq *q = adap->sge.ingr_map[i];
689
690 if (q && q->handler)
691 napi_disable(&q->napi);
692 }
693}
694
695/*
696 * Enable NAPI scheduling and interrupt generation for all Rx queues.
697 */
698static void enable_rx(struct adapter *adap)
699{
700 int i;
701
702 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
703 struct sge_rspq *q = adap->sge.ingr_map[i];
704
705 if (!q)
706 continue;
707 if (q->handler)
708 napi_enable(&q->napi);
709 /* 0-increment GTS to start the timer and enable interrupts */
710 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
711 SEINTARM(q->intr_params) |
712 INGRESSQID(q->cntxt_id));
713 }
714}
715
716/**
717 * setup_sge_queues - configure SGE Tx/Rx/response queues
718 * @adap: the adapter
719 *
720 * Determines how many sets of SGE queues to use and initializes them.
721 * We support multiple queue sets per port if we have MSI-X, otherwise
722 * just one queue set per port.
723 */
724static int setup_sge_queues(struct adapter *adap)
725{
726 int err, msi_idx, i, j;
727 struct sge *s = &adap->sge;
728
729 bitmap_zero(s->starving_fl, MAX_EGRQ);
730 bitmap_zero(s->txq_maperr, MAX_EGRQ);
731
732 if (adap->flags & USING_MSIX)
733 msi_idx = 1; /* vector 0 is for non-queue interrupts */
734 else {
735 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
736 NULL, NULL);
737 if (err)
738 return err;
739 msi_idx = -((int)s->intrq.abs_id + 1);
740 }
741
742 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
743 msi_idx, NULL, fwevtq_handler);
744 if (err) {
745freeout: t4_free_sge_resources(adap);
746 return err;
747 }
748
749 for_each_port(adap, i) {
750 struct net_device *dev = adap->port[i];
751 struct port_info *pi = netdev_priv(dev);
752 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
753 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
754
755 for (j = 0; j < pi->nqsets; j++, q++) {
756 if (msi_idx > 0)
757 msi_idx++;
758 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
759 msi_idx, &q->fl,
760 t4_ethrx_handler);
761 if (err)
762 goto freeout;
763 q->rspq.idx = j;
764 memset(&q->stats, 0, sizeof(q->stats));
765 }
766 for (j = 0; j < pi->nqsets; j++, t++) {
767 err = t4_sge_alloc_eth_txq(adap, t, dev,
768 netdev_get_tx_queue(dev, j),
769 s->fw_evtq.cntxt_id);
770 if (err)
771 goto freeout;
772 }
773 }
774
775 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
776 for_each_ofldrxq(s, i) {
777 struct sge_ofld_rxq *q = &s->ofldrxq[i];
778 struct net_device *dev = adap->port[i / j];
779
780 if (msi_idx > 0)
781 msi_idx++;
782 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
783 &q->fl, uldrx_handler);
784 if (err)
785 goto freeout;
786 memset(&q->stats, 0, sizeof(q->stats));
787 s->ofld_rxq[i] = q->rspq.abs_id;
788 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
789 s->fw_evtq.cntxt_id);
790 if (err)
791 goto freeout;
792 }
793
794 for_each_rdmarxq(s, i) {
795 struct sge_ofld_rxq *q = &s->rdmarxq[i];
796
797 if (msi_idx > 0)
798 msi_idx++;
799 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
800 msi_idx, &q->fl, uldrx_handler);
801 if (err)
802 goto freeout;
803 memset(&q->stats, 0, sizeof(q->stats));
804 s->rdma_rxq[i] = q->rspq.abs_id;
805 }
806
807 for_each_port(adap, i) {
808 /*
809 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
810 * have RDMA queues, and that's the right value.
811 */
812 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
813 s->fw_evtq.cntxt_id,
814 s->rdmarxq[i].rspq.cntxt_id);
815 if (err)
816 goto freeout;
817 }
818
819 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
820 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
821 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
822 return 0;
823}
824
825/*
826 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
827 * started but failed, and a negative errno if flash load couldn't start.
828 */
829static int upgrade_fw(struct adapter *adap)
830{
831 int ret;
832 u32 vers;
833 const struct fw_hdr *hdr;
834 const struct firmware *fw;
835 struct device *dev = adap->pdev_dev;
836
837 ret = request_firmware(&fw, FW_FNAME, dev);
838 if (ret < 0) {
839 dev_err(dev, "unable to load firmware image " FW_FNAME
840 ", error %d\n", ret);
841 return ret;
842 }
843
844 hdr = (const struct fw_hdr *)fw->data;
845 vers = ntohl(hdr->fw_ver);
846 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
847 ret = -EINVAL; /* wrong major version, won't do */
848 goto out;
849 }
850
851 /*
852 * If the flash FW is unusable or we found something newer, load it.
853 */
854 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
855 vers > adap->params.fw_vers) {
856 ret = -t4_load_fw(adap, fw->data, fw->size);
857 if (!ret)
858 dev_info(dev, "firmware upgraded to version %pI4 from "
859 FW_FNAME "\n", &hdr->fw_ver);
860 }
861out: release_firmware(fw);
862 return ret;
863}
864
865/*
866 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
867 * The allocated memory is cleared.
868 */
869void *t4_alloc_mem(size_t size)
870{
871 void *p = kmalloc(size, GFP_KERNEL);
872
873 if (!p)
874 p = vmalloc(size);
875 if (p)
876 memset(p, 0, size);
877 return p;
878}
879
880/*
881 * Free memory allocated through alloc_mem().
882 */
31b9c19b 883static void t4_free_mem(void *addr)
b8ff05a9
DM		 884{
885 if (is_vmalloc_addr(addr))
886 vfree(addr);
887 else
888 kfree(addr);
889}
890
891static inline int is_offload(const struct adapter *adap)
892{
893 return adap->params.offload;
894}
895
896/*
897 * Implementation of ethtool operations.
898 */
899
900static u32 get_msglevel(struct net_device *dev)
901{
902 return netdev2adap(dev)->msg_enable;
903}
904
905static void set_msglevel(struct net_device *dev, u32 val)
906{
907 netdev2adap(dev)->msg_enable = val;
908}
909
910static char stats_strings[][ETH_GSTRING_LEN] = {
911 "TxOctetsOK ",
912 "TxFramesOK ",
913 "TxBroadcastFrames ",
914 "TxMulticastFrames ",
915 "TxUnicastFrames ",
916 "TxErrorFrames ",
917
918 "TxFrames64 ",
919 "TxFrames65To127 ",
920 "TxFrames128To255 ",
921 "TxFrames256To511 ",
922 "TxFrames512To1023 ",
923 "TxFrames1024To1518 ",
924 "TxFrames1519ToMax ",
925
926 "TxFramesDropped ",
927 "TxPauseFrames ",
928 "TxPPP0Frames ",
929 "TxPPP1Frames ",
930 "TxPPP2Frames ",
931 "TxPPP3Frames ",
932 "TxPPP4Frames ",
933 "TxPPP5Frames ",
934 "TxPPP6Frames ",
935 "TxPPP7Frames ",
936
937 "RxOctetsOK ",
938 "RxFramesOK ",
939 "RxBroadcastFrames ",
940 "RxMulticastFrames ",
941 "RxUnicastFrames ",
942
943 "RxFramesTooLong ",
944 "RxJabberErrors ",
945 "RxFCSErrors ",
946 "RxLengthErrors ",
947 "RxSymbolErrors ",
948 "RxRuntFrames ",
949
950 "RxFrames64 ",
951 "RxFrames65To127 ",
952 "RxFrames128To255 ",
953 "RxFrames256To511 ",
954 "RxFrames512To1023 ",
955 "RxFrames1024To1518 ",
956 "RxFrames1519ToMax ",
957
958 "RxPauseFrames ",
959 "RxPPP0Frames ",
960 "RxPPP1Frames ",
961 "RxPPP2Frames ",
962 "RxPPP3Frames ",
963 "RxPPP4Frames ",
964 "RxPPP5Frames ",
965 "RxPPP6Frames ",
966 "RxPPP7Frames ",
967
968 "RxBG0FramesDropped ",
969 "RxBG1FramesDropped ",
970 "RxBG2FramesDropped ",
971 "RxBG3FramesDropped ",
972 "RxBG0FramesTrunc ",
973 "RxBG1FramesTrunc ",
974 "RxBG2FramesTrunc ",
975 "RxBG3FramesTrunc ",
976
977 "TSO ",
978 "TxCsumOffload ",
979 "RxCsumGood ",
980 "VLANextractions ",
981 "VLANinsertions ",
4a6346d4
DM		 982 "GROpackets ",
983 "GROmerged ",
b8ff05a9
DM		 984};
985
986static int get_sset_count(struct net_device *dev, int sset)
987{
988 switch (sset) {
989 case ETH_SS_STATS:
990 return ARRAY_SIZE(stats_strings);
991 default:
992 return -EOPNOTSUPP;
993 }
994}
995
996#define T4_REGMAP_SIZE (160 * 1024)
997
998static int get_regs_len(struct net_device *dev)
999{
1000 return T4_REGMAP_SIZE;
1001}
1002
1003static int get_eeprom_len(struct net_device *dev)
1004{
1005 return EEPROMSIZE;
1006}
1007
1008static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1009{
1010 struct adapter *adapter = netdev2adap(dev);
1011
1012 strcpy(info->driver, KBUILD_MODNAME);
1013 strcpy(info->version, DRV_VERSION);
1014 strcpy(info->bus_info, pci_name(adapter->pdev));
1015
1016 if (!adapter->params.fw_vers)
1017 strcpy(info->fw_version, "N/A");
1018 else
1019 snprintf(info->fw_version, sizeof(info->fw_version),
1020 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1021 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1022 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1023 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1024 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1025 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1026 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1027 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1028 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1029}
1030
1031static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1032{
1033 if (stringset == ETH_SS_STATS)
1034 memcpy(data, stats_strings, sizeof(stats_strings));
1035}
1036
1037/*
1038 * port stats maintained per queue of the port. They should be in the same
1039 * order as in stats_strings above.
1040 */
1041struct queue_port_stats {
1042 u64 tso;
1043 u64 tx_csum;
1044 u64 rx_csum;
1045 u64 vlan_ex;
1046 u64 vlan_ins;
4a6346d4
DM		 1047 u64 gro_pkts;
1048 u64 gro_merged;
b8ff05a9
DM		 1049};
1050
1051static void collect_sge_port_stats(const struct adapter *adap,
1052 const struct port_info *p, struct queue_port_stats *s)
1053{
1054 int i;
1055 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1056 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1057
1058 memset(s, 0, sizeof(*s));
1059 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1060 s->tso += tx->tso;
1061 s->tx_csum += tx->tx_cso;
1062 s->rx_csum += rx->stats.rx_cso;
1063 s->vlan_ex += rx->stats.vlan_ex;
1064 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1065 s->gro_pkts += rx->stats.lro_pkts;
1066 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1067 }
1068}
1069
1070static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1071 u64 *data)
1072{
1073 struct port_info *pi = netdev_priv(dev);
1074 struct adapter *adapter = pi->adapter;
1075
1076 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1077
1078 data += sizeof(struct port_stats) / sizeof(u64);
1079 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1080}
1081
1082/*
1083 * Return a version number to identify the type of adapter. The scheme is:
1084 * - bits 0..9: chip version
1085 * - bits 10..15: chip revision
835bb606 1086 * - bits 16..23: register dump version
b8ff05a9
DM		 1087 */
1088static inline unsigned int mk_adap_vers(const struct adapter *ap)
1089{
835bb606 1090 return 4 | (ap->params.rev << 10) | (1 << 16);
b8ff05a9
DM		 1091}
1092
1093static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1094 unsigned int end)
1095{
1096 u32 *p = buf + start;
1097
1098 for ( ; start <= end; start += sizeof(u32))
1099 *p++ = t4_read_reg(ap, start);
1100}
1101
1102static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1103 void *buf)
1104{
1105 static const unsigned int reg_ranges[] = {
1106 0x1008, 0x1108,
1107 0x1180, 0x11b4,
1108 0x11fc, 0x123c,
1109 0x1300, 0x173c,
1110 0x1800, 0x18fc,
1111 0x3000, 0x30d8,
1112 0x30e0, 0x5924,
1113 0x5960, 0x59d4,
1114 0x5a00, 0x5af8,
1115 0x6000, 0x6098,
1116 0x6100, 0x6150,
1117 0x6200, 0x6208,
1118 0x6240, 0x6248,
1119 0x6280, 0x6338,
1120 0x6370, 0x638c,
1121 0x6400, 0x643c,
1122 0x6500, 0x6524,
1123 0x6a00, 0x6a38,
1124 0x6a60, 0x6a78,
1125 0x6b00, 0x6b84,
1126 0x6bf0, 0x6c84,
1127 0x6cf0, 0x6d84,
1128 0x6df0, 0x6e84,
1129 0x6ef0, 0x6f84,
1130 0x6ff0, 0x7084,
1131 0x70f0, 0x7184,
1132 0x71f0, 0x7284,
1133 0x72f0, 0x7384,
1134 0x73f0, 0x7450,
1135 0x7500, 0x7530,
1136 0x7600, 0x761c,
1137 0x7680, 0x76cc,
1138 0x7700, 0x7798,
1139 0x77c0, 0x77fc,
1140 0x7900, 0x79fc,
1141 0x7b00, 0x7c38,
1142 0x7d00, 0x7efc,
1143 0x8dc0, 0x8e1c,
1144 0x8e30, 0x8e78,
1145 0x8ea0, 0x8f6c,
1146 0x8fc0, 0x9074,
1147 0x90fc, 0x90fc,
1148 0x9400, 0x9458,
1149 0x9600, 0x96bc,
1150 0x9800, 0x9808,
1151 0x9820, 0x983c,
1152 0x9850, 0x9864,
1153 0x9c00, 0x9c6c,
1154 0x9c80, 0x9cec,
1155 0x9d00, 0x9d6c,
1156 0x9d80, 0x9dec,
1157 0x9e00, 0x9e6c,
1158 0x9e80, 0x9eec,
1159 0x9f00, 0x9f6c,
1160 0x9f80, 0x9fec,
1161 0xd004, 0xd03c,
1162 0xdfc0, 0xdfe0,
1163 0xe000, 0xea7c,
1164 0xf000, 0x11190,
835bb606
DM		 1165 0x19040, 0x1906c,
1166 0x19078, 0x19080,
1167 0x1908c, 0x19124,
b8ff05a9
DM		 1168 0x19150, 0x191b0,
1169 0x191d0, 0x191e8,
1170 0x19238, 0x1924c,
1171 0x193f8, 0x19474,
1172 0x19490, 0x194f8,
1173 0x19800, 0x19f30,
1174 0x1a000, 0x1a06c,
1175 0x1a0b0, 0x1a120,
1176 0x1a128, 0x1a138,
1177 0x1a190, 0x1a1c4,
1178 0x1a1fc, 0x1a1fc,
1179 0x1e040, 0x1e04c,
835bb606 1180 0x1e284, 0x1e28c,
b8ff05a9
DM		 1181 0x1e2c0, 0x1e2c0,
1182 0x1e2e0, 0x1e2e0,
1183 0x1e300, 0x1e384,
1184 0x1e3c0, 0x1e3c8,
1185 0x1e440, 0x1e44c,
835bb606 1186 0x1e684, 0x1e68c,
b8ff05a9
DM		 1187 0x1e6c0, 0x1e6c0,
1188 0x1e6e0, 0x1e6e0,
1189 0x1e700, 0x1e784,
1190 0x1e7c0, 0x1e7c8,
1191 0x1e840, 0x1e84c,
835bb606 1192 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1193 0x1eac0, 0x1eac0,
1194 0x1eae0, 0x1eae0,
1195 0x1eb00, 0x1eb84,
1196 0x1ebc0, 0x1ebc8,
1197 0x1ec40, 0x1ec4c,
835bb606 1198 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1199 0x1eec0, 0x1eec0,
1200 0x1eee0, 0x1eee0,
1201 0x1ef00, 0x1ef84,
1202 0x1efc0, 0x1efc8,
1203 0x1f040, 0x1f04c,
835bb606 1204 0x1f284, 0x1f28c,
b8ff05a9
DM		 1205 0x1f2c0, 0x1f2c0,
1206 0x1f2e0, 0x1f2e0,
1207 0x1f300, 0x1f384,
1208 0x1f3c0, 0x1f3c8,
1209 0x1f440, 0x1f44c,
835bb606 1210 0x1f684, 0x1f68c,
b8ff05a9
DM		 1211 0x1f6c0, 0x1f6c0,
1212 0x1f6e0, 0x1f6e0,
1213 0x1f700, 0x1f784,
1214 0x1f7c0, 0x1f7c8,
1215 0x1f840, 0x1f84c,
835bb606 1216 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1217 0x1fac0, 0x1fac0,
1218 0x1fae0, 0x1fae0,
1219 0x1fb00, 0x1fb84,
1220 0x1fbc0, 0x1fbc8,
1221 0x1fc40, 0x1fc4c,
835bb606 1222 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1223 0x1fec0, 0x1fec0,
1224 0x1fee0, 0x1fee0,
1225 0x1ff00, 0x1ff84,
1226 0x1ffc0, 0x1ffc8,
1227 0x20000, 0x2002c,
1228 0x20100, 0x2013c,
1229 0x20190, 0x201c8,
1230 0x20200, 0x20318,
1231 0x20400, 0x20528,
1232 0x20540, 0x20614,
1233 0x21000, 0x21040,
1234 0x2104c, 0x21060,
1235 0x210c0, 0x210ec,
1236 0x21200, 0x21268,
1237 0x21270, 0x21284,
1238 0x212fc, 0x21388,
1239 0x21400, 0x21404,
1240 0x21500, 0x21518,
1241 0x2152c, 0x2153c,
1242 0x21550, 0x21554,
1243 0x21600, 0x21600,
1244 0x21608, 0x21628,
1245 0x21630, 0x2163c,
1246 0x21700, 0x2171c,
1247 0x21780, 0x2178c,
1248 0x21800, 0x21c38,
1249 0x21c80, 0x21d7c,
1250 0x21e00, 0x21e04,
1251 0x22000, 0x2202c,
1252 0x22100, 0x2213c,
1253 0x22190, 0x221c8,
1254 0x22200, 0x22318,
1255 0x22400, 0x22528,
1256 0x22540, 0x22614,
1257 0x23000, 0x23040,
1258 0x2304c, 0x23060,
1259 0x230c0, 0x230ec,
1260 0x23200, 0x23268,
1261 0x23270, 0x23284,
1262 0x232fc, 0x23388,
1263 0x23400, 0x23404,
1264 0x23500, 0x23518,
1265 0x2352c, 0x2353c,
1266 0x23550, 0x23554,
1267 0x23600, 0x23600,
1268 0x23608, 0x23628,
1269 0x23630, 0x2363c,
1270 0x23700, 0x2371c,
1271 0x23780, 0x2378c,
1272 0x23800, 0x23c38,
1273 0x23c80, 0x23d7c,
1274 0x23e00, 0x23e04,
1275 0x24000, 0x2402c,
1276 0x24100, 0x2413c,
1277 0x24190, 0x241c8,
1278 0x24200, 0x24318,
1279 0x24400, 0x24528,
1280 0x24540, 0x24614,
1281 0x25000, 0x25040,
1282 0x2504c, 0x25060,
1283 0x250c0, 0x250ec,
1284 0x25200, 0x25268,
1285 0x25270, 0x25284,
1286 0x252fc, 0x25388,
1287 0x25400, 0x25404,
1288 0x25500, 0x25518,
1289 0x2552c, 0x2553c,
1290 0x25550, 0x25554,
1291 0x25600, 0x25600,
1292 0x25608, 0x25628,
1293 0x25630, 0x2563c,
1294 0x25700, 0x2571c,
1295 0x25780, 0x2578c,
1296 0x25800, 0x25c38,
1297 0x25c80, 0x25d7c,
1298 0x25e00, 0x25e04,
1299 0x26000, 0x2602c,
1300 0x26100, 0x2613c,
1301 0x26190, 0x261c8,
1302 0x26200, 0x26318,
1303 0x26400, 0x26528,
1304 0x26540, 0x26614,
1305 0x27000, 0x27040,
1306 0x2704c, 0x27060,
1307 0x270c0, 0x270ec,
1308 0x27200, 0x27268,
1309 0x27270, 0x27284,
1310 0x272fc, 0x27388,
1311 0x27400, 0x27404,
1312 0x27500, 0x27518,
1313 0x2752c, 0x2753c,
1314 0x27550, 0x27554,
1315 0x27600, 0x27600,
1316 0x27608, 0x27628,
1317 0x27630, 0x2763c,
1318 0x27700, 0x2771c,
1319 0x27780, 0x2778c,
1320 0x27800, 0x27c38,
1321 0x27c80, 0x27d7c,
1322 0x27e00, 0x27e04
1323 };
1324
1325 int i;
1326 struct adapter *ap = netdev2adap(dev);
1327
1328 regs->version = mk_adap_vers(ap);
1329
1330 memset(buf, 0, T4_REGMAP_SIZE);
1331 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1332 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1333}
1334
1335static int restart_autoneg(struct net_device *dev)
1336{
1337 struct port_info *p = netdev_priv(dev);
1338
1339 if (!netif_running(dev))
1340 return -EAGAIN;
1341 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1342 return -EINVAL;
060e0c75 1343 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 1344 return 0;
1345}
1346
1347static int identify_port(struct net_device *dev, u32 data)
1348{
060e0c75
DM		 1349 struct adapter *adap = netdev2adap(dev);
1350
b8ff05a9
DM		 1351 if (data == 0)
1352 data = 2; /* default to 2 seconds */
1353
060e0c75 1354 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid,
b8ff05a9
DM		 1355 data * 5);
1356}
1357
1358static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1359{
1360 unsigned int v = 0;
1361
a0881cab
DM		 1362 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1363 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 1364 v |= SUPPORTED_TP;
1365 if (caps & FW_PORT_CAP_SPEED_100M)
1366 v |= SUPPORTED_100baseT_Full;
1367 if (caps & FW_PORT_CAP_SPEED_1G)
1368 v |= SUPPORTED_1000baseT_Full;
1369 if (caps & FW_PORT_CAP_SPEED_10G)
1370 v |= SUPPORTED_10000baseT_Full;
1371 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1372 v |= SUPPORTED_Backplane;
1373 if (caps & FW_PORT_CAP_SPEED_1G)
1374 v |= SUPPORTED_1000baseKX_Full;
1375 if (caps & FW_PORT_CAP_SPEED_10G)
1376 v |= SUPPORTED_10000baseKX4_Full;
1377 } else if (type == FW_PORT_TYPE_KR)
1378 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab
DM		 1379 else if (type == FW_PORT_TYPE_BP_AP)
1380 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1381 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1382 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9
DM		 1383 v |= SUPPORTED_FIBRE;
1384
1385 if (caps & FW_PORT_CAP_ANEG)
1386 v |= SUPPORTED_Autoneg;
1387 return v;
1388}
1389
1390static unsigned int to_fw_linkcaps(unsigned int caps)
1391{
1392 unsigned int v = 0;
1393
1394 if (caps & ADVERTISED_100baseT_Full)
1395 v |= FW_PORT_CAP_SPEED_100M;
1396 if (caps & ADVERTISED_1000baseT_Full)
1397 v |= FW_PORT_CAP_SPEED_1G;
1398 if (caps & ADVERTISED_10000baseT_Full)
1399 v |= FW_PORT_CAP_SPEED_10G;
1400 return v;
1401}
1402
1403static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1404{
1405 const struct port_info *p = netdev_priv(dev);
1406
1407 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 1408 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 1409 p->port_type == FW_PORT_TYPE_BT_XAUI)
1410 cmd->port = PORT_TP;
a0881cab
DM		 1411 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1412 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 1413 cmd->port = PORT_FIBRE;
a0881cab
DM		 1414 else if (p->port_type == FW_PORT_TYPE_SFP) {
1415 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1416 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1417 cmd->port = PORT_DA;
1418 else
1419 cmd->port = PORT_FIBRE;
1420 } else
b8ff05a9
DM		 1421 cmd->port = PORT_OTHER;
1422
1423 if (p->mdio_addr >= 0) {
1424 cmd->phy_address = p->mdio_addr;
1425 cmd->transceiver = XCVR_EXTERNAL;
1426 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1427 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1428 } else {
1429 cmd->phy_address = 0; /* not really, but no better option */
1430 cmd->transceiver = XCVR_INTERNAL;
1431 cmd->mdio_support = 0;
1432 }
1433
1434 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1435 cmd->advertising = from_fw_linkcaps(p->port_type,
1436 p->link_cfg.advertising);
1437 cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1438 cmd->duplex = DUPLEX_FULL;
1439 cmd->autoneg = p->link_cfg.autoneg;
1440 cmd->maxtxpkt = 0;
1441 cmd->maxrxpkt = 0;
1442 return 0;
1443}
1444
1445static unsigned int speed_to_caps(int speed)
1446{
1447 if (speed == SPEED_100)
1448 return FW_PORT_CAP_SPEED_100M;
1449 if (speed == SPEED_1000)
1450 return FW_PORT_CAP_SPEED_1G;
1451 if (speed == SPEED_10000)
1452 return FW_PORT_CAP_SPEED_10G;
1453 return 0;
1454}
1455
1456static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1457{
1458 unsigned int cap;
1459 struct port_info *p = netdev_priv(dev);
1460 struct link_config *lc = &p->link_cfg;
1461
1462 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1463 return -EINVAL;
1464
1465 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1466 /*
1467 * PHY offers a single speed. See if that's what's
1468 * being requested.
1469 */
1470 if (cmd->autoneg == AUTONEG_DISABLE &&
1471 (lc->supported & speed_to_caps(cmd->speed)))
1472 return 0;
1473 return -EINVAL;
1474 }
1475
1476 if (cmd->autoneg == AUTONEG_DISABLE) {
1477 cap = speed_to_caps(cmd->speed);
1478
1479 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1480 cmd->speed == SPEED_10000)
1481 return -EINVAL;
1482 lc->requested_speed = cap;
1483 lc->advertising = 0;
1484 } else {
1485 cap = to_fw_linkcaps(cmd->advertising);
1486 if (!(lc->supported & cap))
1487 return -EINVAL;
1488 lc->requested_speed = 0;
1489 lc->advertising = cap | FW_PORT_CAP_ANEG;
1490 }
1491 lc->autoneg = cmd->autoneg;
1492
1493 if (netif_running(dev))
060e0c75
DM		 1494 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1495 lc);
b8ff05a9
DM		 1496 return 0;
1497}
1498
1499static void get_pauseparam(struct net_device *dev,
1500 struct ethtool_pauseparam *epause)
1501{
1502 struct port_info *p = netdev_priv(dev);
1503
1504 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1505 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1506 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1507}
1508
1509static int set_pauseparam(struct net_device *dev,
1510 struct ethtool_pauseparam *epause)
1511{
1512 struct port_info *p = netdev_priv(dev);
1513 struct link_config *lc = &p->link_cfg;
1514
1515 if (epause->autoneg == AUTONEG_DISABLE)
1516 lc->requested_fc = 0;
1517 else if (lc->supported & FW_PORT_CAP_ANEG)
1518 lc->requested_fc = PAUSE_AUTONEG;
1519 else
1520 return -EINVAL;
1521
1522 if (epause->rx_pause)
1523 lc->requested_fc |= PAUSE_RX;
1524 if (epause->tx_pause)
1525 lc->requested_fc |= PAUSE_TX;
1526 if (netif_running(dev))
060e0c75
DM		 1527 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1528 lc);
b8ff05a9
DM		 1529 return 0;
1530}
1531
1532static u32 get_rx_csum(struct net_device *dev)
1533{
1534 struct port_info *p = netdev_priv(dev);
1535
1536 return p->rx_offload & RX_CSO;
1537}
1538
1539static int set_rx_csum(struct net_device *dev, u32 data)
1540{
1541 struct port_info *p = netdev_priv(dev);
1542
1543 if (data)
1544 p->rx_offload |= RX_CSO;
1545 else
1546 p->rx_offload &= ~RX_CSO;
1547 return 0;
1548}
1549
1550static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1551{
1552 const struct port_info *pi = netdev_priv(dev);
1553 const struct sge *s = &pi->adapter->sge;
1554
1555 e->rx_max_pending = MAX_RX_BUFFERS;
1556 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1557 e->rx_jumbo_max_pending = 0;
1558 e->tx_max_pending = MAX_TXQ_ENTRIES;
1559
1560 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1561 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1562 e->rx_jumbo_pending = 0;
1563 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1564}
1565
1566static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1567{
1568 int i;
1569 const struct port_info *pi = netdev_priv(dev);
1570 struct adapter *adapter = pi->adapter;
1571 struct sge *s = &adapter->sge;
1572
1573 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1574 e->tx_pending > MAX_TXQ_ENTRIES ||
1575 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1576 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1577 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1578 return -EINVAL;
1579
1580 if (adapter->flags & FULL_INIT_DONE)
1581 return -EBUSY;
1582
1583 for (i = 0; i < pi->nqsets; ++i) {
1584 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1585 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1586 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1587 }
1588 return 0;
1589}
1590
1591static int closest_timer(const struct sge *s, int time)
1592{
1593 int i, delta, match = 0, min_delta = INT_MAX;
1594
1595 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1596 delta = time - s->timer_val[i];
1597 if (delta < 0)
1598 delta = -delta;
1599 if (delta < min_delta) {
1600 min_delta = delta;
1601 match = i;
1602 }
1603 }
1604 return match;
1605}
1606
1607static int closest_thres(const struct sge *s, int thres)
1608{
1609 int i, delta, match = 0, min_delta = INT_MAX;
1610
1611 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1612 delta = thres - s->counter_val[i];
1613 if (delta < 0)
1614 delta = -delta;
1615 if (delta < min_delta) {
1616 min_delta = delta;
1617 match = i;
1618 }
1619 }
1620 return match;
1621}
1622
1623/*
1624 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1625 */
1626static unsigned int qtimer_val(const struct adapter *adap,
1627 const struct sge_rspq *q)
1628{
1629 unsigned int idx = q->intr_params >> 1;
1630
1631 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1632}
1633
1634/**
1635 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1636 * @adap: the adapter
1637 * @q: the Rx queue
1638 * @us: the hold-off time in us, or 0 to disable timer
1639 * @cnt: the hold-off packet count, or 0 to disable counter
1640 *
1641 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1642 * one of the two needs to be enabled for the queue to generate interrupts.
1643 */
1644static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1645 unsigned int us, unsigned int cnt)
1646{
1647 if ((us | cnt) == 0)
1648 cnt = 1;
1649
1650 if (cnt) {
1651 int err;
1652 u32 v, new_idx;
1653
1654 new_idx = closest_thres(&adap->sge, cnt);
1655 if (q->desc && q->pktcnt_idx != new_idx) {
1656 /* the queue has already been created, update it */
1657 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1658 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1659 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 1660 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1661 &new_idx);
b8ff05a9
DM		 1662 if (err)
1663 return err;
1664 }
1665 q->pktcnt_idx = new_idx;
1666 }
1667
1668 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1669 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1670 return 0;
1671}
1672
1673static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1674{
1675 const struct port_info *pi = netdev_priv(dev);
1676 struct adapter *adap = pi->adapter;
1677
1678 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1679 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1680}
1681
1682static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1683{
1684 const struct port_info *pi = netdev_priv(dev);
1685 const struct adapter *adap = pi->adapter;
1686 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1687
1688 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1689 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1690 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1691 return 0;
1692}
1693
1478b3ee
DM		 1694/**
1695 * eeprom_ptov - translate a physical EEPROM address to virtual
1696 * @phys_addr: the physical EEPROM address
1697 * @fn: the PCI function number
1698 * @sz: size of function-specific area
1699 *
1700 * Translate a physical EEPROM address to virtual. The first 1K is
1701 * accessed through virtual addresses starting at 31K, the rest is
1702 * accessed through virtual addresses starting at 0.
1703 *
1704 * The mapping is as follows:
1705 * [0..1K) -> [31K..32K)
1706 * [1K..1K+A) -> [31K-A..31K)
1707 * [1K+A..ES) -> [0..ES-A-1K)
1708 *
1709 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 1710 */
1478b3ee 1711static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 1712{
1478b3ee 1713 fn *= sz;
b8ff05a9
DM		 1714 if (phys_addr < 1024)
1715 return phys_addr + (31 << 10);
1478b3ee
DM		 1716 if (phys_addr < 1024 + fn)
1717 return 31744 - fn + phys_addr - 1024;
b8ff05a9 1718 if (phys_addr < EEPROMSIZE)
1478b3ee 1719 return phys_addr - 1024 - fn;
b8ff05a9
DM		 1720 return -EINVAL;
1721}
1722
1723/*
1724 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 1725 */
1726static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1727{
1478b3ee 1728 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1729
1730 if (vaddr >= 0)
1731 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1732 return vaddr < 0 ? vaddr : 0;
1733}
1734
1735static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1736{
1478b3ee 1737 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1738
1739 if (vaddr >= 0)
1740 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1741 return vaddr < 0 ? vaddr : 0;
1742}
1743
1744#define EEPROM_MAGIC 0x38E2F10C
1745
1746static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1747 u8 *data)
1748{
1749 int i, err = 0;
1750 struct adapter *adapter = netdev2adap(dev);
1751
1752 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1753 if (!buf)
1754 return -ENOMEM;
1755
1756 e->magic = EEPROM_MAGIC;
1757 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1758 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1759
1760 if (!err)
1761 memcpy(data, buf + e->offset, e->len);
1762 kfree(buf);
1763 return err;
1764}
1765
1766static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1767 u8 *data)
1768{
1769 u8 *buf;
1770 int err = 0;
1771 u32 aligned_offset, aligned_len, *p;
1772 struct adapter *adapter = netdev2adap(dev);
1773
1774 if (eeprom->magic != EEPROM_MAGIC)
1775 return -EINVAL;
1776
1777 aligned_offset = eeprom->offset & ~3;
1778 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1779
1478b3ee
DM		 1780 if (adapter->fn > 0) {
1781 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1782
1783 if (aligned_offset < start ||
1784 aligned_offset + aligned_len > start + EEPROMPFSIZE)
1785 return -EPERM;
1786 }
1787
b8ff05a9
DM		 1788 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1789 /*
1790 * RMW possibly needed for first or last words.
1791 */
1792 buf = kmalloc(aligned_len, GFP_KERNEL);
1793 if (!buf)
1794 return -ENOMEM;
1795 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1796 if (!err && aligned_len > 4)
1797 err = eeprom_rd_phys(adapter,
1798 aligned_offset + aligned_len - 4,
1799 (u32 *)&buf[aligned_len - 4]);
1800 if (err)
1801 goto out;
1802 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1803 } else
1804 buf = data;
1805
1806 err = t4_seeprom_wp(adapter, false);
1807 if (err)
1808 goto out;
1809
1810 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1811 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1812 aligned_offset += 4;
1813 }
1814
1815 if (!err)
1816 err = t4_seeprom_wp(adapter, true);
1817out:
1818 if (buf != data)
1819 kfree(buf);
1820 return err;
1821}
1822
1823static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1824{
1825 int ret;
1826 const struct firmware *fw;
1827 struct adapter *adap = netdev2adap(netdev);
1828
1829 ef->data[sizeof(ef->data) - 1] = '\0';
1830 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1831 if (ret < 0)
1832 return ret;
1833
1834 ret = t4_load_fw(adap, fw->data, fw->size);
1835 release_firmware(fw);
1836 if (!ret)
1837 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1838 return ret;
1839}
1840
1841#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1842#define BCAST_CRC 0xa0ccc1a6
1843
1844static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1845{
1846 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1847 wol->wolopts = netdev2adap(dev)->wol;
1848 memset(&wol->sopass, 0, sizeof(wol->sopass));
1849}
1850
1851static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1852{
1853 int err = 0;
1854 struct port_info *pi = netdev_priv(dev);
1855
1856 if (wol->wolopts & ~WOL_SUPPORTED)
1857 return -EINVAL;
1858 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1859 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1860 if (wol->wolopts & WAKE_BCAST) {
1861 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1862 ~0ULL, 0, false);
1863 if (!err)
1864 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1865 ~6ULL, ~0ULL, BCAST_CRC, true);
1866 } else
1867 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1868 return err;
1869}
1870
35d35682
DM		 1871#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1872
b8ff05a9
DM		 1873static int set_tso(struct net_device *dev, u32 value)
1874{
1875 if (value)
35d35682 1876 dev->features |= TSO_FLAGS;
b8ff05a9 1877 else
35d35682 1878 dev->features &= ~TSO_FLAGS;
b8ff05a9
DM		 1879 return 0;
1880}
1881
87b6cf51
DM		 1882static int set_flags(struct net_device *dev, u32 flags)
1883{
19ecae2c
DM		 1884 int err;
1885 unsigned long old_feat = dev->features;
1886
1887 err = ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH |
1888 ETH_FLAG_RXVLAN | ETH_FLAG_TXVLAN);
1889 if (err)
1890 return err;
1891
1892 if ((old_feat ^ dev->features) & NETIF_F_HW_VLAN_RX) {
1893 const struct port_info *pi = netdev_priv(dev);
1894
1895 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1896 -1, -1, -1, !!(flags & ETH_FLAG_RXVLAN),
1897 true);
1898 if (err)
1899 dev->features = old_feat;
1900 }
1901 return err;
87b6cf51
DM		 1902}
1903
671b0060
DM		 1904static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1905{
1906 const struct port_info *pi = netdev_priv(dev);
1907 unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1908
1909 p->size = pi->rss_size;
1910 while (n--)
1911 p->ring_index[n] = pi->rss[n];
1912 return 0;
1913}
1914
1915static int set_rss_table(struct net_device *dev,
1916 const struct ethtool_rxfh_indir *p)
1917{
1918 unsigned int i;
1919 struct port_info *pi = netdev_priv(dev);
1920
1921 if (p->size != pi->rss_size)
1922 return -EINVAL;
1923 for (i = 0; i < p->size; i++)
1924 if (p->ring_index[i] >= pi->nqsets)
1925 return -EINVAL;
1926 for (i = 0; i < p->size; i++)
1927 pi->rss[i] = p->ring_index[i];
1928 if (pi->adapter->flags & FULL_INIT_DONE)
1929 return write_rss(pi, pi->rss);
1930 return 0;
1931}
1932
1933static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1934 void *rules)
1935{
f796564a
DM		 1936 const struct port_info *pi = netdev_priv(dev);
1937
671b0060 1938 switch (info->cmd) {
f796564a
DM		 1939 case ETHTOOL_GRXFH: {
1940 unsigned int v = pi->rss_mode;
1941
1942 info->data = 0;
1943 switch (info->flow_type) {
1944 case TCP_V4_FLOW:
1945 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1946 info->data = RXH_IP_SRC | RXH_IP_DST |
1947 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1948 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1949 info->data = RXH_IP_SRC | RXH_IP_DST;
1950 break;
1951 case UDP_V4_FLOW:
1952 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1953 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1954 info->data = RXH_IP_SRC | RXH_IP_DST |
1955 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1956 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1957 info->data = RXH_IP_SRC | RXH_IP_DST;
1958 break;
1959 case SCTP_V4_FLOW:
1960 case AH_ESP_V4_FLOW:
1961 case IPV4_FLOW:
1962 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1963 info->data = RXH_IP_SRC | RXH_IP_DST;
1964 break;
1965 case TCP_V6_FLOW:
1966 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1967 info->data = RXH_IP_SRC | RXH_IP_DST |
1968 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1969 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1970 info->data = RXH_IP_SRC | RXH_IP_DST;
1971 break;
1972 case UDP_V6_FLOW:
1973 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1974 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1975 info->data = RXH_IP_SRC | RXH_IP_DST |
1976 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1977 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1978 info->data = RXH_IP_SRC | RXH_IP_DST;
1979 break;
1980 case SCTP_V6_FLOW:
1981 case AH_ESP_V6_FLOW:
1982 case IPV6_FLOW:
1983 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1984 info->data = RXH_IP_SRC | RXH_IP_DST;
1985 break;
1986 }
1987 return 0;
1988 }
671b0060 1989 case ETHTOOL_GRXRINGS:
f796564a 1990 info->data = pi->nqsets;
671b0060
DM		 1991 return 0;
1992 }
1993 return -EOPNOTSUPP;
1994}
1995
b8ff05a9
DM		 1996static struct ethtool_ops cxgb_ethtool_ops = {
1997 .get_settings = get_settings,
1998 .set_settings = set_settings,
1999 .get_drvinfo = get_drvinfo,
2000 .get_msglevel = get_msglevel,
2001 .set_msglevel = set_msglevel,
2002 .get_ringparam = get_sge_param,
2003 .set_ringparam = set_sge_param,
2004 .get_coalesce = get_coalesce,
2005 .set_coalesce = set_coalesce,
2006 .get_eeprom_len = get_eeprom_len,
2007 .get_eeprom = get_eeprom,
2008 .set_eeprom = set_eeprom,
2009 .get_pauseparam = get_pauseparam,
2010 .set_pauseparam = set_pauseparam,
2011 .get_rx_csum = get_rx_csum,
2012 .set_rx_csum = set_rx_csum,
2013 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
2014 .set_sg = ethtool_op_set_sg,
2015 .get_link = ethtool_op_get_link,
2016 .get_strings = get_strings,
2017 .phys_id = identify_port,
2018 .nway_reset = restart_autoneg,
2019 .get_sset_count = get_sset_count,
2020 .get_ethtool_stats = get_stats,
2021 .get_regs_len = get_regs_len,
2022 .get_regs = get_regs,
2023 .get_wol = get_wol,
2024 .set_wol = set_wol,
2025 .set_tso = set_tso,
87b6cf51 2026 .set_flags = set_flags,
671b0060
DM		 2027 .get_rxnfc = get_rxnfc,
2028 .get_rxfh_indir = get_rss_table,
2029 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 2030 .flash_device = set_flash,
2031};
2032
2033/*
2034 * debugfs support
2035 */
2036
2037static int mem_open(struct inode *inode, struct file *file)
2038{
2039 file->private_data = inode->i_private;
2040 return 0;
2041}
2042
2043static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2044 loff_t *ppos)
2045{
2046 loff_t pos = *ppos;
2047 loff_t avail = file->f_path.dentry->d_inode->i_size;
2048 unsigned int mem = (uintptr_t)file->private_data & 3;
2049 struct adapter *adap = file->private_data - mem;
2050
2051 if (pos < 0)
2052 return -EINVAL;
2053 if (pos >= avail)
2054 return 0;
2055 if (count > avail - pos)
2056 count = avail - pos;
2057
2058 while (count) {
2059 size_t len;
2060 int ret, ofst;
2061 __be32 data[16];
2062
2063 if (mem == MEM_MC)
2064 ret = t4_mc_read(adap, pos, data, NULL);
2065 else
2066 ret = t4_edc_read(adap, mem, pos, data, NULL);
2067 if (ret)
2068 return ret;
2069
2070 ofst = pos % sizeof(data);
2071 len = min(count, sizeof(data) - ofst);
2072 if (copy_to_user(buf, (u8 *)data + ofst, len))
2073 return -EFAULT;
2074
2075 buf += len;
2076 pos += len;
2077 count -= len;
2078 }
2079 count = pos - *ppos;
2080 *ppos = pos;
2081 return count;
2082}
2083
2084static const struct file_operations mem_debugfs_fops = {
2085 .owner = THIS_MODULE,
2086 .open = mem_open,
2087 .read = mem_read,
6038f373 2088 .llseek = default_llseek,
b8ff05a9
DM		 2089};
2090
2091static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2092 unsigned int idx, unsigned int size_mb)
2093{
2094 struct dentry *de;
2095
2096 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2097 (void *)adap + idx, &mem_debugfs_fops);
2098 if (de && de->d_inode)
2099 de->d_inode->i_size = size_mb << 20;
2100}
2101
2102static int __devinit setup_debugfs(struct adapter *adap)
2103{
2104 int i;
2105
2106 if (IS_ERR_OR_NULL(adap->debugfs_root))
2107 return -1;
2108
2109 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2110 if (i & EDRAM0_ENABLE)
2111 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2112 if (i & EDRAM1_ENABLE)
2113 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2114 if (i & EXT_MEM_ENABLE)
2115 add_debugfs_mem(adap, "mc", MEM_MC,
2116 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2117 if (adap->l2t)
2118 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2119 &t4_l2t_fops);
2120 return 0;
2121}
2122
2123/*
2124 * upper-layer driver support
2125 */
2126
2127/*
2128 * Allocate an active-open TID and set it to the supplied value.
2129 */
2130int cxgb4_alloc_atid(struct tid_info *t, void *data)
2131{
2132 int atid = -1;
2133
2134 spin_lock_bh(&t->atid_lock);
2135 if (t->afree) {
2136 union aopen_entry *p = t->afree;
2137
2138 atid = p - t->atid_tab;
2139 t->afree = p->next;
2140 p->data = data;
2141 t->atids_in_use++;
2142 }
2143 spin_unlock_bh(&t->atid_lock);
2144 return atid;
2145}
2146EXPORT_SYMBOL(cxgb4_alloc_atid);
2147
2148/*
2149 * Release an active-open TID.
2150 */
2151void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2152{
2153 union aopen_entry *p = &t->atid_tab[atid];
2154
2155 spin_lock_bh(&t->atid_lock);
2156 p->next = t->afree;
2157 t->afree = p;
2158 t->atids_in_use--;
2159 spin_unlock_bh(&t->atid_lock);
2160}
2161EXPORT_SYMBOL(cxgb4_free_atid);
2162
2163/*
2164 * Allocate a server TID and set it to the supplied value.
2165 */
2166int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2167{
2168 int stid;
2169
2170 spin_lock_bh(&t->stid_lock);
2171 if (family == PF_INET) {
2172 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2173 if (stid < t->nstids)
2174 __set_bit(stid, t->stid_bmap);
2175 else
2176 stid = -1;
2177 } else {
2178 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2179 if (stid < 0)
2180 stid = -1;
2181 }
2182 if (stid >= 0) {
2183 t->stid_tab[stid].data = data;
2184 stid += t->stid_base;
2185 t->stids_in_use++;
2186 }
2187 spin_unlock_bh(&t->stid_lock);
2188 return stid;
2189}
2190EXPORT_SYMBOL(cxgb4_alloc_stid);
2191
2192/*
2193 * Release a server TID.
2194 */
2195void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2196{
2197 stid -= t->stid_base;
2198 spin_lock_bh(&t->stid_lock);
2199 if (family == PF_INET)
2200 __clear_bit(stid, t->stid_bmap);
2201 else
2202 bitmap_release_region(t->stid_bmap, stid, 2);
2203 t->stid_tab[stid].data = NULL;
2204 t->stids_in_use--;
2205 spin_unlock_bh(&t->stid_lock);
2206}
2207EXPORT_SYMBOL(cxgb4_free_stid);
2208
2209/*
2210 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2211 */
2212static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2213 unsigned int tid)
2214{
2215 struct cpl_tid_release *req;
2216
2217 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2218 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2219 INIT_TP_WR(req, tid);
2220 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2221}
2222
2223/*
2224 * Queue a TID release request and if necessary schedule a work queue to
2225 * process it.
2226 */
31b9c19b 2227static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2228 unsigned int tid)
b8ff05a9
DM		 2229{
2230 void **p = &t->tid_tab[tid];
2231 struct adapter *adap = container_of(t, struct adapter, tids);
2232
2233 spin_lock_bh(&adap->tid_release_lock);
2234 *p = adap->tid_release_head;
2235 /* Low 2 bits encode the Tx channel number */
2236 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2237 if (!adap->tid_release_task_busy) {
2238 adap->tid_release_task_busy = true;
2239 schedule_work(&adap->tid_release_task);
2240 }
2241 spin_unlock_bh(&adap->tid_release_lock);
2242}
b8ff05a9
DM		 2243
2244/*
2245 * Process the list of pending TID release requests.
2246 */
2247static void process_tid_release_list(struct work_struct *work)
2248{
2249 struct sk_buff *skb;
2250 struct adapter *adap;
2251
2252 adap = container_of(work, struct adapter, tid_release_task);
2253
2254 spin_lock_bh(&adap->tid_release_lock);
2255 while (adap->tid_release_head) {
2256 void **p = adap->tid_release_head;
2257 unsigned int chan = (uintptr_t)p & 3;
2258 p = (void *)p - chan;
2259
2260 adap->tid_release_head = *p;
2261 *p = NULL;
2262 spin_unlock_bh(&adap->tid_release_lock);
2263
2264 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2265 GFP_KERNEL)))
2266 schedule_timeout_uninterruptible(1);
2267
2268 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2269 t4_ofld_send(adap, skb);
2270 spin_lock_bh(&adap->tid_release_lock);
2271 }
2272 adap->tid_release_task_busy = false;
2273 spin_unlock_bh(&adap->tid_release_lock);
2274}
2275
2276/*
2277 * Release a TID and inform HW. If we are unable to allocate the release
2278 * message we defer to a work queue.
2279 */
2280void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2281{
2282 void *old;
2283 struct sk_buff *skb;
2284 struct adapter *adap = container_of(t, struct adapter, tids);
2285
2286 old = t->tid_tab[tid];
2287 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2288 if (likely(skb)) {
2289 t->tid_tab[tid] = NULL;
2290 mk_tid_release(skb, chan, tid);
2291 t4_ofld_send(adap, skb);
2292 } else
2293 cxgb4_queue_tid_release(t, chan, tid);
2294 if (old)
2295 atomic_dec(&t->tids_in_use);
2296}
2297EXPORT_SYMBOL(cxgb4_remove_tid);
2298
2299/*
2300 * Allocate and initialize the TID tables. Returns 0 on success.
2301 */
2302static int tid_init(struct tid_info *t)
2303{
2304 size_t size;
2305 unsigned int natids = t->natids;
2306
2307 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2308 t->nstids * sizeof(*t->stid_tab) +
2309 BITS_TO_LONGS(t->nstids) * sizeof(long);
2310 t->tid_tab = t4_alloc_mem(size);
2311 if (!t->tid_tab)
2312 return -ENOMEM;
2313
2314 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2315 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2316 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2317 spin_lock_init(&t->stid_lock);
2318 spin_lock_init(&t->atid_lock);
2319
2320 t->stids_in_use = 0;
2321 t->afree = NULL;
2322 t->atids_in_use = 0;
2323 atomic_set(&t->tids_in_use, 0);
2324
2325 /* Setup the free list for atid_tab and clear the stid bitmap. */
2326 if (natids) {
2327 while (--natids)
2328 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2329 t->afree = t->atid_tab;
2330 }
2331 bitmap_zero(t->stid_bmap, t->nstids);
2332 return 0;
2333}
2334
2335/**
2336 * cxgb4_create_server - create an IP server
2337 * @dev: the device
2338 * @stid: the server TID
2339 * @sip: local IP address to bind server to
2340 * @sport: the server's TCP port
2341 * @queue: queue to direct messages from this server to
2342 *
2343 * Create an IP server for the given port and address.
2344 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2345 */
2346int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2347 __be32 sip, __be16 sport, unsigned int queue)
2348{
2349 unsigned int chan;
2350 struct sk_buff *skb;
2351 struct adapter *adap;
2352 struct cpl_pass_open_req *req;
2353
2354 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2355 if (!skb)
2356 return -ENOMEM;
2357
2358 adap = netdev2adap(dev);
2359 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2360 INIT_TP_WR(req, 0);
2361 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2362 req->local_port = sport;
2363 req->peer_port = htons(0);
2364 req->local_ip = sip;
2365 req->peer_ip = htonl(0);
e46dab4d 2366 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 2367 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2368 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2369 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2370 return t4_mgmt_tx(adap, skb);
2371}
2372EXPORT_SYMBOL(cxgb4_create_server);
2373
b8ff05a9
DM		 2374/**
2375 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2376 * @mtus: the HW MTU table
2377 * @mtu: the target MTU
2378 * @idx: index of selected entry in the MTU table
2379 *
2380 * Returns the index and the value in the HW MTU table that is closest to
2381 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2382 * table, in which case that smallest available value is selected.
2383 */
2384unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2385 unsigned int *idx)
2386{
2387 unsigned int i = 0;
2388
2389 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2390 ++i;
2391 if (idx)
2392 *idx = i;
2393 return mtus[i];
2394}
2395EXPORT_SYMBOL(cxgb4_best_mtu);
2396
2397/**
2398 * cxgb4_port_chan - get the HW channel of a port
2399 * @dev: the net device for the port
2400 *
2401 * Return the HW Tx channel of the given port.
2402 */
2403unsigned int cxgb4_port_chan(const struct net_device *dev)
2404{
2405 return netdev2pinfo(dev)->tx_chan;
2406}
2407EXPORT_SYMBOL(cxgb4_port_chan);
2408
2409/**
2410 * cxgb4_port_viid - get the VI id of a port
2411 * @dev: the net device for the port
2412 *
2413 * Return the VI id of the given port.
2414 */
2415unsigned int cxgb4_port_viid(const struct net_device *dev)
2416{
2417 return netdev2pinfo(dev)->viid;
2418}
2419EXPORT_SYMBOL(cxgb4_port_viid);
2420
2421/**
2422 * cxgb4_port_idx - get the index of a port
2423 * @dev: the net device for the port
2424 *
2425 * Return the index of the given port.
2426 */
2427unsigned int cxgb4_port_idx(const struct net_device *dev)
2428{
2429 return netdev2pinfo(dev)->port_id;
2430}
2431EXPORT_SYMBOL(cxgb4_port_idx);
2432
b8ff05a9
DM		 2433void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2434 struct tp_tcp_stats *v6)
2435{
2436 struct adapter *adap = pci_get_drvdata(pdev);
2437
2438 spin_lock(&adap->stats_lock);
2439 t4_tp_get_tcp_stats(adap, v4, v6);
2440 spin_unlock(&adap->stats_lock);
2441}
2442EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2443
2444void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2445 const unsigned int *pgsz_order)
2446{
2447 struct adapter *adap = netdev2adap(dev);
2448
2449 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2450 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2451 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2452 HPZ3(pgsz_order[3]));
2453}
2454EXPORT_SYMBOL(cxgb4_iscsi_init);
2455
2456static struct pci_driver cxgb4_driver;
2457
2458static void check_neigh_update(struct neighbour *neigh)
2459{
2460 const struct device *parent;
2461 const struct net_device *netdev = neigh->dev;
2462
2463 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2464 netdev = vlan_dev_real_dev(netdev);
2465 parent = netdev->dev.parent;
2466 if (parent && parent->driver == &cxgb4_driver.driver)
2467 t4_l2t_update(dev_get_drvdata(parent), neigh);
2468}
2469
2470static int netevent_cb(struct notifier_block *nb, unsigned long event,
2471 void *data)
2472{
2473 switch (event) {
2474 case NETEVENT_NEIGH_UPDATE:
2475 check_neigh_update(data);
2476 break;
2477 case NETEVENT_PMTU_UPDATE:
2478 case NETEVENT_REDIRECT:
2479 default:
2480 break;
2481 }
2482 return 0;
2483}
2484
2485static bool netevent_registered;
2486static struct notifier_block cxgb4_netevent_nb = {
2487 .notifier_call = netevent_cb
2488};
2489
2490static void uld_attach(struct adapter *adap, unsigned int uld)
2491{
2492 void *handle;
2493 struct cxgb4_lld_info lli;
2494
2495 lli.pdev = adap->pdev;
2496 lli.l2t = adap->l2t;
2497 lli.tids = &adap->tids;
2498 lli.ports = adap->port;
2499 lli.vr = &adap->vres;
2500 lli.mtus = adap->params.mtus;
2501 if (uld == CXGB4_ULD_RDMA) {
2502 lli.rxq_ids = adap->sge.rdma_rxq;
2503 lli.nrxq = adap->sge.rdmaqs;
2504 } else if (uld == CXGB4_ULD_ISCSI) {
2505 lli.rxq_ids = adap->sge.ofld_rxq;
2506 lli.nrxq = adap->sge.ofldqsets;
2507 }
2508 lli.ntxq = adap->sge.ofldqsets;
2509 lli.nchan = adap->params.nports;
2510 lli.nports = adap->params.nports;
2511 lli.wr_cred = adap->params.ofldq_wr_cred;
2512 lli.adapter_type = adap->params.rev;
2513 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2514 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2515 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2516 (adap->fn * 4));
b8ff05a9 2517 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2518 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2519 (adap->fn * 4));
b8ff05a9
DM		 2520 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2521 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2522 lli.fw_vers = adap->params.fw_vers;
2523
2524 handle = ulds[uld].add(&lli);
2525 if (IS_ERR(handle)) {
2526 dev_warn(adap->pdev_dev,
2527 "could not attach to the %s driver, error %ld\n",
2528 uld_str[uld], PTR_ERR(handle));
2529 return;
2530 }
2531
2532 adap->uld_handle[uld] = handle;
2533
2534 if (!netevent_registered) {
2535 register_netevent_notifier(&cxgb4_netevent_nb);
2536 netevent_registered = true;
2537 }
e29f5dbc
DM		 2538
2539 if (adap->flags & FULL_INIT_DONE)
2540 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 2541}
2542
2543static void attach_ulds(struct adapter *adap)
2544{
2545 unsigned int i;
2546
2547 mutex_lock(&uld_mutex);
2548 list_add_tail(&adap->list_node, &adapter_list);
2549 for (i = 0; i < CXGB4_ULD_MAX; i++)
2550 if (ulds[i].add)
2551 uld_attach(adap, i);
2552 mutex_unlock(&uld_mutex);
2553}
2554
2555static void detach_ulds(struct adapter *adap)
2556{
2557 unsigned int i;
2558
2559 mutex_lock(&uld_mutex);
2560 list_del(&adap->list_node);
2561 for (i = 0; i < CXGB4_ULD_MAX; i++)
2562 if (adap->uld_handle[i]) {
2563 ulds[i].state_change(adap->uld_handle[i],
2564 CXGB4_STATE_DETACH);
2565 adap->uld_handle[i] = NULL;
2566 }
2567 if (netevent_registered && list_empty(&adapter_list)) {
2568 unregister_netevent_notifier(&cxgb4_netevent_nb);
2569 netevent_registered = false;
2570 }
2571 mutex_unlock(&uld_mutex);
2572}
2573
2574static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2575{
2576 unsigned int i;
2577
2578 mutex_lock(&uld_mutex);
2579 for (i = 0; i < CXGB4_ULD_MAX; i++)
2580 if (adap->uld_handle[i])
2581 ulds[i].state_change(adap->uld_handle[i], new_state);
2582 mutex_unlock(&uld_mutex);
2583}
2584
2585/**
2586 * cxgb4_register_uld - register an upper-layer driver
2587 * @type: the ULD type
2588 * @p: the ULD methods
2589 *
2590 * Registers an upper-layer driver with this driver and notifies the ULD
2591 * about any presently available devices that support its type. Returns
2592 * %-EBUSY if a ULD of the same type is already registered.
2593 */
2594int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2595{
2596 int ret = 0;
2597 struct adapter *adap;
2598
2599 if (type >= CXGB4_ULD_MAX)
2600 return -EINVAL;
2601 mutex_lock(&uld_mutex);
2602 if (ulds[type].add) {
2603 ret = -EBUSY;
2604 goto out;
2605 }
2606 ulds[type] = *p;
2607 list_for_each_entry(adap, &adapter_list, list_node)
2608 uld_attach(adap, type);
2609out: mutex_unlock(&uld_mutex);
2610 return ret;
2611}
2612EXPORT_SYMBOL(cxgb4_register_uld);
2613
2614/**
2615 * cxgb4_unregister_uld - unregister an upper-layer driver
2616 * @type: the ULD type
2617 *
2618 * Unregisters an existing upper-layer driver.
2619 */
2620int cxgb4_unregister_uld(enum cxgb4_uld type)
2621{
2622 struct adapter *adap;
2623
2624 if (type >= CXGB4_ULD_MAX)
2625 return -EINVAL;
2626 mutex_lock(&uld_mutex);
2627 list_for_each_entry(adap, &adapter_list, list_node)
2628 adap->uld_handle[type] = NULL;
2629 ulds[type].add = NULL;
2630 mutex_unlock(&uld_mutex);
2631 return 0;
2632}
2633EXPORT_SYMBOL(cxgb4_unregister_uld);
2634
2635/**
2636 * cxgb_up - enable the adapter
2637 * @adap: adapter being enabled
2638 *
2639 * Called when the first port is enabled, this function performs the
2640 * actions necessary to make an adapter operational, such as completing
2641 * the initialization of HW modules, and enabling interrupts.
2642 *
2643 * Must be called with the rtnl lock held.
2644 */
2645static int cxgb_up(struct adapter *adap)
2646{
aaefae9b 2647 int err;
b8ff05a9 2648
aaefae9b
DM		 2649 err = setup_sge_queues(adap);
2650 if (err)
2651 goto out;
2652 err = setup_rss(adap);
2653 if (err)
2654 goto freeq;
b8ff05a9
DM		 2655
2656 if (adap->flags & USING_MSIX) {
aaefae9b 2657 name_msix_vecs(adap);
b8ff05a9
DM		 2658 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2659 adap->msix_info[0].desc, adap);
2660 if (err)
2661 goto irq_err;
2662
2663 err = request_msix_queue_irqs(adap);
2664 if (err) {
2665 free_irq(adap->msix_info[0].vec, adap);
2666 goto irq_err;
2667 }
2668 } else {
2669 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2670 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2671 adap->name, adap);
2672 if (err)
2673 goto irq_err;
2674 }
2675 enable_rx(adap);
2676 t4_sge_start(adap);
2677 t4_intr_enable(adap);
aaefae9b 2678 adap->flags |= FULL_INIT_DONE;
b8ff05a9
DM		 2679 notify_ulds(adap, CXGB4_STATE_UP);
2680 out:
2681 return err;
2682 irq_err:
2683 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 2684 freeq:
2685 t4_free_sge_resources(adap);
b8ff05a9
DM		 2686 goto out;
2687}
2688
2689static void cxgb_down(struct adapter *adapter)
2690{
2691 t4_intr_disable(adapter);
2692 cancel_work_sync(&adapter->tid_release_task);
2693 adapter->tid_release_task_busy = false;
204dc3c0 2694 adapter->tid_release_head = NULL;
b8ff05a9
DM		 2695
2696 if (adapter->flags & USING_MSIX) {
2697 free_msix_queue_irqs(adapter);
2698 free_irq(adapter->msix_info[0].vec, adapter);
2699 } else
2700 free_irq(adapter->pdev->irq, adapter);
2701 quiesce_rx(adapter);
aaefae9b
DM		 2702 t4_sge_stop(adapter);
2703 t4_free_sge_resources(adapter);
2704 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 2705}
2706
2707/*
2708 * net_device operations
2709 */
2710static int cxgb_open(struct net_device *dev)
2711{
2712 int err;
2713 struct port_info *pi = netdev_priv(dev);
2714 struct adapter *adapter = pi->adapter;
2715
aaefae9b
DM		 2716 if (!(adapter->flags & FULL_INIT_DONE)) {
2717 err = cxgb_up(adapter);
2718 if (err < 0)
2719 return err;
2720 }
b8ff05a9 2721
a020ed4b
BH		 2722 netif_set_real_num_tx_queues(dev, pi->nqsets);
2723 err = netif_set_real_num_rx_queues(dev, pi->nqsets);
2724 if (err)
2725 return err;
f68707b8
DM		 2726 err = link_start(dev);
2727 if (!err)
2728 netif_tx_start_all_queues(dev);
2729 return err;
b8ff05a9
DM		 2730}
2731
2732static int cxgb_close(struct net_device *dev)
2733{
b8ff05a9
DM		 2734 struct port_info *pi = netdev_priv(dev);
2735 struct adapter *adapter = pi->adapter;
2736
2737 netif_tx_stop_all_queues(dev);
2738 netif_carrier_off(dev);
060e0c75 2739 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 2740}
2741
f5152c90
DM		 2742static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2743 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 2744{
2745 struct port_stats stats;
2746 struct port_info *p = netdev_priv(dev);
2747 struct adapter *adapter = p->adapter;
b8ff05a9
DM		 2748
2749 spin_lock(&adapter->stats_lock);
2750 t4_get_port_stats(adapter, p->tx_chan, &stats);
2751 spin_unlock(&adapter->stats_lock);
2752
2753 ns->tx_bytes = stats.tx_octets;
2754 ns->tx_packets = stats.tx_frames;
2755 ns->rx_bytes = stats.rx_octets;
2756 ns->rx_packets = stats.rx_frames;
2757 ns->multicast = stats.rx_mcast_frames;
2758
2759 /* detailed rx_errors */
2760 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2761 stats.rx_runt;
2762 ns->rx_over_errors = 0;
2763 ns->rx_crc_errors = stats.rx_fcs_err;
2764 ns->rx_frame_errors = stats.rx_symbol_err;
2765 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2766 stats.rx_ovflow2 + stats.rx_ovflow3 +
2767 stats.rx_trunc0 + stats.rx_trunc1 +
2768 stats.rx_trunc2 + stats.rx_trunc3;
2769 ns->rx_missed_errors = 0;
2770
2771 /* detailed tx_errors */
2772 ns->tx_aborted_errors = 0;
2773 ns->tx_carrier_errors = 0;
2774 ns->tx_fifo_errors = 0;
2775 ns->tx_heartbeat_errors = 0;
2776 ns->tx_window_errors = 0;
2777
2778 ns->tx_errors = stats.tx_error_frames;
2779 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2780 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2781 return ns;
2782}
2783
2784static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2785{
060e0c75 2786 unsigned int mbox;
b8ff05a9
DM		 2787 int ret = 0, prtad, devad;
2788 struct port_info *pi = netdev_priv(dev);
2789 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2790
2791 switch (cmd) {
2792 case SIOCGMIIPHY:
2793 if (pi->mdio_addr < 0)
2794 return -EOPNOTSUPP;
2795 data->phy_id = pi->mdio_addr;
2796 break;
2797 case SIOCGMIIREG:
2798 case SIOCSMIIREG:
2799 if (mdio_phy_id_is_c45(data->phy_id)) {
2800 prtad = mdio_phy_id_prtad(data->phy_id);
2801 devad = mdio_phy_id_devad(data->phy_id);
2802 } else if (data->phy_id < 32) {
2803 prtad = data->phy_id;
2804 devad = 0;
2805 data->reg_num &= 0x1f;
2806 } else
2807 return -EINVAL;
2808
060e0c75 2809 mbox = pi->adapter->fn;
b8ff05a9 2810 if (cmd == SIOCGMIIREG)
060e0c75 2811 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 2812 data->reg_num, &data->val_out);
2813 else
060e0c75 2814 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 2815 data->reg_num, data->val_in);
2816 break;
2817 default:
2818 return -EOPNOTSUPP;
2819 }
2820 return ret;
2821}
2822
2823static void cxgb_set_rxmode(struct net_device *dev)
2824{
2825 /* unfortunately we can't return errors to the stack */
2826 set_rxmode(dev, -1, false);
2827}
2828
2829static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2830{
2831 int ret;
2832 struct port_info *pi = netdev_priv(dev);
2833
2834 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2835 return -EINVAL;
060e0c75
DM		 2836 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2837 -1, -1, -1, true);
b8ff05a9
DM		 2838 if (!ret)
2839 dev->mtu = new_mtu;
2840 return ret;
2841}
2842
2843static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2844{
2845 int ret;
2846 struct sockaddr *addr = p;
2847 struct port_info *pi = netdev_priv(dev);
2848
2849 if (!is_valid_ether_addr(addr->sa_data))
2850 return -EINVAL;
2851
060e0c75
DM		 2852 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2853 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 2854 if (ret < 0)
2855 return ret;
2856
2857 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2858 pi->xact_addr_filt = ret;
2859 return 0;
2860}
2861
b8ff05a9
DM		 2862#ifdef CONFIG_NET_POLL_CONTROLLER
2863static void cxgb_netpoll(struct net_device *dev)
2864{
2865 struct port_info *pi = netdev_priv(dev);
2866 struct adapter *adap = pi->adapter;
2867
2868 if (adap->flags & USING_MSIX) {
2869 int i;
2870 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2871
2872 for (i = pi->nqsets; i; i--, rx++)
2873 t4_sge_intr_msix(0, &rx->rspq);
2874 } else
2875 t4_intr_handler(adap)(0, adap);
2876}
2877#endif
2878
2879static const struct net_device_ops cxgb4_netdev_ops = {
2880 .ndo_open = cxgb_open,
2881 .ndo_stop = cxgb_close,
2882 .ndo_start_xmit = t4_eth_xmit,
9be793bf 2883 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 2884 .ndo_set_rx_mode = cxgb_set_rxmode,
2885 .ndo_set_mac_address = cxgb_set_mac_addr,
2886 .ndo_validate_addr = eth_validate_addr,
2887 .ndo_do_ioctl = cxgb_ioctl,
2888 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 2889#ifdef CONFIG_NET_POLL_CONTROLLER
2890 .ndo_poll_controller = cxgb_netpoll,
2891#endif
2892};
2893
2894void t4_fatal_err(struct adapter *adap)
2895{
2896 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2897 t4_intr_disable(adap);
2898 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2899}
2900
2901static void setup_memwin(struct adapter *adap)
2902{
2903 u32 bar0;
2904
2905 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2906 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2907 (bar0 + MEMWIN0_BASE) | BIR(0) |
2908 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2909 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2910 (bar0 + MEMWIN1_BASE) | BIR(0) |
2911 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2912 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2913 (bar0 + MEMWIN2_BASE) | BIR(0) |
2914 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1ae970e0
DM		 2915 if (adap->vres.ocq.size) {
2916 unsigned int start, sz_kb;
2917
2918 start = pci_resource_start(adap->pdev, 2) +
2919 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2920 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2921 t4_write_reg(adap,
2922 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2923 start | BIR(1) | WINDOW(ilog2(sz_kb)));
2924 t4_write_reg(adap,
2925 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2926 adap->vres.ocq.start);
2927 t4_read_reg(adap,
2928 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2929 }
b8ff05a9
DM		 2930}
2931
02b5fb8e
DM		 2932static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2933{
2934 u32 v;
2935 int ret;
2936
2937 /* get device capabilities */
2938 memset(c, 0, sizeof(*c));
2939 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2940 FW_CMD_REQUEST | FW_CMD_READ);
2941 c->retval_len16 = htonl(FW_LEN16(*c));
060e0c75 2942 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 2943 if (ret < 0)
2944 return ret;
2945
2946 /* select capabilities we'll be using */
2947 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2948 if (!vf_acls)
2949 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2950 else
2951 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2952 } else if (vf_acls) {
2953 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2954 return ret;
2955 }
2956 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2957 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 2958 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 2959 if (ret < 0)
2960 return ret;
2961
060e0c75 2962 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 2963 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2964 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2965 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2966 if (ret < 0)
2967 return ret;
2968
060e0c75
DM		 2969 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2970 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 2971 if (ret < 0)
2972 return ret;
2973
2974 t4_sge_init(adap);
2975
02b5fb8e
DM		 2976 /* tweak some settings */
2977 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2978 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2979 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2980 v = t4_read_reg(adap, TP_PIO_DATA);
2981 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75
DM		 2982
2983 /* get basic stuff going */
2984 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 2985}
2986
b8ff05a9
DM		 2987/*
2988 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2989 */
2990#define MAX_ATIDS 8192U
2991
2992/*
2993 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2994 */
2995static int adap_init0(struct adapter *adap)
2996{
2997 int ret;
2998 u32 v, port_vec;
2999 enum dev_state state;
3000 u32 params[7], val[7];
3001 struct fw_caps_config_cmd c;
3002
3003 ret = t4_check_fw_version(adap);
3004 if (ret == -EINVAL || ret > 0) {
3005 if (upgrade_fw(adap) >= 0) /* recache FW version */
3006 ret = t4_check_fw_version(adap);
3007 }
3008 if (ret < 0)
3009 return ret;
3010
3011 /* contact FW, request master */
060e0c75 3012 ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
b8ff05a9
DM		 3013 if (ret < 0) {
3014 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3015 ret);
3016 return ret;
3017 }
3018
3019 /* reset device */
060e0c75 3020 ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
b8ff05a9
DM		 3021 if (ret < 0)
3022 goto bye;
3023
b8ff05a9
DM		 3024 for (v = 0; v < SGE_NTIMERS - 1; v++)
3025 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3026 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3027 adap->sge.counter_val[0] = 1;
3028 for (v = 1; v < SGE_NCOUNTERS; v++)
3029 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3030 THRESHOLD_3_MASK);
b8ff05a9
DM		 3031#define FW_PARAM_DEV(param) \
3032 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3033 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3034
a0881cab 3035 params[0] = FW_PARAM_DEV(CCLK);
060e0c75 3036 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
a0881cab
DM		 3037 if (ret < 0)
3038 goto bye;
3039 adap->params.vpd.cclk = val[0];
3040
3041 ret = adap_init1(adap, &c);
3042 if (ret < 0)
3043 goto bye;
3044
b8ff05a9
DM		 3045#define FW_PARAM_PFVF(param) \
3046 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
060e0c75
DM		 3047 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3048 FW_PARAMS_PARAM_Y(adap->fn))
b8ff05a9
DM		 3049
3050 params[0] = FW_PARAM_DEV(PORTVEC);
3051 params[1] = FW_PARAM_PFVF(L2T_START);
3052 params[2] = FW_PARAM_PFVF(L2T_END);
3053 params[3] = FW_PARAM_PFVF(FILTER_START);
3054 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d
DM		 3055 params[5] = FW_PARAM_PFVF(IQFLINT_START);
3056 params[6] = FW_PARAM_PFVF(EQ_START);
3057 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
b8ff05a9
DM		 3058 if (ret < 0)
3059 goto bye;
3060 port_vec = val[0];
3061 adap->tids.ftid_base = val[3];
3062 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d
DM		 3063 adap->sge.ingr_start = val[5];
3064 adap->sge.egr_start = val[6];
b8ff05a9
DM		 3065
3066 if (c.ofldcaps) {
3067 /* query offload-related parameters */
3068 params[0] = FW_PARAM_DEV(NTID);
3069 params[1] = FW_PARAM_PFVF(SERVER_START);
3070 params[2] = FW_PARAM_PFVF(SERVER_END);
3071 params[3] = FW_PARAM_PFVF(TDDP_START);
3072 params[4] = FW_PARAM_PFVF(TDDP_END);
3073 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
060e0c75
DM		 3074 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3075 val);
b8ff05a9
DM		 3076 if (ret < 0)
3077 goto bye;
3078 adap->tids.ntids = val[0];
3079 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3080 adap->tids.stid_base = val[1];
3081 adap->tids.nstids = val[2] - val[1] + 1;
3082 adap->vres.ddp.start = val[3];
3083 adap->vres.ddp.size = val[4] - val[3] + 1;
3084 adap->params.ofldq_wr_cred = val[5];
3085 adap->params.offload = 1;
3086 }
3087 if (c.rdmacaps) {
3088 params[0] = FW_PARAM_PFVF(STAG_START);
3089 params[1] = FW_PARAM_PFVF(STAG_END);
3090 params[2] = FW_PARAM_PFVF(RQ_START);
3091 params[3] = FW_PARAM_PFVF(RQ_END);
3092 params[4] = FW_PARAM_PFVF(PBL_START);
3093 params[5] = FW_PARAM_PFVF(PBL_END);
060e0c75
DM		 3094 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3095 val);
b8ff05a9
DM		 3096 if (ret < 0)
3097 goto bye;
3098 adap->vres.stag.start = val[0];
3099 adap->vres.stag.size = val[1] - val[0] + 1;
3100 adap->vres.rq.start = val[2];
3101 adap->vres.rq.size = val[3] - val[2] + 1;
3102 adap->vres.pbl.start = val[4];
3103 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 3104
3105 params[0] = FW_PARAM_PFVF(SQRQ_START);
3106 params[1] = FW_PARAM_PFVF(SQRQ_END);
3107 params[2] = FW_PARAM_PFVF(CQ_START);
3108 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 3109 params[4] = FW_PARAM_PFVF(OCQ_START);
3110 params[5] = FW_PARAM_PFVF(OCQ_END);
060e0c75
DM		 3111 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3112 val);
a0881cab
DM		 3113 if (ret < 0)
3114 goto bye;
3115 adap->vres.qp.start = val[0];
3116 adap->vres.qp.size = val[1] - val[0] + 1;
3117 adap->vres.cq.start = val[2];
3118 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 3119 adap->vres.ocq.start = val[4];
3120 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9
DM		 3121 }
3122 if (c.iscsicaps) {
3123 params[0] = FW_PARAM_PFVF(ISCSI_START);
3124 params[1] = FW_PARAM_PFVF(ISCSI_END);
060e0c75
DM		 3125 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3126 val);
b8ff05a9
DM		 3127 if (ret < 0)
3128 goto bye;
3129 adap->vres.iscsi.start = val[0];
3130 adap->vres.iscsi.size = val[1] - val[0] + 1;
3131 }
3132#undef FW_PARAM_PFVF
3133#undef FW_PARAM_DEV
3134
3135 adap->params.nports = hweight32(port_vec);
3136 adap->params.portvec = port_vec;
3137 adap->flags |= FW_OK;
3138
3139 /* These are finalized by FW initialization, load their values now */
3140 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3141 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3142 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3143 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3144 adap->params.b_wnd);
7ee9ff94
CL		 3145
3146#ifdef CONFIG_PCI_IOV
3147 /*
3148 * Provision resource limits for Virtual Functions. We currently
3149 * grant them all the same static resource limits except for the Port
3150 * Access Rights Mask which we're assigning based on the PF. All of
3151 * the static provisioning stuff for both the PF and VF really needs
3152 * to be managed in a persistent manner for each device which the
3153 * firmware controls.
3154 */
3155 {
3156 int pf, vf;
3157
3158 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3159 if (num_vf[pf] <= 0)
3160 continue;
3161
3162 /* VF numbering starts at 1! */
3163 for (vf = 1; vf <= num_vf[pf]; vf++) {
060e0c75 3164 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
7ee9ff94
CL		 3165 VFRES_NEQ, VFRES_NETHCTRL,
3166 VFRES_NIQFLINT, VFRES_NIQ,
3167 VFRES_TC, VFRES_NVI,
3168 FW_PFVF_CMD_CMASK_MASK,
3169 pfvfres_pmask(adap, pf, vf),
3170 VFRES_NEXACTF,
3171 VFRES_R_CAPS, VFRES_WX_CAPS);
3172 if (ret < 0)
3173 dev_warn(adap->pdev_dev, "failed to "
3174 "provision pf/vf=%d/%d; "
3175 "err=%d\n", pf, vf, ret);
3176 }
3177 }
3178 }
3179#endif
3180
1ae970e0 3181 setup_memwin(adap);
b8ff05a9
DM		 3182 return 0;
3183
3184 /*
3185 * If a command timed out or failed with EIO FW does not operate within
3186 * its spec or something catastrophic happened to HW/FW, stop issuing
3187 * commands.
3188 */
3189bye: if (ret != -ETIMEDOUT && ret != -EIO)
060e0c75 3190 t4_fw_bye(adap, adap->fn);
b8ff05a9
DM		 3191 return ret;
3192}
3193
204dc3c0
DM		 3194/* EEH callbacks */
3195
3196static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3197 pci_channel_state_t state)
3198{
3199 int i;
3200 struct adapter *adap = pci_get_drvdata(pdev);
3201
3202 if (!adap)
3203 goto out;
3204
3205 rtnl_lock();
3206 adap->flags &= ~FW_OK;
3207 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3208 for_each_port(adap, i) {
3209 struct net_device *dev = adap->port[i];
3210
3211 netif_device_detach(dev);
3212 netif_carrier_off(dev);
3213 }
3214 if (adap->flags & FULL_INIT_DONE)
3215 cxgb_down(adap);
3216 rtnl_unlock();
3217 pci_disable_device(pdev);
3218out: return state == pci_channel_io_perm_failure ?
3219 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3220}
3221
3222static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3223{
3224 int i, ret;
3225 struct fw_caps_config_cmd c;
3226 struct adapter *adap = pci_get_drvdata(pdev);
3227
3228 if (!adap) {
3229 pci_restore_state(pdev);
3230 pci_save_state(pdev);
3231 return PCI_ERS_RESULT_RECOVERED;
3232 }
3233
3234 if (pci_enable_device(pdev)) {
3235 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3236 return PCI_ERS_RESULT_DISCONNECT;
3237 }
3238
3239 pci_set_master(pdev);
3240 pci_restore_state(pdev);
3241 pci_save_state(pdev);
3242 pci_cleanup_aer_uncorrect_error_status(pdev);
3243
3244 if (t4_wait_dev_ready(adap) < 0)
3245 return PCI_ERS_RESULT_DISCONNECT;
060e0c75 3246 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
204dc3c0
DM		 3247 return PCI_ERS_RESULT_DISCONNECT;
3248 adap->flags |= FW_OK;
3249 if (adap_init1(adap, &c))
3250 return PCI_ERS_RESULT_DISCONNECT;
3251
3252 for_each_port(adap, i) {
3253 struct port_info *p = adap2pinfo(adap, i);
3254
060e0c75
DM		 3255 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3256 NULL, NULL);
204dc3c0
DM		 3257 if (ret < 0)
3258 return PCI_ERS_RESULT_DISCONNECT;
3259 p->viid = ret;
3260 p->xact_addr_filt = -1;
3261 }
3262
3263 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3264 adap->params.b_wnd);
1ae970e0 3265 setup_memwin(adap);
204dc3c0
DM		 3266 if (cxgb_up(adap))
3267 return PCI_ERS_RESULT_DISCONNECT;
3268 return PCI_ERS_RESULT_RECOVERED;
3269}
3270
3271static void eeh_resume(struct pci_dev *pdev)
3272{
3273 int i;
3274 struct adapter *adap = pci_get_drvdata(pdev);
3275
3276 if (!adap)
3277 return;
3278
3279 rtnl_lock();
3280 for_each_port(adap, i) {
3281 struct net_device *dev = adap->port[i];
3282
3283 if (netif_running(dev)) {
3284 link_start(dev);
3285 cxgb_set_rxmode(dev);
3286 }
3287 netif_device_attach(dev);
3288 }
3289 rtnl_unlock();
3290}
3291
3292static struct pci_error_handlers cxgb4_eeh = {
3293 .error_detected = eeh_err_detected,
3294 .slot_reset = eeh_slot_reset,
3295 .resume = eeh_resume,
3296};
3297
b8ff05a9
DM		 3298static inline bool is_10g_port(const struct link_config *lc)
3299{
3300 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3301}
3302
3303static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3304 unsigned int size, unsigned int iqe_size)
3305{
3306 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3307 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3308 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3309 q->iqe_len = iqe_size;
3310 q->size = size;
3311}
3312
3313/*
3314 * Perform default configuration of DMA queues depending on the number and type
3315 * of ports we found and the number of available CPUs. Most settings can be
3316 * modified by the admin prior to actual use.
3317 */
3318static void __devinit cfg_queues(struct adapter *adap)
3319{
3320 struct sge *s = &adap->sge;
3321 int i, q10g = 0, n10g = 0, qidx = 0;
3322
3323 for_each_port(adap, i)
3324 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3325
3326 /*
3327 * We default to 1 queue per non-10G port and up to # of cores queues
3328 * per 10G port.
3329 */
3330 if (n10g)
3331 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3332 if (q10g > num_online_cpus())
3333 q10g = num_online_cpus();
3334
3335 for_each_port(adap, i) {
3336 struct port_info *pi = adap2pinfo(adap, i);
3337
3338 pi->first_qset = qidx;
3339 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3340 qidx += pi->nqsets;
3341 }
3342
3343 s->ethqsets = qidx;
3344 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3345
3346 if (is_offload(adap)) {
3347 /*
3348 * For offload we use 1 queue/channel if all ports are up to 1G,
3349 * otherwise we divide all available queues amongst the channels
3350 * capped by the number of available cores.
3351 */
3352 if (n10g) {
3353 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3354 num_online_cpus());
3355 s->ofldqsets = roundup(i, adap->params.nports);
3356 } else
3357 s->ofldqsets = adap->params.nports;
3358 /* For RDMA one Rx queue per channel suffices */
3359 s->rdmaqs = adap->params.nports;
3360 }
3361
3362 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3363 struct sge_eth_rxq *r = &s->ethrxq[i];
3364
3365 init_rspq(&r->rspq, 0, 0, 1024, 64);
3366 r->fl.size = 72;
3367 }
3368
3369 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3370 s->ethtxq[i].q.size = 1024;
3371
3372 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3373 s->ctrlq[i].q.size = 512;
3374
3375 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3376 s->ofldtxq[i].q.size = 1024;
3377
3378 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3379 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3380
3381 init_rspq(&r->rspq, 0, 0, 1024, 64);
3382 r->rspq.uld = CXGB4_ULD_ISCSI;
3383 r->fl.size = 72;
3384 }
3385
3386 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3387 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3388
3389 init_rspq(&r->rspq, 0, 0, 511, 64);
3390 r->rspq.uld = CXGB4_ULD_RDMA;
3391 r->fl.size = 72;
3392 }
3393
3394 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3395 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3396}
3397
3398/*
3399 * Reduce the number of Ethernet queues across all ports to at most n.
3400 * n provides at least one queue per port.
3401 */
3402static void __devinit reduce_ethqs(struct adapter *adap, int n)
3403{
3404 int i;
3405 struct port_info *pi;
3406
3407 while (n < adap->sge.ethqsets)
3408 for_each_port(adap, i) {
3409 pi = adap2pinfo(adap, i);
3410 if (pi->nqsets > 1) {
3411 pi->nqsets--;
3412 adap->sge.ethqsets--;
3413 if (adap->sge.ethqsets <= n)
3414 break;
3415 }
3416 }
3417
3418 n = 0;
3419 for_each_port(adap, i) {
3420 pi = adap2pinfo(adap, i);
3421 pi->first_qset = n;
3422 n += pi->nqsets;
3423 }
3424}
3425
3426/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3427#define EXTRA_VECS 2
3428
3429static int __devinit enable_msix(struct adapter *adap)
3430{
3431 int ofld_need = 0;
3432 int i, err, want, need;
3433 struct sge *s = &adap->sge;
3434 unsigned int nchan = adap->params.nports;
3435 struct msix_entry entries[MAX_INGQ + 1];
3436
3437 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3438 entries[i].entry = i;
3439
3440 want = s->max_ethqsets + EXTRA_VECS;
3441 if (is_offload(adap)) {
3442 want += s->rdmaqs + s->ofldqsets;
3443 /* need nchan for each possible ULD */
3444 ofld_need = 2 * nchan;
3445 }
3446 need = adap->params.nports + EXTRA_VECS + ofld_need;
3447
3448 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3449 want = err;
3450
3451 if (!err) {
3452 /*
3453 * Distribute available vectors to the various queue groups.
3454 * Every group gets its minimum requirement and NIC gets top
3455 * priority for leftovers.
3456 */
3457 i = want - EXTRA_VECS - ofld_need;
3458 if (i < s->max_ethqsets) {
3459 s->max_ethqsets = i;
3460 if (i < s->ethqsets)
3461 reduce_ethqs(adap, i);
3462 }
3463 if (is_offload(adap)) {
3464 i = want - EXTRA_VECS - s->max_ethqsets;
3465 i -= ofld_need - nchan;
3466 s->ofldqsets = (i / nchan) * nchan; /* round down */
3467 }
3468 for (i = 0; i < want; ++i)
3469 adap->msix_info[i].vec = entries[i].vector;
3470 } else if (err > 0)
3471 dev_info(adap->pdev_dev,
3472 "only %d MSI-X vectors left, not using MSI-X\n", err);
3473 return err;
3474}
3475
3476#undef EXTRA_VECS
3477
671b0060
DM		 3478static int __devinit init_rss(struct adapter *adap)
3479{
3480 unsigned int i, j;
3481
3482 for_each_port(adap, i) {
3483 struct port_info *pi = adap2pinfo(adap, i);
3484
3485 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3486 if (!pi->rss)
3487 return -ENOMEM;
3488 for (j = 0; j < pi->rss_size; j++)
3489 pi->rss[j] = j % pi->nqsets;
3490 }
3491 return 0;
3492}
3493
b8ff05a9
DM		 3494static void __devinit print_port_info(struct adapter *adap)
3495{
3496 static const char *base[] = {
a0881cab
DM		 3497 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3498 "KX", "KR", "KR SFP+", "KR FEC"
b8ff05a9
DM		 3499 };
3500
3501 int i;
3502 char buf[80];
f1a051b9
DM		 3503 const char *spd = "";
3504
3505 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3506 spd = " 2.5 GT/s";
3507 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3508 spd = " 5 GT/s";
b8ff05a9
DM		 3509
3510 for_each_port(adap, i) {
3511 struct net_device *dev = adap->port[i];
3512 const struct port_info *pi = netdev_priv(dev);
3513 char *bufp = buf;
3514
3515 if (!test_bit(i, &adap->registered_device_map))
3516 continue;
3517
3518 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3519 bufp += sprintf(bufp, "100/");
3520 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3521 bufp += sprintf(bufp, "1000/");
3522 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3523 bufp += sprintf(bufp, "10G/");
3524 if (bufp != buf)
3525 --bufp;
3526 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3527
f1a051b9 3528 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
b8ff05a9
DM		 3529 adap->params.vpd.id, adap->params.rev,
3530 buf, is_offload(adap) ? "R" : "",
f1a051b9 3531 adap->params.pci.width, spd,
b8ff05a9
DM		 3532 (adap->flags & USING_MSIX) ? " MSI-X" :
3533 (adap->flags & USING_MSI) ? " MSI" : "");
3534 if (adap->name == dev->name)
3535 netdev_info(dev, "S/N: %s, E/C: %s\n",
3536 adap->params.vpd.sn, adap->params.vpd.ec);
3537 }
3538}
3539
06546391
DM		 3540/*
3541 * Free the following resources:
3542 * - memory used for tables
3543 * - MSI/MSI-X
3544 * - net devices
3545 * - resources FW is holding for us
3546 */
3547static void free_some_resources(struct adapter *adapter)
3548{
3549 unsigned int i;
3550
3551 t4_free_mem(adapter->l2t);
3552 t4_free_mem(adapter->tids.tid_tab);
3553 disable_msi(adapter);
3554
3555 for_each_port(adapter, i)
671b0060
DM		 3556 if (adapter->port[i]) {
3557 kfree(adap2pinfo(adapter, i)->rss);
06546391 3558 free_netdev(adapter->port[i]);
671b0060 3559 }
06546391 3560 if (adapter->flags & FW_OK)
060e0c75 3561 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 3562}
3563
35d35682 3564#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9
DM		 3565 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3566
3567static int __devinit init_one(struct pci_dev *pdev,
3568 const struct pci_device_id *ent)
3569{
3570 int func, i, err;
3571 struct port_info *pi;
3572 unsigned int highdma = 0;
3573 struct adapter *adapter = NULL;
3574
3575 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3576
3577 err = pci_request_regions(pdev, KBUILD_MODNAME);
3578 if (err) {
3579 /* Just info, some other driver may have claimed the device. */
3580 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3581 return err;
3582 }
3583
060e0c75 3584 /* We control everything through one PF */
b8ff05a9 3585 func = PCI_FUNC(pdev->devfn);
060e0c75 3586 if (func != ent->driver_data) {
204dc3c0 3587 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 3588 goto sriov;
204dc3c0 3589 }
b8ff05a9
DM		 3590
3591 err = pci_enable_device(pdev);
3592 if (err) {
3593 dev_err(&pdev->dev, "cannot enable PCI device\n");
3594 goto out_release_regions;
3595 }
3596
3597 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3598 highdma = NETIF_F_HIGHDMA;
3599 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3600 if (err) {
3601 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3602 "coherent allocations\n");
3603 goto out_disable_device;
3604 }
3605 } else {
3606 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3607 if (err) {
3608 dev_err(&pdev->dev, "no usable DMA configuration\n");
3609 goto out_disable_device;
3610 }
3611 }
3612
3613 pci_enable_pcie_error_reporting(pdev);
3614 pci_set_master(pdev);
3615 pci_save_state(pdev);
3616
3617 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3618 if (!adapter) {
3619 err = -ENOMEM;
3620 goto out_disable_device;
3621 }
3622
3623 adapter->regs = pci_ioremap_bar(pdev, 0);
3624 if (!adapter->regs) {
3625 dev_err(&pdev->dev, "cannot map device registers\n");
3626 err = -ENOMEM;
3627 goto out_free_adapter;
3628 }
3629
3630 adapter->pdev = pdev;
3631 adapter->pdev_dev = &pdev->dev;
060e0c75 3632 adapter->fn = func;
b8ff05a9
DM		 3633 adapter->name = pci_name(pdev);
3634 adapter->msg_enable = dflt_msg_enable;
3635 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3636
3637 spin_lock_init(&adapter->stats_lock);
3638 spin_lock_init(&adapter->tid_release_lock);
3639
3640 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3641
3642 err = t4_prep_adapter(adapter);
3643 if (err)
3644 goto out_unmap_bar;
3645 err = adap_init0(adapter);
3646 if (err)
3647 goto out_unmap_bar;
3648
3649 for_each_port(adapter, i) {
3650 struct net_device *netdev;
3651
3652 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3653 MAX_ETH_QSETS);
3654 if (!netdev) {
3655 err = -ENOMEM;
3656 goto out_free_dev;
3657 }
3658
3659 SET_NETDEV_DEV(netdev, &pdev->dev);
3660
3661 adapter->port[i] = netdev;
3662 pi = netdev_priv(netdev);
3663 pi->adapter = adapter;
3664 pi->xact_addr_filt = -1;
3665 pi->rx_offload = RX_CSO;
3666 pi->port_id = i;
3667 netif_carrier_off(netdev);
b8ff05a9
DM		 3668 netdev->irq = pdev->irq;
3669
35d35682 3670 netdev->features |= NETIF_F_SG | TSO_FLAGS;
b8ff05a9 3671 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
87b6cf51 3672 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
b8ff05a9
DM		 3673 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3674 netdev->vlan_features = netdev->features & VLAN_FEAT;
3675
3676 netdev->netdev_ops = &cxgb4_netdev_ops;
3677 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3678 }
3679
3680 pci_set_drvdata(pdev, adapter);
3681
3682 if (adapter->flags & FW_OK) {
060e0c75 3683 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 3684 if (err)
3685 goto out_free_dev;
3686 }
3687
3688 /*
3689 * Configure queues and allocate tables now, they can be needed as
3690 * soon as the first register_netdev completes.
3691 */
3692 cfg_queues(adapter);
3693
3694 adapter->l2t = t4_init_l2t();
3695 if (!adapter->l2t) {
3696 /* We tolerate a lack of L2T, giving up some functionality */
3697 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3698 adapter->params.offload = 0;
3699 }
3700
3701 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3702 dev_warn(&pdev->dev, "could not allocate TID table, "
3703 "continuing\n");
3704 adapter->params.offload = 0;
3705 }
3706
f7cabcdd
DM		 3707 /* See what interrupts we'll be using */
3708 if (msi > 1 && enable_msix(adapter) == 0)
3709 adapter->flags |= USING_MSIX;
3710 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3711 adapter->flags |= USING_MSI;
3712
671b0060
DM		 3713 err = init_rss(adapter);
3714 if (err)
3715 goto out_free_dev;
3716
b8ff05a9
DM		 3717 /*
3718 * The card is now ready to go. If any errors occur during device
3719 * registration we do not fail the whole card but rather proceed only
3720 * with the ports we manage to register successfully. However we must
3721 * register at least one net device.
3722 */
3723 for_each_port(adapter, i) {
3724 err = register_netdev(adapter->port[i]);
3725 if (err)
3726 dev_warn(&pdev->dev,
3727 "cannot register net device %s, skipping\n",
3728 adapter->port[i]->name);
3729 else {
3730 /*
3731 * Change the name we use for messages to the name of
3732 * the first successfully registered interface.
3733 */
3734 if (!adapter->registered_device_map)
3735 adapter->name = adapter->port[i]->name;
3736
3737 __set_bit(i, &adapter->registered_device_map);
3738 adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3739 }
3740 }
3741 if (!adapter->registered_device_map) {
3742 dev_err(&pdev->dev, "could not register any net devices\n");
3743 goto out_free_dev;
3744 }
3745
3746 if (cxgb4_debugfs_root) {
3747 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3748 cxgb4_debugfs_root);
3749 setup_debugfs(adapter);
3750 }
3751
b8ff05a9
DM		 3752 if (is_offload(adapter))
3753 attach_ulds(adapter);
3754
3755 print_port_info(adapter);
3756
3757sriov:
3758#ifdef CONFIG_PCI_IOV
3759 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3760 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3761 dev_info(&pdev->dev,
3762 "instantiated %u virtual functions\n",
3763 num_vf[func]);
3764#endif
3765 return 0;
3766
3767 out_free_dev:
06546391 3768 free_some_resources(adapter);
b8ff05a9
DM		 3769 out_unmap_bar:
3770 iounmap(adapter->regs);
3771 out_free_adapter:
3772 kfree(adapter);
3773 out_disable_device:
3774 pci_disable_pcie_error_reporting(pdev);
3775 pci_disable_device(pdev);
3776 out_release_regions:
3777 pci_release_regions(pdev);
3778 pci_set_drvdata(pdev, NULL);
3779 return err;
3780}
3781
3782static void __devexit remove_one(struct pci_dev *pdev)
3783{
3784 struct adapter *adapter = pci_get_drvdata(pdev);
3785
3786 pci_disable_sriov(pdev);
3787
3788 if (adapter) {
3789 int i;
3790
3791 if (is_offload(adapter))
3792 detach_ulds(adapter);
3793
3794 for_each_port(adapter, i)
3795 if (test_bit(i, &adapter->registered_device_map))
3796 unregister_netdev(adapter->port[i]);
3797
3798 if (adapter->debugfs_root)
3799 debugfs_remove_recursive(adapter->debugfs_root);
3800
aaefae9b
DM		 3801 if (adapter->flags & FULL_INIT_DONE)
3802 cxgb_down(adapter);
b8ff05a9 3803
06546391 3804 free_some_resources(adapter);
b8ff05a9
DM		 3805 iounmap(adapter->regs);
3806 kfree(adapter);
3807 pci_disable_pcie_error_reporting(pdev);
3808 pci_disable_device(pdev);
3809 pci_release_regions(pdev);
3810 pci_set_drvdata(pdev, NULL);
a069ec91 3811 } else
b8ff05a9
DM		 3812 pci_release_regions(pdev);
3813}
3814
3815static struct pci_driver cxgb4_driver = {
3816 .name = KBUILD_MODNAME,
3817 .id_table = cxgb4_pci_tbl,
3818 .probe = init_one,
3819 .remove = __devexit_p(remove_one),
204dc3c0 3820 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 3821};
3822
3823static int __init cxgb4_init_module(void)
3824{
3825 int ret;
3826
3827 /* Debugfs support is optional, just warn if this fails */
3828 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3829 if (!cxgb4_debugfs_root)
3830 pr_warning("could not create debugfs entry, continuing\n");
3831
3832 ret = pci_register_driver(&cxgb4_driver);
3833 if (ret < 0)
3834 debugfs_remove(cxgb4_debugfs_root);
3835 return ret;
3836}
3837
3838static void __exit cxgb4_cleanup_module(void)
3839{
3840 pci_unregister_driver(&cxgb4_driver);
3841 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3842}
3843
3844module_init(cxgb4_init_module);
3845module_exit(cxgb4_cleanup_module);
Clone of davem's net-next-2.6 tree