drivers/net/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if_vlan.h>
  45 #include <linux/init.h>
  46 #include <linux/log2.h>
  47 #include <linux/mdio.h>
  48 #include <linux/module.h>
  49 #include <linux/moduleparam.h>
  50 #include <linux/mutex.h>
  51 #include <linux/netdevice.h>
  52 #include <linux/pci.h>
  53 #include <linux/aer.h>
  54 #include <linux/rtnetlink.h>
  55 #include <linux/sched.h>
  56 #include <linux/seq_file.h>
  57 #include <linux/sockios.h>
  58 #include <linux/vmalloc.h>
  59 #include <linux/workqueue.h>
  60 #include <net/neighbour.h>
  61 #include <net/netevent.h>
  62 #include <asm/uaccess.h>
  63
  64 #include "cxgb4.h"
  65 #include "t4_regs.h"
  66 #include "t4_msg.h"
  67 #include "t4fw_api.h"
  68 #include "l2t.h"
  69
  70 #define DRV_VERSION "1.3.0-ko"
  71 #define DRV_DESC "Chelsio T4 Network Driver"
  72
  73 /*
  74  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  75  * under extreme memory pressure so it's largish to give the system time to
  76  * recover.
  77  */
  78 #define MAX_SGE_TIMERVAL 200U
  79
  80 #ifdef CONFIG_PCI_IOV
  81 /*
  82  * Virtual Function provisioning constants.  We need two extra Ingress Queues
  83  * with Interrupt capability to serve as the VF's Firmware Event Queue and
  84  * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
  85  * Lists associated with them).  For each Ethernet/Control Egress Queue and
  86  * for each Free List, we need an Egress Context.
  87  */
  88 enum {
  89         VFRES_NPORTS = 1,               /* # of "ports" per VF */
  90         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
  91
  92         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
  93         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
  94         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
  95         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
  96         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
  97         VFRES_TC = 0,                   /* PCI-E traffic class */
  98         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
  99
 100         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 101         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 102 };
 103
 104 /*
 105  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 106  * static and likely not to be useful in the long run.  We really need to
 107  * implement some form of persistent configuration which the firmware
 108  * controls.
 109  */
 110 static unsigned int pfvfres_pmask(struct adapter *adapter,
 111                                   unsigned int pf, unsigned int vf)
 112 {
 113         unsigned int portn, portvec;
 114
 115         /*
 116          * Give PF's access to all of the ports.
 117          */
 118         if (vf == 0)
 119                 return FW_PFVF_CMD_PMASK_MASK;
 120
 121         /*
 122          * For VFs, we'll assign them access to the ports based purely on the
 123          * PF.  We assign active ports in order, wrapping around if there are
 124          * fewer active ports than PFs: e.g. active port[pf % nports].
 125          * Unfortunately the adapter's port_info structs haven't been
 126          * initialized yet so we have to compute this.
 127          */
 128         if (adapter->params.nports == 0)
 129                 return 0;
 130
 131         portn = pf % adapter->params.nports;
 132         portvec = adapter->params.portvec;
 133         for (;;) {
 134                 /*
 135                  * Isolate the lowest set bit in the port vector.  If we're at
 136                  * the port number that we want, return that as the pmask.
 137                  * otherwise mask that bit out of the port vector and
 138                  * decrement our port number ...
 139                  */
 140                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 141                 if (portn == 0)
 142                         return pmask;
 143                 portn--;
 144                 portvec &= ~pmask;
 145         }
 146         /*NOTREACHED*/
 147 }
 148 #endif
 149
 150 enum {
 151         MEMWIN0_APERTURE = 65536,
 152         MEMWIN0_BASE     = 0x30000,
 153         MEMWIN1_APERTURE = 32768,
 154         MEMWIN1_BASE     = 0x28000,
 155         MEMWIN2_APERTURE = 2048,
 156         MEMWIN2_BASE     = 0x1b800,
 157 };
 158
 159 enum {
 160         MAX_TXQ_ENTRIES      = 16384,
 161         MAX_CTRL_TXQ_ENTRIES = 1024,
 162         MAX_RSPQ_ENTRIES     = 16384,
 163         MAX_RX_BUFFERS       = 16384,
 164         MIN_TXQ_ENTRIES      = 32,
 165         MIN_CTRL_TXQ_ENTRIES = 32,
 166         MIN_RSPQ_ENTRIES     = 128,
 167         MIN_FL_ENTRIES       = 16
 168 };
 169
 170 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 171                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 172                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 173
 174 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 175
 176 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 177         CH_DEVICE(0xa000, 0),  /* PE10K */
 178         CH_DEVICE(0x4001, -1),
 179         CH_DEVICE(0x4002, -1),
 180         CH_DEVICE(0x4003, -1),
 181         CH_DEVICE(0x4004, -1),
 182         CH_DEVICE(0x4005, -1),
 183         CH_DEVICE(0x4006, -1),
 184         CH_DEVICE(0x4007, -1),
 185         CH_DEVICE(0x4008, -1),
 186         CH_DEVICE(0x4009, -1),
 187         CH_DEVICE(0x400a, -1),
 188         CH_DEVICE(0x4401, 4),
 189         CH_DEVICE(0x4402, 4),
 190         CH_DEVICE(0x4403, 4),
 191         CH_DEVICE(0x4404, 4),
 192         CH_DEVICE(0x4405, 4),
 193         CH_DEVICE(0x4406, 4),
 194         CH_DEVICE(0x4407, 4),
 195         CH_DEVICE(0x4408, 4),
 196         CH_DEVICE(0x4409, 4),
 197         CH_DEVICE(0x440a, 4),
 198         { 0, }
 199 };
 200
 201 #define FW_FNAME "cxgb4/t4fw.bin"
 202
 203 MODULE_DESCRIPTION(DRV_DESC);
 204 MODULE_AUTHOR("Chelsio Communications");
 205 MODULE_LICENSE("Dual BSD/GPL");
 206 MODULE_VERSION(DRV_VERSION);
 207 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 208 MODULE_FIRMWARE(FW_FNAME);
 209
 210 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 211
 212 module_param(dflt_msg_enable, int, 0644);
 213 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 214
 215 /*
 216  * The driver uses the best interrupt scheme available on a platform in the
 217  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 218  * of these schemes the driver may consider as follows:
 219  *
 220  * msi = 2: choose from among all three options
 221  * msi = 1: only consider MSI and INTx interrupts
 222  * msi = 0: force INTx interrupts
 223  */
 224 static int msi = 2;
 225
 226 module_param(msi, int, 0644);
 227 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 228
 229 /*
 230  * Queue interrupt hold-off timer values.  Queues default to the first of these
 231  * upon creation.
 232  */
 233 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 234
 235 module_param_array(intr_holdoff, uint, NULL, 0644);
 236 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 237                  "0..4 in microseconds");
 238
 239 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 240
 241 module_param_array(intr_cnt, uint, NULL, 0644);
 242 MODULE_PARM_DESC(intr_cnt,
 243                  "thresholds 1..3 for queue interrupt packet counters");
 244
 245 static int vf_acls;
 246
 247 #ifdef CONFIG_PCI_IOV
 248 module_param(vf_acls, bool, 0644);
 249 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 250
 251 static unsigned int num_vf[4];
 252
 253 module_param_array(num_vf, uint, NULL, 0644);
 254 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 255 #endif
 256
 257 static struct dentry *cxgb4_debugfs_root;
 258
 259 static LIST_HEAD(adapter_list);
 260 static DEFINE_MUTEX(uld_mutex);
 261 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 262 static const char *uld_str[] = { "RDMA", "iSCSI" };
 263
 264 static void link_report(struct net_device *dev)
 265 {
 266         if (!netif_carrier_ok(dev))
 267                 netdev_info(dev, "link down\n");
 268         else {
 269                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 270
 271                 const char *s = "10Mbps";
 272                 const struct port_info *p = netdev_priv(dev);
 273
 274                 switch (p->link_cfg.speed) {
 275                 case SPEED_10000:
 276                         s = "10Gbps";
 277                         break;
 278                 case SPEED_1000:
 279                         s = "1000Mbps";
 280                         break;
 281                 case SPEED_100:
 282                         s = "100Mbps";
 283                         break;
 284                 }
 285
 286                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 287                             fc[p->link_cfg.fc]);
 288         }
 289 }
 290
 291 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 292 {
 293         struct net_device *dev = adapter->port[port_id];
 294
 295         /* Skip changes from disabled ports. */
 296         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 297                 if (link_stat)
 298                         netif_carrier_on(dev);
 299                 else
 300                         netif_carrier_off(dev);
 301
 302                 link_report(dev);
 303         }
 304 }
 305
 306 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 307 {
 308         static const char *mod_str[] = {
 309                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 310         };
 311
 312         const struct net_device *dev = adap->port[port_id];
 313         const struct port_info *pi = netdev_priv(dev);
 314
 315         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 316                 netdev_info(dev, "port module unplugged\n");
 317         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 318                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 319 }
 320
 321 /*
 322  * Configure the exact and hash address filters to handle a port's multicast
 323  * and secondary unicast MAC addresses.
 324  */
 325 static int set_addr_filters(const struct net_device *dev, bool sleep)
 326 {
 327         u64 mhash = 0;
 328         u64 uhash = 0;
 329         bool free = true;
 330         u16 filt_idx[7];
 331         const u8 *addr[7];
 332         int ret, naddr = 0;
 333         const struct netdev_hw_addr *ha;
 334         int uc_cnt = netdev_uc_count(dev);
 335         int mc_cnt = netdev_mc_count(dev);
 336         const struct port_info *pi = netdev_priv(dev);
 337         unsigned int mb = pi->adapter->fn;
 338
 339         /* first do the secondary unicast addresses */
 340         netdev_for_each_uc_addr(ha, dev) {
 341                 addr[naddr++] = ha->addr;
 342                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 343                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 344                                         naddr, addr, filt_idx, &uhash, sleep);
 345                         if (ret < 0)
 346                                 return ret;
 347
 348                         free = false;
 349                         naddr = 0;
 350                 }
 351         }
 352
 353         /* next set up the multicast addresses */
 354         netdev_for_each_mc_addr(ha, dev) {
 355                 addr[naddr++] = ha->addr;
 356                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 357                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 358                                         naddr, addr, filt_idx, &mhash, sleep);
 359                         if (ret < 0)
 360                                 return ret;
 361
 362                         free = false;
 363                         naddr = 0;
 364                 }
 365         }
 366
 367         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 368                                 uhash | mhash, sleep);
 369 }
 370
 371 /*
 372  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 373  * If @mtu is -1 it is left unchanged.
 374  */
 375 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 376 {
 377         int ret;
 378         struct port_info *pi = netdev_priv(dev);
 379
 380         ret = set_addr_filters(dev, sleep_ok);
 381         if (ret == 0)
 382                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 383                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 384                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 385                                     sleep_ok);
 386         return ret;
 387 }
 388
 389 /**
 390  *      link_start - enable a port
 391  *      @dev: the port to enable
 392  *
 393  *      Performs the MAC and PHY actions needed to enable a port.
 394  */
 395 static int link_start(struct net_device *dev)
 396 {
 397         int ret;
 398         struct port_info *pi = netdev_priv(dev);
 399         unsigned int mb = pi->adapter->fn;
 400
 401         /*
 402          * We do not set address filters and promiscuity here, the stack does
 403          * that step explicitly.
 404          */
 405         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 406                             pi->vlan_grp != NULL, true);
 407         if (ret == 0) {
 408                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 409                                     pi->xact_addr_filt, dev->dev_addr, true,
 410                                     true);
 411                 if (ret >= 0) {
 412                         pi->xact_addr_filt = ret;
 413                         ret = 0;
 414                 }
 415         }
 416         if (ret == 0)
 417                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 418                                     &pi->link_cfg);
 419         if (ret == 0)
 420                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 421         return ret;
 422 }
 423
 424 /*
 425  * Response queue handler for the FW event queue.
 426  */
 427 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 428                           const struct pkt_gl *gl)
 429 {
 430         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 431
 432         rsp++;                                          /* skip RSS header */
 433         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 434                 const struct cpl_sge_egr_update *p = (void *)rsp;
 435                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 436                 struct sge_txq *txq;
 437
 438                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 439                 txq->restarts++;
 440                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 441                         struct sge_eth_txq *eq;
 442
 443                         eq = container_of(txq, struct sge_eth_txq, q);
 444                         netif_tx_wake_queue(eq->txq);
 445                 } else {
 446                         struct sge_ofld_txq *oq;
 447
 448                         oq = container_of(txq, struct sge_ofld_txq, q);
 449                         tasklet_schedule(&oq->qresume_tsk);
 450                 }
 451         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 452                 const struct cpl_fw6_msg *p = (void *)rsp;
 453
 454                 if (p->type == 0)
 455                         t4_handle_fw_rpl(q->adap, p->data);
 456         } else if (opcode == CPL_L2T_WRITE_RPL) {
 457                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 458
 459                 do_l2t_write_rpl(q->adap, p);
 460         } else
 461                 dev_err(q->adap->pdev_dev,
 462                         "unexpected CPL %#x on FW event queue\n", opcode);
 463         return 0;
 464 }
 465
 466 /**
 467  *      uldrx_handler - response queue handler for ULD queues
 468  *      @q: the response queue that received the packet
 469  *      @rsp: the response queue descriptor holding the offload message
 470  *      @gl: the gather list of packet fragments
 471  *
 472  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 473  *      the ULD, we just maintain statistics.
 474  */
 475 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 476                          const struct pkt_gl *gl)
 477 {
 478         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 479
 480         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 481                 rxq->stats.nomem++;
 482                 return -1;
 483         }
 484         if (gl == NULL)
 485                 rxq->stats.imm++;
 486         else if (gl == CXGB4_MSG_AN)
 487                 rxq->stats.an++;
 488         else
 489                 rxq->stats.pkts++;
 490         return 0;
 491 }
 492
 493 static void disable_msi(struct adapter *adapter)
 494 {
 495         if (adapter->flags & USING_MSIX) {
 496                 pci_disable_msix(adapter->pdev);
 497                 adapter->flags &= ~USING_MSIX;
 498         } else if (adapter->flags & USING_MSI) {
 499                 pci_disable_msi(adapter->pdev);
 500                 adapter->flags &= ~USING_MSI;
 501         }
 502 }
 503
 504 /*
 505  * Interrupt handler for non-data events used with MSI-X.
 506  */
 507 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 508 {
 509         struct adapter *adap = cookie;
 510
 511         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 512         if (v & PFSW) {
 513                 adap->swintr = 1;
 514                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 515         }
 516         t4_slow_intr_handler(adap);
 517         return IRQ_HANDLED;
 518 }
 519
 520 /*
 521  * Name the MSI-X interrupts.
 522  */
 523 static void name_msix_vecs(struct adapter *adap)
 524 {
 525         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
 526
 527         /* non-data interrupts */
 528         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 529         adap->msix_info[0].desc[n] = 0;
 530
 531         /* FW events */
 532         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
 533         adap->msix_info[1].desc[n] = 0;
 534
 535         /* Ethernet queues */
 536         for_each_port(adap, j) {
 537                 struct net_device *d = adap->port[j];
 538                 const struct port_info *pi = netdev_priv(d);
 539
 540                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 541                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 542                                  d->name, i);
 543                         adap->msix_info[msi_idx].desc[n] = 0;
 544                 }
 545         }
 546
 547         /* offload queues */
 548         for_each_ofldrxq(&adap->sge, i) {
 549                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
 550                          adap->name, i);
 551                 adap->msix_info[msi_idx++].desc[n] = 0;
 552         }
 553         for_each_rdmarxq(&adap->sge, i) {
 554                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
 555                          adap->name, i);
 556                 adap->msix_info[msi_idx++].desc[n] = 0;
 557         }
 558 }
 559
 560 static int request_msix_queue_irqs(struct adapter *adap)
 561 {
 562         struct sge *s = &adap->sge;
 563         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
 564
 565         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 566                           adap->msix_info[1].desc, &s->fw_evtq);
 567         if (err)
 568                 return err;
 569
 570         for_each_ethrxq(s, ethqidx) {
 571                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 572                                   adap->msix_info[msi].desc,
 573                                   &s->ethrxq[ethqidx].rspq);
 574                 if (err)
 575                         goto unwind;
 576                 msi++;
 577         }
 578         for_each_ofldrxq(s, ofldqidx) {
 579                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 580                                   adap->msix_info[msi].desc,
 581                                   &s->ofldrxq[ofldqidx].rspq);
 582                 if (err)
 583                         goto unwind;
 584                 msi++;
 585         }
 586         for_each_rdmarxq(s, rdmaqidx) {
 587                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 588                                   adap->msix_info[msi].desc,
 589                                   &s->rdmarxq[rdmaqidx].rspq);
 590                 if (err)
 591                         goto unwind;
 592                 msi++;
 593         }
 594         return 0;
 595
 596 unwind:
 597         while (--rdmaqidx >= 0)
 598                 free_irq(adap->msix_info[--msi].vec,
 599                          &s->rdmarxq[rdmaqidx].rspq);
 600         while (--ofldqidx >= 0)
 601                 free_irq(adap->msix_info[--msi].vec,
 602                          &s->ofldrxq[ofldqidx].rspq);
 603         while (--ethqidx >= 0)
 604                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
 605         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 606         return err;
 607 }
 608
 609 static void free_msix_queue_irqs(struct adapter *adap)
 610 {
 611         int i, msi = 2;
 612         struct sge *s = &adap->sge;
 613
 614         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 615         for_each_ethrxq(s, i)
 616                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
 617         for_each_ofldrxq(s, i)
 618                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
 619         for_each_rdmarxq(s, i)
 620                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
 621 }
 622
 623 /**
 624  *      write_rss - write the RSS table for a given port
 625  *      @pi: the port
 626  *      @queues: array of queue indices for RSS
 627  *
 628  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 629  *      packets to the Rx queues in @queues.
 630  */
 631 static int write_rss(const struct port_info *pi, const u16 *queues)
 632 {
 633         u16 *rss;
 634         int i, err;
 635         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 636
 637         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 638         if (!rss)
 639                 return -ENOMEM;
 640
 641         /* map the queue indices to queue ids */
 642         for (i = 0; i < pi->rss_size; i++, queues++)
 643                 rss[i] = q[*queues].rspq.abs_id;
 644
 645         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 646                                   pi->rss_size, rss, pi->rss_size);
 647         kfree(rss);
 648         return err;
 649 }
 650
 651 /**
 652  *      setup_rss - configure RSS
 653  *      @adap: the adapter
 654  *
 655  *      Sets up RSS for each port.
 656  */
 657 static int setup_rss(struct adapter *adap)
 658 {
 659         int i, err;
 660
 661         for_each_port(adap, i) {
 662                 const struct port_info *pi = adap2pinfo(adap, i);
 663
 664                 err = write_rss(pi, pi->rss);
 665                 if (err)
 666                         return err;
 667         }
 668         return 0;
 669 }
 670
 671 /*
 672  * Return the channel of the ingress queue with the given qid.
 673  */
 674 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 675 {
 676         qid -= p->ingr_start;
 677         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 678 }
 679
 680 /*
 681  * Wait until all NAPI handlers are descheduled.
 682  */
 683 static void quiesce_rx(struct adapter *adap)
 684 {
 685         int i;
 686
 687         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 688                 struct sge_rspq *q = adap->sge.ingr_map[i];
 689
 690                 if (q && q->handler)
 691                         napi_disable(&q->napi);
 692         }
 693 }
 694
 695 /*
 696  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 697  */
 698 static void enable_rx(struct adapter *adap)
 699 {
 700         int i;
 701
 702         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 703                 struct sge_rspq *q = adap->sge.ingr_map[i];
 704
 705                 if (!q)
 706                         continue;
 707                 if (q->handler)
 708                         napi_enable(&q->napi);
 709                 /* 0-increment GTS to start the timer and enable interrupts */
 710                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 711                              SEINTARM(q->intr_params) |
 712                              INGRESSQID(q->cntxt_id));
 713         }
 714 }
 715
 716 /**
 717  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 718  *      @adap: the adapter
 719  *
 720  *      Determines how many sets of SGE queues to use and initializes them.
 721  *      We support multiple queue sets per port if we have MSI-X, otherwise
 722  *      just one queue set per port.
 723  */
 724 static int setup_sge_queues(struct adapter *adap)
 725 {
 726         int err, msi_idx, i, j;
 727         struct sge *s = &adap->sge;
 728
 729         bitmap_zero(s->starving_fl, MAX_EGRQ);
 730         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 731
 732         if (adap->flags & USING_MSIX)
 733                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 734         else {
 735                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 736                                        NULL, NULL);
 737                 if (err)
 738                         return err;
 739                 msi_idx = -((int)s->intrq.abs_id + 1);
 740         }
 741
 742         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 743                                msi_idx, NULL, fwevtq_handler);
 744         if (err) {
 745 freeout:        t4_free_sge_resources(adap);
 746                 return err;
 747         }
 748
 749         for_each_port(adap, i) {
 750                 struct net_device *dev = adap->port[i];
 751                 struct port_info *pi = netdev_priv(dev);
 752                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 753                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 754
 755                 for (j = 0; j < pi->nqsets; j++, q++) {
 756                         if (msi_idx > 0)
 757                                 msi_idx++;
 758                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 759                                                msi_idx, &q->fl,
 760                                                t4_ethrx_handler);
 761                         if (err)
 762                                 goto freeout;
 763                         q->rspq.idx = j;
 764                         memset(&q->stats, 0, sizeof(q->stats));
 765                 }
 766                 for (j = 0; j < pi->nqsets; j++, t++) {
 767                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 768                                         netdev_get_tx_queue(dev, j),
 769                                         s->fw_evtq.cntxt_id);
 770                         if (err)
 771                                 goto freeout;
 772                 }
 773         }
 774
 775         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 776         for_each_ofldrxq(s, i) {
 777                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 778                 struct net_device *dev = adap->port[i / j];
 779
 780                 if (msi_idx > 0)
 781                         msi_idx++;
 782                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 783                                        &q->fl, uldrx_handler);
 784                 if (err)
 785                         goto freeout;
 786                 memset(&q->stats, 0, sizeof(q->stats));
 787                 s->ofld_rxq[i] = q->rspq.abs_id;
 788                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 789                                             s->fw_evtq.cntxt_id);
 790                 if (err)
 791                         goto freeout;
 792         }
 793
 794         for_each_rdmarxq(s, i) {
 795                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 796
 797                 if (msi_idx > 0)
 798                         msi_idx++;
 799                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 800                                        msi_idx, &q->fl, uldrx_handler);
 801                 if (err)
 802                         goto freeout;
 803                 memset(&q->stats, 0, sizeof(q->stats));
 804                 s->rdma_rxq[i] = q->rspq.abs_id;
 805         }
 806
 807         for_each_port(adap, i) {
 808                 /*
 809                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 810                  * have RDMA queues, and that's the right value.
 811                  */
 812                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 813                                             s->fw_evtq.cntxt_id,
 814                                             s->rdmarxq[i].rspq.cntxt_id);
 815                 if (err)
 816                         goto freeout;
 817         }
 818
 819         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
 820                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
 821                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
 822         return 0;
 823 }
 824
 825 /*
 826  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
 827  * started but failed, and a negative errno if flash load couldn't start.
 828  */
 829 static int upgrade_fw(struct adapter *adap)
 830 {
 831         int ret;
 832         u32 vers;
 833         const struct fw_hdr *hdr;
 834         const struct firmware *fw;
 835         struct device *dev = adap->pdev_dev;
 836
 837         ret = request_firmware(&fw, FW_FNAME, dev);
 838         if (ret < 0) {
 839                 dev_err(dev, "unable to load firmware image " FW_FNAME
 840                         ", error %d\n", ret);
 841                 return ret;
 842         }
 843
 844         hdr = (const struct fw_hdr *)fw->data;
 845         vers = ntohl(hdr->fw_ver);
 846         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
 847                 ret = -EINVAL;              /* wrong major version, won't do */
 848                 goto out;
 849         }
 850
 851         /*
 852          * If the flash FW is unusable or we found something newer, load it.
 853          */
 854         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
 855             vers > adap->params.fw_vers) {
 856                 ret = -t4_load_fw(adap, fw->data, fw->size);
 857                 if (!ret)
 858                         dev_info(dev, "firmware upgraded to version %pI4 from "
 859                                  FW_FNAME "\n", &hdr->fw_ver);
 860         }
 861 out:    release_firmware(fw);
 862         return ret;
 863 }
 864
 865 /*
 866  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 867  * The allocated memory is cleared.
 868  */
 869 void *t4_alloc_mem(size_t size)
 870 {
 871         void *p = kmalloc(size, GFP_KERNEL);
 872
 873         if (!p)
 874                 p = vmalloc(size);
 875         if (p)
 876                 memset(p, 0, size);
 877         return p;
 878 }
 879
 880 /*
 881  * Free memory allocated through alloc_mem().
 882  */
 883 void t4_free_mem(void *addr)
 884 {
 885         if (is_vmalloc_addr(addr))
 886                 vfree(addr);
 887         else
 888                 kfree(addr);
 889 }
 890
 891 static inline int is_offload(const struct adapter *adap)
 892 {
 893         return adap->params.offload;
 894 }
 895
 896 /*
 897  * Implementation of ethtool operations.
 898  */
 899
 900 static u32 get_msglevel(struct net_device *dev)
 901 {
 902         return netdev2adap(dev)->msg_enable;
 903 }
 904
 905 static void set_msglevel(struct net_device *dev, u32 val)
 906 {
 907         netdev2adap(dev)->msg_enable = val;
 908 }
 909
 910 static char stats_strings[][ETH_GSTRING_LEN] = {
 911         "TxOctetsOK         ",
 912         "TxFramesOK         ",
 913         "TxBroadcastFrames  ",
 914         "TxMulticastFrames  ",
 915         "TxUnicastFrames    ",
 916         "TxErrorFrames      ",
 917
 918         "TxFrames64         ",
 919         "TxFrames65To127    ",
 920         "TxFrames128To255   ",
 921         "TxFrames256To511   ",
 922         "TxFrames512To1023  ",
 923         "TxFrames1024To1518 ",
 924         "TxFrames1519ToMax  ",
 925
 926         "TxFramesDropped    ",
 927         "TxPauseFrames      ",
 928         "TxPPP0Frames       ",
 929         "TxPPP1Frames       ",
 930         "TxPPP2Frames       ",
 931         "TxPPP3Frames       ",
 932         "TxPPP4Frames       ",
 933         "TxPPP5Frames       ",
 934         "TxPPP6Frames       ",
 935         "TxPPP7Frames       ",
 936
 937         "RxOctetsOK         ",
 938         "RxFramesOK         ",
 939         "RxBroadcastFrames  ",
 940         "RxMulticastFrames  ",
 941         "RxUnicastFrames    ",
 942
 943         "RxFramesTooLong    ",
 944         "RxJabberErrors     ",
 945         "RxFCSErrors        ",
 946         "RxLengthErrors     ",
 947         "RxSymbolErrors     ",
 948         "RxRuntFrames       ",
 949
 950         "RxFrames64         ",
 951         "RxFrames65To127    ",
 952         "RxFrames128To255   ",
 953         "RxFrames256To511   ",
 954         "RxFrames512To1023  ",
 955         "RxFrames1024To1518 ",
 956         "RxFrames1519ToMax  ",
 957
 958         "RxPauseFrames      ",
 959         "RxPPP0Frames       ",
 960         "RxPPP1Frames       ",
 961         "RxPPP2Frames       ",
 962         "RxPPP3Frames       ",
 963         "RxPPP4Frames       ",
 964         "RxPPP5Frames       ",
 965         "RxPPP6Frames       ",
 966         "RxPPP7Frames       ",
 967
 968         "RxBG0FramesDropped ",
 969         "RxBG1FramesDropped ",
 970         "RxBG2FramesDropped ",
 971         "RxBG3FramesDropped ",
 972         "RxBG0FramesTrunc   ",
 973         "RxBG1FramesTrunc   ",
 974         "RxBG2FramesTrunc   ",
 975         "RxBG3FramesTrunc   ",
 976
 977         "TSO                ",
 978         "TxCsumOffload      ",
 979         "RxCsumGood         ",
 980         "VLANextractions    ",
 981         "VLANinsertions     ",
 982         "GROpackets         ",
 983         "GROmerged          ",
 984 };
 985
 986 static int get_sset_count(struct net_device *dev, int sset)
 987 {
 988         switch (sset) {
 989         case ETH_SS_STATS:
 990                 return ARRAY_SIZE(stats_strings);
 991         default:
 992                 return -EOPNOTSUPP;
 993         }
 994 }
 995
 996 #define T4_REGMAP_SIZE (160 * 1024)
 997
 998 static int get_regs_len(struct net_device *dev)
 999 {
1000         return T4_REGMAP_SIZE;
1001 }
1002
1003 static int get_eeprom_len(struct net_device *dev)
1004 {
1005         return EEPROMSIZE;
1006 }
1007
1008 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1009 {
1010         struct adapter *adapter = netdev2adap(dev);
1011
1012         strcpy(info->driver, KBUILD_MODNAME);
1013         strcpy(info->version, DRV_VERSION);
1014         strcpy(info->bus_info, pci_name(adapter->pdev));
1015
1016         if (!adapter->params.fw_vers)
1017                 strcpy(info->fw_version, "N/A");
1018         else
1019                 snprintf(info->fw_version, sizeof(info->fw_version),
1020                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1021                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1022                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1023                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1024                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1025                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1026                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1027                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1028                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1029 }
1030
1031 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1032 {
1033         if (stringset == ETH_SS_STATS)
1034                 memcpy(data, stats_strings, sizeof(stats_strings));
1035 }
1036
1037 /*
1038  * port stats maintained per queue of the port.  They should be in the same
1039  * order as in stats_strings above.
1040  */
1041 struct queue_port_stats {
1042         u64 tso;
1043         u64 tx_csum;
1044         u64 rx_csum;
1045         u64 vlan_ex;
1046         u64 vlan_ins;
1047         u64 gro_pkts;
1048         u64 gro_merged;
1049 };
1050
1051 static void collect_sge_port_stats(const struct adapter *adap,
1052                 const struct port_info *p, struct queue_port_stats *s)
1053 {
1054         int i;
1055         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1056         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1057
1058         memset(s, 0, sizeof(*s));
1059         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1060                 s->tso += tx->tso;
1061                 s->tx_csum += tx->tx_cso;
1062                 s->rx_csum += rx->stats.rx_cso;
1063                 s->vlan_ex += rx->stats.vlan_ex;
1064                 s->vlan_ins += tx->vlan_ins;
1065                 s->gro_pkts += rx->stats.lro_pkts;
1066                 s->gro_merged += rx->stats.lro_merged;
1067         }
1068 }
1069
1070 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1071                       u64 *data)
1072 {
1073         struct port_info *pi = netdev_priv(dev);
1074         struct adapter *adapter = pi->adapter;
1075
1076         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1077
1078         data += sizeof(struct port_stats) / sizeof(u64);
1079         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1080 }
1081
1082 /*
1083  * Return a version number to identify the type of adapter.  The scheme is:
1084  * - bits 0..9: chip version
1085  * - bits 10..15: chip revision
1086  * - bits 16..23: register dump version
1087  */
1088 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1089 {
1090         return 4 | (ap->params.rev << 10) | (1 << 16);
1091 }
1092
1093 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1094                            unsigned int end)
1095 {
1096         u32 *p = buf + start;
1097
1098         for ( ; start <= end; start += sizeof(u32))
1099                 *p++ = t4_read_reg(ap, start);
1100 }
1101
1102 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1103                      void *buf)
1104 {
1105         static const unsigned int reg_ranges[] = {
1106                 0x1008, 0x1108,
1107                 0x1180, 0x11b4,
1108                 0x11fc, 0x123c,
1109                 0x1300, 0x173c,
1110                 0x1800, 0x18fc,
1111                 0x3000, 0x30d8,
1112                 0x30e0, 0x5924,
1113                 0x5960, 0x59d4,
1114                 0x5a00, 0x5af8,
1115                 0x6000, 0x6098,
1116                 0x6100, 0x6150,
1117                 0x6200, 0x6208,
1118                 0x6240, 0x6248,
1119                 0x6280, 0x6338,
1120                 0x6370, 0x638c,
1121                 0x6400, 0x643c,
1122                 0x6500, 0x6524,
1123                 0x6a00, 0x6a38,
1124                 0x6a60, 0x6a78,
1125                 0x6b00, 0x6b84,
1126                 0x6bf0, 0x6c84,
1127                 0x6cf0, 0x6d84,
1128                 0x6df0, 0x6e84,
1129                 0x6ef0, 0x6f84,
1130                 0x6ff0, 0x7084,
1131                 0x70f0, 0x7184,
1132                 0x71f0, 0x7284,
1133                 0x72f0, 0x7384,
1134                 0x73f0, 0x7450,
1135                 0x7500, 0x7530,
1136                 0x7600, 0x761c,
1137                 0x7680, 0x76cc,
1138                 0x7700, 0x7798,
1139                 0x77c0, 0x77fc,
1140                 0x7900, 0x79fc,
1141                 0x7b00, 0x7c38,
1142                 0x7d00, 0x7efc,
1143                 0x8dc0, 0x8e1c,
1144                 0x8e30, 0x8e78,
1145                 0x8ea0, 0x8f6c,
1146                 0x8fc0, 0x9074,
1147                 0x90fc, 0x90fc,
1148                 0x9400, 0x9458,
1149                 0x9600, 0x96bc,
1150                 0x9800, 0x9808,
1151                 0x9820, 0x983c,
1152                 0x9850, 0x9864,
1153                 0x9c00, 0x9c6c,
1154                 0x9c80, 0x9cec,
1155                 0x9d00, 0x9d6c,
1156                 0x9d80, 0x9dec,
1157                 0x9e00, 0x9e6c,
1158                 0x9e80, 0x9eec,
1159                 0x9f00, 0x9f6c,
1160                 0x9f80, 0x9fec,
1161                 0xd004, 0xd03c,
1162                 0xdfc0, 0xdfe0,
1163                 0xe000, 0xea7c,
1164                 0xf000, 0x11190,
1165                 0x19040, 0x1906c,
1166                 0x19078, 0x19080,
1167                 0x1908c, 0x19124,
1168                 0x19150, 0x191b0,
1169                 0x191d0, 0x191e8,
1170                 0x19238, 0x1924c,
1171                 0x193f8, 0x19474,
1172                 0x19490, 0x194f8,
1173                 0x19800, 0x19f30,
1174                 0x1a000, 0x1a06c,
1175                 0x1a0b0, 0x1a120,
1176                 0x1a128, 0x1a138,
1177                 0x1a190, 0x1a1c4,
1178                 0x1a1fc, 0x1a1fc,
1179                 0x1e040, 0x1e04c,
1180                 0x1e284, 0x1e28c,
1181                 0x1e2c0, 0x1e2c0,
1182                 0x1e2e0, 0x1e2e0,
1183                 0x1e300, 0x1e384,
1184                 0x1e3c0, 0x1e3c8,
1185                 0x1e440, 0x1e44c,
1186                 0x1e684, 0x1e68c,
1187                 0x1e6c0, 0x1e6c0,
1188                 0x1e6e0, 0x1e6e0,
1189                 0x1e700, 0x1e784,
1190                 0x1e7c0, 0x1e7c8,
1191                 0x1e840, 0x1e84c,
1192                 0x1ea84, 0x1ea8c,
1193                 0x1eac0, 0x1eac0,
1194                 0x1eae0, 0x1eae0,
1195                 0x1eb00, 0x1eb84,
1196                 0x1ebc0, 0x1ebc8,
1197                 0x1ec40, 0x1ec4c,
1198                 0x1ee84, 0x1ee8c,
1199                 0x1eec0, 0x1eec0,
1200                 0x1eee0, 0x1eee0,
1201                 0x1ef00, 0x1ef84,
1202                 0x1efc0, 0x1efc8,
1203                 0x1f040, 0x1f04c,
1204                 0x1f284, 0x1f28c,
1205                 0x1f2c0, 0x1f2c0,
1206                 0x1f2e0, 0x1f2e0,
1207                 0x1f300, 0x1f384,
1208                 0x1f3c0, 0x1f3c8,
1209                 0x1f440, 0x1f44c,
1210                 0x1f684, 0x1f68c,
1211                 0x1f6c0, 0x1f6c0,
1212                 0x1f6e0, 0x1f6e0,
1213                 0x1f700, 0x1f784,
1214                 0x1f7c0, 0x1f7c8,
1215                 0x1f840, 0x1f84c,
1216                 0x1fa84, 0x1fa8c,
1217                 0x1fac0, 0x1fac0,
1218                 0x1fae0, 0x1fae0,
1219                 0x1fb00, 0x1fb84,
1220                 0x1fbc0, 0x1fbc8,
1221                 0x1fc40, 0x1fc4c,
1222                 0x1fe84, 0x1fe8c,
1223                 0x1fec0, 0x1fec0,
1224                 0x1fee0, 0x1fee0,
1225                 0x1ff00, 0x1ff84,
1226                 0x1ffc0, 0x1ffc8,
1227                 0x20000, 0x2002c,
1228                 0x20100, 0x2013c,
1229                 0x20190, 0x201c8,
1230                 0x20200, 0x20318,
1231                 0x20400, 0x20528,
1232                 0x20540, 0x20614,
1233                 0x21000, 0x21040,
1234                 0x2104c, 0x21060,
1235                 0x210c0, 0x210ec,
1236                 0x21200, 0x21268,
1237                 0x21270, 0x21284,
1238                 0x212fc, 0x21388,
1239                 0x21400, 0x21404,
1240                 0x21500, 0x21518,
1241                 0x2152c, 0x2153c,
1242                 0x21550, 0x21554,
1243                 0x21600, 0x21600,
1244                 0x21608, 0x21628,
1245                 0x21630, 0x2163c,
1246                 0x21700, 0x2171c,
1247                 0x21780, 0x2178c,
1248                 0x21800, 0x21c38,
1249                 0x21c80, 0x21d7c,
1250                 0x21e00, 0x21e04,
1251                 0x22000, 0x2202c,
1252                 0x22100, 0x2213c,
1253                 0x22190, 0x221c8,
1254                 0x22200, 0x22318,
1255                 0x22400, 0x22528,
1256                 0x22540, 0x22614,
1257                 0x23000, 0x23040,
1258                 0x2304c, 0x23060,
1259                 0x230c0, 0x230ec,
1260                 0x23200, 0x23268,
1261                 0x23270, 0x23284,
1262                 0x232fc, 0x23388,
1263                 0x23400, 0x23404,
1264                 0x23500, 0x23518,
1265                 0x2352c, 0x2353c,
1266                 0x23550, 0x23554,
1267                 0x23600, 0x23600,
1268                 0x23608, 0x23628,
1269                 0x23630, 0x2363c,
1270                 0x23700, 0x2371c,
1271                 0x23780, 0x2378c,
1272                 0x23800, 0x23c38,
1273                 0x23c80, 0x23d7c,
1274                 0x23e00, 0x23e04,
1275                 0x24000, 0x2402c,
1276                 0x24100, 0x2413c,
1277                 0x24190, 0x241c8,
1278                 0x24200, 0x24318,
1279                 0x24400, 0x24528,
1280                 0x24540, 0x24614,
1281                 0x25000, 0x25040,
1282                 0x2504c, 0x25060,
1283                 0x250c0, 0x250ec,
1284                 0x25200, 0x25268,
1285                 0x25270, 0x25284,
1286                 0x252fc, 0x25388,
1287                 0x25400, 0x25404,
1288                 0x25500, 0x25518,
1289                 0x2552c, 0x2553c,
1290                 0x25550, 0x25554,
1291                 0x25600, 0x25600,
1292                 0x25608, 0x25628,
1293                 0x25630, 0x2563c,
1294                 0x25700, 0x2571c,
1295                 0x25780, 0x2578c,
1296                 0x25800, 0x25c38,
1297                 0x25c80, 0x25d7c,
1298                 0x25e00, 0x25e04,
1299                 0x26000, 0x2602c,
1300                 0x26100, 0x2613c,
1301                 0x26190, 0x261c8,
1302                 0x26200, 0x26318,
1303                 0x26400, 0x26528,
1304                 0x26540, 0x26614,
1305                 0x27000, 0x27040,
1306                 0x2704c, 0x27060,
1307                 0x270c0, 0x270ec,
1308                 0x27200, 0x27268,
1309                 0x27270, 0x27284,
1310                 0x272fc, 0x27388,
1311                 0x27400, 0x27404,
1312                 0x27500, 0x27518,
1313                 0x2752c, 0x2753c,
1314                 0x27550, 0x27554,
1315                 0x27600, 0x27600,
1316                 0x27608, 0x27628,
1317                 0x27630, 0x2763c,
1318                 0x27700, 0x2771c,
1319                 0x27780, 0x2778c,
1320                 0x27800, 0x27c38,
1321                 0x27c80, 0x27d7c,
1322                 0x27e00, 0x27e04
1323         };
1324
1325         int i;
1326         struct adapter *ap = netdev2adap(dev);
1327
1328         regs->version = mk_adap_vers(ap);
1329
1330         memset(buf, 0, T4_REGMAP_SIZE);
1331         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1332                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1333 }
1334
1335 static int restart_autoneg(struct net_device *dev)
1336 {
1337         struct port_info *p = netdev_priv(dev);
1338
1339         if (!netif_running(dev))
1340                 return -EAGAIN;
1341         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1342                 return -EINVAL;
1343         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1344         return 0;
1345 }
1346
1347 static int identify_port(struct net_device *dev, u32 data)
1348 {
1349         struct adapter *adap = netdev2adap(dev);
1350
1351         if (data == 0)
1352                 data = 2;     /* default to 2 seconds */
1353
1354         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid,
1355                                 data * 5);
1356 }
1357
1358 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1359 {
1360         unsigned int v = 0;
1361
1362         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1363             type == FW_PORT_TYPE_BT_XAUI) {
1364                 v |= SUPPORTED_TP;
1365                 if (caps & FW_PORT_CAP_SPEED_100M)
1366                         v |= SUPPORTED_100baseT_Full;
1367                 if (caps & FW_PORT_CAP_SPEED_1G)
1368                         v |= SUPPORTED_1000baseT_Full;
1369                 if (caps & FW_PORT_CAP_SPEED_10G)
1370                         v |= SUPPORTED_10000baseT_Full;
1371         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1372                 v |= SUPPORTED_Backplane;
1373                 if (caps & FW_PORT_CAP_SPEED_1G)
1374                         v |= SUPPORTED_1000baseKX_Full;
1375                 if (caps & FW_PORT_CAP_SPEED_10G)
1376                         v |= SUPPORTED_10000baseKX4_Full;
1377         } else if (type == FW_PORT_TYPE_KR)
1378                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1379         else if (type == FW_PORT_TYPE_BP_AP)
1380                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1381         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1382                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1383                 v |= SUPPORTED_FIBRE;
1384
1385         if (caps & FW_PORT_CAP_ANEG)
1386                 v |= SUPPORTED_Autoneg;
1387         return v;
1388 }
1389
1390 static unsigned int to_fw_linkcaps(unsigned int caps)
1391 {
1392         unsigned int v = 0;
1393
1394         if (caps & ADVERTISED_100baseT_Full)
1395                 v |= FW_PORT_CAP_SPEED_100M;
1396         if (caps & ADVERTISED_1000baseT_Full)
1397                 v |= FW_PORT_CAP_SPEED_1G;
1398         if (caps & ADVERTISED_10000baseT_Full)
1399                 v |= FW_PORT_CAP_SPEED_10G;
1400         return v;
1401 }
1402
1403 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1404 {
1405         const struct port_info *p = netdev_priv(dev);
1406
1407         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1408             p->port_type == FW_PORT_TYPE_BT_XFI ||
1409             p->port_type == FW_PORT_TYPE_BT_XAUI)
1410                 cmd->port = PORT_TP;
1411         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1412                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1413                 cmd->port = PORT_FIBRE;
1414         else if (p->port_type == FW_PORT_TYPE_SFP) {
1415                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1416                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1417                         cmd->port = PORT_DA;
1418                 else
1419                         cmd->port = PORT_FIBRE;
1420         } else
1421                 cmd->port = PORT_OTHER;
1422
1423         if (p->mdio_addr >= 0) {
1424                 cmd->phy_address = p->mdio_addr;
1425                 cmd->transceiver = XCVR_EXTERNAL;
1426                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1427                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1428         } else {
1429                 cmd->phy_address = 0;  /* not really, but no better option */
1430                 cmd->transceiver = XCVR_INTERNAL;
1431                 cmd->mdio_support = 0;
1432         }
1433
1434         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1435         cmd->advertising = from_fw_linkcaps(p->port_type,
1436                                             p->link_cfg.advertising);
1437         cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1438         cmd->duplex = DUPLEX_FULL;
1439         cmd->autoneg = p->link_cfg.autoneg;
1440         cmd->maxtxpkt = 0;
1441         cmd->maxrxpkt = 0;
1442         return 0;
1443 }
1444
1445 static unsigned int speed_to_caps(int speed)
1446 {
1447         if (speed == SPEED_100)
1448                 return FW_PORT_CAP_SPEED_100M;
1449         if (speed == SPEED_1000)
1450                 return FW_PORT_CAP_SPEED_1G;
1451         if (speed == SPEED_10000)
1452                 return FW_PORT_CAP_SPEED_10G;
1453         return 0;
1454 }
1455
1456 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1457 {
1458         unsigned int cap;
1459         struct port_info *p = netdev_priv(dev);
1460         struct link_config *lc = &p->link_cfg;
1461
1462         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1463                 return -EINVAL;
1464
1465         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1466                 /*
1467                  * PHY offers a single speed.  See if that's what's
1468                  * being requested.
1469                  */
1470                 if (cmd->autoneg == AUTONEG_DISABLE &&
1471                     (lc->supported & speed_to_caps(cmd->speed)))
1472                                 return 0;
1473                 return -EINVAL;
1474         }
1475
1476         if (cmd->autoneg == AUTONEG_DISABLE) {
1477                 cap = speed_to_caps(cmd->speed);
1478
1479                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1480                     cmd->speed == SPEED_10000)
1481                         return -EINVAL;
1482                 lc->requested_speed = cap;
1483                 lc->advertising = 0;
1484         } else {
1485                 cap = to_fw_linkcaps(cmd->advertising);
1486                 if (!(lc->supported & cap))
1487                         return -EINVAL;
1488                 lc->requested_speed = 0;
1489                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1490         }
1491         lc->autoneg = cmd->autoneg;
1492
1493         if (netif_running(dev))
1494                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1495                                      lc);
1496         return 0;
1497 }
1498
1499 static void get_pauseparam(struct net_device *dev,
1500                            struct ethtool_pauseparam *epause)
1501 {
1502         struct port_info *p = netdev_priv(dev);
1503
1504         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1505         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1506         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1507 }
1508
1509 static int set_pauseparam(struct net_device *dev,
1510                           struct ethtool_pauseparam *epause)
1511 {
1512         struct port_info *p = netdev_priv(dev);
1513         struct link_config *lc = &p->link_cfg;
1514
1515         if (epause->autoneg == AUTONEG_DISABLE)
1516                 lc->requested_fc = 0;
1517         else if (lc->supported & FW_PORT_CAP_ANEG)
1518                 lc->requested_fc = PAUSE_AUTONEG;
1519         else
1520                 return -EINVAL;
1521
1522         if (epause->rx_pause)
1523                 lc->requested_fc |= PAUSE_RX;
1524         if (epause->tx_pause)
1525                 lc->requested_fc |= PAUSE_TX;
1526         if (netif_running(dev))
1527                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1528                                      lc);
1529         return 0;
1530 }
1531
1532 static u32 get_rx_csum(struct net_device *dev)
1533 {
1534         struct port_info *p = netdev_priv(dev);
1535
1536         return p->rx_offload & RX_CSO;
1537 }
1538
1539 static int set_rx_csum(struct net_device *dev, u32 data)
1540 {
1541         struct port_info *p = netdev_priv(dev);
1542
1543         if (data)
1544                 p->rx_offload |= RX_CSO;
1545         else
1546                 p->rx_offload &= ~RX_CSO;
1547         return 0;
1548 }
1549
1550 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1551 {
1552         const struct port_info *pi = netdev_priv(dev);
1553         const struct sge *s = &pi->adapter->sge;
1554
1555         e->rx_max_pending = MAX_RX_BUFFERS;
1556         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1557         e->rx_jumbo_max_pending = 0;
1558         e->tx_max_pending = MAX_TXQ_ENTRIES;
1559
1560         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1561         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1562         e->rx_jumbo_pending = 0;
1563         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1564 }
1565
1566 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1567 {
1568         int i;
1569         const struct port_info *pi = netdev_priv(dev);
1570         struct adapter *adapter = pi->adapter;
1571         struct sge *s = &adapter->sge;
1572
1573         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1574             e->tx_pending > MAX_TXQ_ENTRIES ||
1575             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1576             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1577             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1578                 return -EINVAL;
1579
1580         if (adapter->flags & FULL_INIT_DONE)
1581                 return -EBUSY;
1582
1583         for (i = 0; i < pi->nqsets; ++i) {
1584                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1585                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1586                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1587         }
1588         return 0;
1589 }
1590
1591 static int closest_timer(const struct sge *s, int time)
1592 {
1593         int i, delta, match = 0, min_delta = INT_MAX;
1594
1595         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1596                 delta = time - s->timer_val[i];
1597                 if (delta < 0)
1598                         delta = -delta;
1599                 if (delta < min_delta) {
1600                         min_delta = delta;
1601                         match = i;
1602                 }
1603         }
1604         return match;
1605 }
1606
1607 static int closest_thres(const struct sge *s, int thres)
1608 {
1609         int i, delta, match = 0, min_delta = INT_MAX;
1610
1611         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1612                 delta = thres - s->counter_val[i];
1613                 if (delta < 0)
1614                         delta = -delta;
1615                 if (delta < min_delta) {
1616                         min_delta = delta;
1617                         match = i;
1618                 }
1619         }
1620         return match;
1621 }
1622
1623 /*
1624  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1625  */
1626 static unsigned int qtimer_val(const struct adapter *adap,
1627                                const struct sge_rspq *q)
1628 {
1629         unsigned int idx = q->intr_params >> 1;
1630
1631         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1632 }
1633
1634 /**
1635  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1636  *      @adap: the adapter
1637  *      @q: the Rx queue
1638  *      @us: the hold-off time in us, or 0 to disable timer
1639  *      @cnt: the hold-off packet count, or 0 to disable counter
1640  *
1641  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1642  *      one of the two needs to be enabled for the queue to generate interrupts.
1643  */
1644 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1645                                unsigned int us, unsigned int cnt)
1646 {
1647         if ((us | cnt) == 0)
1648                 cnt = 1;
1649
1650         if (cnt) {
1651                 int err;
1652                 u32 v, new_idx;
1653
1654                 new_idx = closest_thres(&adap->sge, cnt);
1655                 if (q->desc && q->pktcnt_idx != new_idx) {
1656                         /* the queue has already been created, update it */
1657                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1658                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1659                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1660                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1661                                             &new_idx);
1662                         if (err)
1663                                 return err;
1664                 }
1665                 q->pktcnt_idx = new_idx;
1666         }
1667
1668         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1669         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1670         return 0;
1671 }
1672
1673 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1674 {
1675         const struct port_info *pi = netdev_priv(dev);
1676         struct adapter *adap = pi->adapter;
1677
1678         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1679                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1680 }
1681
1682 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1683 {
1684         const struct port_info *pi = netdev_priv(dev);
1685         const struct adapter *adap = pi->adapter;
1686         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1687
1688         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1689         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1690                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1691         return 0;
1692 }
1693
1694 /**
1695  *      eeprom_ptov - translate a physical EEPROM address to virtual
1696  *      @phys_addr: the physical EEPROM address
1697  *      @fn: the PCI function number
1698  *      @sz: size of function-specific area
1699  *
1700  *      Translate a physical EEPROM address to virtual.  The first 1K is
1701  *      accessed through virtual addresses starting at 31K, the rest is
1702  *      accessed through virtual addresses starting at 0.
1703  *
1704  *      The mapping is as follows:
1705  *      [0..1K) -> [31K..32K)
1706  *      [1K..1K+A) -> [31K-A..31K)
1707  *      [1K+A..ES) -> [0..ES-A-1K)
1708  *
1709  *      where A = @fn * @sz, and ES = EEPROM size.
1710  */
1711 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1712 {
1713         fn *= sz;
1714         if (phys_addr < 1024)
1715                 return phys_addr + (31 << 10);
1716         if (phys_addr < 1024 + fn)
1717                 return 31744 - fn + phys_addr - 1024;
1718         if (phys_addr < EEPROMSIZE)
1719                 return phys_addr - 1024 - fn;
1720         return -EINVAL;
1721 }
1722
1723 /*
1724  * The next two routines implement eeprom read/write from physical addresses.
1725  */
1726 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1727 {
1728         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1729
1730         if (vaddr >= 0)
1731                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1732         return vaddr < 0 ? vaddr : 0;
1733 }
1734
1735 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1736 {
1737         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1738
1739         if (vaddr >= 0)
1740                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1741         return vaddr < 0 ? vaddr : 0;
1742 }
1743
1744 #define EEPROM_MAGIC 0x38E2F10C
1745
1746 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1747                       u8 *data)
1748 {
1749         int i, err = 0;
1750         struct adapter *adapter = netdev2adap(dev);
1751
1752         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1753         if (!buf)
1754                 return -ENOMEM;
1755
1756         e->magic = EEPROM_MAGIC;
1757         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1758                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1759
1760         if (!err)
1761                 memcpy(data, buf + e->offset, e->len);
1762         kfree(buf);
1763         return err;
1764 }
1765
1766 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1767                       u8 *data)
1768 {
1769         u8 *buf;
1770         int err = 0;
1771         u32 aligned_offset, aligned_len, *p;
1772         struct adapter *adapter = netdev2adap(dev);
1773
1774         if (eeprom->magic != EEPROM_MAGIC)
1775                 return -EINVAL;
1776
1777         aligned_offset = eeprom->offset & ~3;
1778         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1779
1780         if (adapter->fn > 0) {
1781                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1782
1783                 if (aligned_offset < start ||
1784                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
1785                         return -EPERM;
1786         }
1787
1788         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1789                 /*
1790                  * RMW possibly needed for first or last words.
1791                  */
1792                 buf = kmalloc(aligned_len, GFP_KERNEL);
1793                 if (!buf)
1794                         return -ENOMEM;
1795                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1796                 if (!err && aligned_len > 4)
1797                         err = eeprom_rd_phys(adapter,
1798                                              aligned_offset + aligned_len - 4,
1799                                              (u32 *)&buf[aligned_len - 4]);
1800                 if (err)
1801                         goto out;
1802                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1803         } else
1804                 buf = data;
1805
1806         err = t4_seeprom_wp(adapter, false);
1807         if (err)
1808                 goto out;
1809
1810         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1811                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1812                 aligned_offset += 4;
1813         }
1814
1815         if (!err)
1816                 err = t4_seeprom_wp(adapter, true);
1817 out:
1818         if (buf != data)
1819                 kfree(buf);
1820         return err;
1821 }
1822
1823 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1824 {
1825         int ret;
1826         const struct firmware *fw;
1827         struct adapter *adap = netdev2adap(netdev);
1828
1829         ef->data[sizeof(ef->data) - 1] = '\0';
1830         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1831         if (ret < 0)
1832                 return ret;
1833
1834         ret = t4_load_fw(adap, fw->data, fw->size);
1835         release_firmware(fw);
1836         if (!ret)
1837                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1838         return ret;
1839 }
1840
1841 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1842 #define BCAST_CRC 0xa0ccc1a6
1843
1844 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1845 {
1846         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1847         wol->wolopts = netdev2adap(dev)->wol;
1848         memset(&wol->sopass, 0, sizeof(wol->sopass));
1849 }
1850
1851 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1852 {
1853         int err = 0;
1854         struct port_info *pi = netdev_priv(dev);
1855
1856         if (wol->wolopts & ~WOL_SUPPORTED)
1857                 return -EINVAL;
1858         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1859                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1860         if (wol->wolopts & WAKE_BCAST) {
1861                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1862                                         ~0ULL, 0, false);
1863                 if (!err)
1864                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1865                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1866         } else
1867                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1868         return err;
1869 }
1870
1871 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1872
1873 static int set_tso(struct net_device *dev, u32 value)
1874 {
1875         if (value)
1876                 dev->features |= TSO_FLAGS;
1877         else
1878                 dev->features &= ~TSO_FLAGS;
1879         return 0;
1880 }
1881
1882 static int set_flags(struct net_device *dev, u32 flags)
1883 {
1884         return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
1885 }
1886
1887 static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1888 {
1889         const struct port_info *pi = netdev_priv(dev);
1890         unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1891
1892         p->size = pi->rss_size;
1893         while (n--)
1894                 p->ring_index[n] = pi->rss[n];
1895         return 0;
1896 }
1897
1898 static int set_rss_table(struct net_device *dev,
1899                          const struct ethtool_rxfh_indir *p)
1900 {
1901         unsigned int i;
1902         struct port_info *pi = netdev_priv(dev);
1903
1904         if (p->size != pi->rss_size)
1905                 return -EINVAL;
1906         for (i = 0; i < p->size; i++)
1907                 if (p->ring_index[i] >= pi->nqsets)
1908                         return -EINVAL;
1909         for (i = 0; i < p->size; i++)
1910                 pi->rss[i] = p->ring_index[i];
1911         if (pi->adapter->flags & FULL_INIT_DONE)
1912                 return write_rss(pi, pi->rss);
1913         return 0;
1914 }
1915
1916 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1917                      void *rules)
1918 {
1919         const struct port_info *pi = netdev_priv(dev);
1920
1921         switch (info->cmd) {
1922         case ETHTOOL_GRXFH: {
1923                 unsigned int v = pi->rss_mode;
1924
1925                 info->data = 0;
1926                 switch (info->flow_type) {
1927                 case TCP_V4_FLOW:
1928                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1929                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1930                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1931                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1932                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1933                         break;
1934                 case UDP_V4_FLOW:
1935                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1936                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1937                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1938                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1939                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1940                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1941                         break;
1942                 case SCTP_V4_FLOW:
1943                 case AH_ESP_V4_FLOW:
1944                 case IPV4_FLOW:
1945                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1946                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1947                         break;
1948                 case TCP_V6_FLOW:
1949                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1950                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1951                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1952                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1953                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1954                         break;
1955                 case UDP_V6_FLOW:
1956                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1957                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1958                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1959                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1960                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1961                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1962                         break;
1963                 case SCTP_V6_FLOW:
1964                 case AH_ESP_V6_FLOW:
1965                 case IPV6_FLOW:
1966                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1967                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1968                         break;
1969                 }
1970                 return 0;
1971         }
1972         case ETHTOOL_GRXRINGS:
1973                 info->data = pi->nqsets;
1974                 return 0;
1975         }
1976         return -EOPNOTSUPP;
1977 }
1978
1979 static struct ethtool_ops cxgb_ethtool_ops = {
1980         .get_settings      = get_settings,
1981         .set_settings      = set_settings,
1982         .get_drvinfo       = get_drvinfo,
1983         .get_msglevel      = get_msglevel,
1984         .set_msglevel      = set_msglevel,
1985         .get_ringparam     = get_sge_param,
1986         .set_ringparam     = set_sge_param,
1987         .get_coalesce      = get_coalesce,
1988         .set_coalesce      = set_coalesce,
1989         .get_eeprom_len    = get_eeprom_len,
1990         .get_eeprom        = get_eeprom,
1991         .set_eeprom        = set_eeprom,
1992         .get_pauseparam    = get_pauseparam,
1993         .set_pauseparam    = set_pauseparam,
1994         .get_rx_csum       = get_rx_csum,
1995         .set_rx_csum       = set_rx_csum,
1996         .set_tx_csum       = ethtool_op_set_tx_ipv6_csum,
1997         .set_sg            = ethtool_op_set_sg,
1998         .get_link          = ethtool_op_get_link,
1999         .get_strings       = get_strings,
2000         .phys_id           = identify_port,
2001         .nway_reset        = restart_autoneg,
2002         .get_sset_count    = get_sset_count,
2003         .get_ethtool_stats = get_stats,
2004         .get_regs_len      = get_regs_len,
2005         .get_regs          = get_regs,
2006         .get_wol           = get_wol,
2007         .set_wol           = set_wol,
2008         .set_tso           = set_tso,
2009         .set_flags         = set_flags,
2010         .get_rxnfc         = get_rxnfc,
2011         .get_rxfh_indir    = get_rss_table,
2012         .set_rxfh_indir    = set_rss_table,
2013         .flash_device      = set_flash,
2014 };
2015
2016 /*
2017  * debugfs support
2018  */
2019
2020 static int mem_open(struct inode *inode, struct file *file)
2021 {
2022         file->private_data = inode->i_private;
2023         return 0;
2024 }
2025
2026 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2027                         loff_t *ppos)
2028 {
2029         loff_t pos = *ppos;
2030         loff_t avail = file->f_path.dentry->d_inode->i_size;
2031         unsigned int mem = (uintptr_t)file->private_data & 3;
2032         struct adapter *adap = file->private_data - mem;
2033
2034         if (pos < 0)
2035                 return -EINVAL;
2036         if (pos >= avail)
2037                 return 0;
2038         if (count > avail - pos)
2039                 count = avail - pos;
2040
2041         while (count) {
2042                 size_t len;
2043                 int ret, ofst;
2044                 __be32 data[16];
2045
2046                 if (mem == MEM_MC)
2047                         ret = t4_mc_read(adap, pos, data, NULL);
2048                 else
2049                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2050                 if (ret)
2051                         return ret;
2052
2053                 ofst = pos % sizeof(data);
2054                 len = min(count, sizeof(data) - ofst);
2055                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2056                         return -EFAULT;
2057
2058                 buf += len;
2059                 pos += len;
2060                 count -= len;
2061         }
2062         count = pos - *ppos;
2063         *ppos = pos;
2064         return count;
2065 }
2066
2067 static const struct file_operations mem_debugfs_fops = {
2068         .owner   = THIS_MODULE,
2069         .open    = mem_open,
2070         .read    = mem_read,
2071 };
2072
2073 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2074                                       unsigned int idx, unsigned int size_mb)
2075 {
2076         struct dentry *de;
2077
2078         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2079                                  (void *)adap + idx, &mem_debugfs_fops);
2080         if (de && de->d_inode)
2081                 de->d_inode->i_size = size_mb << 20;
2082 }
2083
2084 static int __devinit setup_debugfs(struct adapter *adap)
2085 {
2086         int i;
2087
2088         if (IS_ERR_OR_NULL(adap->debugfs_root))
2089                 return -1;
2090
2091         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2092         if (i & EDRAM0_ENABLE)
2093                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2094         if (i & EDRAM1_ENABLE)
2095                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2096         if (i & EXT_MEM_ENABLE)
2097                 add_debugfs_mem(adap, "mc", MEM_MC,
2098                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2099         if (adap->l2t)
2100                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2101                                     &t4_l2t_fops);
2102         return 0;
2103 }
2104
2105 /*
2106  * upper-layer driver support
2107  */
2108
2109 /*
2110  * Allocate an active-open TID and set it to the supplied value.
2111  */
2112 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2113 {
2114         int atid = -1;
2115
2116         spin_lock_bh(&t->atid_lock);
2117         if (t->afree) {
2118                 union aopen_entry *p = t->afree;
2119
2120                 atid = p - t->atid_tab;
2121                 t->afree = p->next;
2122                 p->data = data;
2123                 t->atids_in_use++;
2124         }
2125         spin_unlock_bh(&t->atid_lock);
2126         return atid;
2127 }
2128 EXPORT_SYMBOL(cxgb4_alloc_atid);
2129
2130 /*
2131  * Release an active-open TID.
2132  */
2133 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2134 {
2135         union aopen_entry *p = &t->atid_tab[atid];
2136
2137         spin_lock_bh(&t->atid_lock);
2138         p->next = t->afree;
2139         t->afree = p;
2140         t->atids_in_use--;
2141         spin_unlock_bh(&t->atid_lock);
2142 }
2143 EXPORT_SYMBOL(cxgb4_free_atid);
2144
2145 /*
2146  * Allocate a server TID and set it to the supplied value.
2147  */
2148 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2149 {
2150         int stid;
2151
2152         spin_lock_bh(&t->stid_lock);
2153         if (family == PF_INET) {
2154                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2155                 if (stid < t->nstids)
2156                         __set_bit(stid, t->stid_bmap);
2157                 else
2158                         stid = -1;
2159         } else {
2160                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2161                 if (stid < 0)
2162                         stid = -1;
2163         }
2164         if (stid >= 0) {
2165                 t->stid_tab[stid].data = data;
2166                 stid += t->stid_base;
2167                 t->stids_in_use++;
2168         }
2169         spin_unlock_bh(&t->stid_lock);
2170         return stid;
2171 }
2172 EXPORT_SYMBOL(cxgb4_alloc_stid);
2173
2174 /*
2175  * Release a server TID.
2176  */
2177 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2178 {
2179         stid -= t->stid_base;
2180         spin_lock_bh(&t->stid_lock);
2181         if (family == PF_INET)
2182                 __clear_bit(stid, t->stid_bmap);
2183         else
2184                 bitmap_release_region(t->stid_bmap, stid, 2);
2185         t->stid_tab[stid].data = NULL;
2186         t->stids_in_use--;
2187         spin_unlock_bh(&t->stid_lock);
2188 }
2189 EXPORT_SYMBOL(cxgb4_free_stid);
2190
2191 /*
2192  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2193  */
2194 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2195                            unsigned int tid)
2196 {
2197         struct cpl_tid_release *req;
2198
2199         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2200         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2201         INIT_TP_WR(req, tid);
2202         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2203 }
2204
2205 /*
2206  * Queue a TID release request and if necessary schedule a work queue to
2207  * process it.
2208  */
2209 void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2210                              unsigned int tid)
2211 {
2212         void **p = &t->tid_tab[tid];
2213         struct adapter *adap = container_of(t, struct adapter, tids);
2214
2215         spin_lock_bh(&adap->tid_release_lock);
2216         *p = adap->tid_release_head;
2217         /* Low 2 bits encode the Tx channel number */
2218         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2219         if (!adap->tid_release_task_busy) {
2220                 adap->tid_release_task_busy = true;
2221                 schedule_work(&adap->tid_release_task);
2222         }
2223         spin_unlock_bh(&adap->tid_release_lock);
2224 }
2225 EXPORT_SYMBOL(cxgb4_queue_tid_release);
2226
2227 /*
2228  * Process the list of pending TID release requests.
2229  */
2230 static void process_tid_release_list(struct work_struct *work)
2231 {
2232         struct sk_buff *skb;
2233         struct adapter *adap;
2234
2235         adap = container_of(work, struct adapter, tid_release_task);
2236
2237         spin_lock_bh(&adap->tid_release_lock);
2238         while (adap->tid_release_head) {
2239                 void **p = adap->tid_release_head;
2240                 unsigned int chan = (uintptr_t)p & 3;
2241                 p = (void *)p - chan;
2242
2243                 adap->tid_release_head = *p;
2244                 *p = NULL;
2245                 spin_unlock_bh(&adap->tid_release_lock);
2246
2247                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2248                                          GFP_KERNEL)))
2249                         schedule_timeout_uninterruptible(1);
2250
2251                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2252                 t4_ofld_send(adap, skb);
2253                 spin_lock_bh(&adap->tid_release_lock);
2254         }
2255         adap->tid_release_task_busy = false;
2256         spin_unlock_bh(&adap->tid_release_lock);
2257 }
2258
2259 /*
2260  * Release a TID and inform HW.  If we are unable to allocate the release
2261  * message we defer to a work queue.
2262  */
2263 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2264 {
2265         void *old;
2266         struct sk_buff *skb;
2267         struct adapter *adap = container_of(t, struct adapter, tids);
2268
2269         old = t->tid_tab[tid];
2270         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2271         if (likely(skb)) {
2272                 t->tid_tab[tid] = NULL;
2273                 mk_tid_release(skb, chan, tid);
2274                 t4_ofld_send(adap, skb);
2275         } else
2276                 cxgb4_queue_tid_release(t, chan, tid);
2277         if (old)
2278                 atomic_dec(&t->tids_in_use);
2279 }
2280 EXPORT_SYMBOL(cxgb4_remove_tid);
2281
2282 /*
2283  * Allocate and initialize the TID tables.  Returns 0 on success.
2284  */
2285 static int tid_init(struct tid_info *t)
2286 {
2287         size_t size;
2288         unsigned int natids = t->natids;
2289
2290         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2291                t->nstids * sizeof(*t->stid_tab) +
2292                BITS_TO_LONGS(t->nstids) * sizeof(long);
2293         t->tid_tab = t4_alloc_mem(size);
2294         if (!t->tid_tab)
2295                 return -ENOMEM;
2296
2297         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2298         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2299         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2300         spin_lock_init(&t->stid_lock);
2301         spin_lock_init(&t->atid_lock);
2302
2303         t->stids_in_use = 0;
2304         t->afree = NULL;
2305         t->atids_in_use = 0;
2306         atomic_set(&t->tids_in_use, 0);
2307
2308         /* Setup the free list for atid_tab and clear the stid bitmap. */
2309         if (natids) {
2310                 while (--natids)
2311                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2312                 t->afree = t->atid_tab;
2313         }
2314         bitmap_zero(t->stid_bmap, t->nstids);
2315         return 0;
2316 }
2317
2318 /**
2319  *      cxgb4_create_server - create an IP server
2320  *      @dev: the device
2321  *      @stid: the server TID
2322  *      @sip: local IP address to bind server to
2323  *      @sport: the server's TCP port
2324  *      @queue: queue to direct messages from this server to
2325  *
2326  *      Create an IP server for the given port and address.
2327  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2328  */
2329 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2330                         __be32 sip, __be16 sport, unsigned int queue)
2331 {
2332         unsigned int chan;
2333         struct sk_buff *skb;
2334         struct adapter *adap;
2335         struct cpl_pass_open_req *req;
2336
2337         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2338         if (!skb)
2339                 return -ENOMEM;
2340
2341         adap = netdev2adap(dev);
2342         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2343         INIT_TP_WR(req, 0);
2344         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2345         req->local_port = sport;
2346         req->peer_port = htons(0);
2347         req->local_ip = sip;
2348         req->peer_ip = htonl(0);
2349         chan = rxq_to_chan(&adap->sge, queue);
2350         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2351         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2352                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2353         return t4_mgmt_tx(adap, skb);
2354 }
2355 EXPORT_SYMBOL(cxgb4_create_server);
2356
2357 /**
2358  *      cxgb4_create_server6 - create an IPv6 server
2359  *      @dev: the device
2360  *      @stid: the server TID
2361  *      @sip: local IPv6 address to bind server to
2362  *      @sport: the server's TCP port
2363  *      @queue: queue to direct messages from this server to
2364  *
2365  *      Create an IPv6 server for the given port and address.
2366  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2367  */
2368 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2369                          const struct in6_addr *sip, __be16 sport,
2370                          unsigned int queue)
2371 {
2372         unsigned int chan;
2373         struct sk_buff *skb;
2374         struct adapter *adap;
2375         struct cpl_pass_open_req6 *req;
2376
2377         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2378         if (!skb)
2379                 return -ENOMEM;
2380
2381         adap = netdev2adap(dev);
2382         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2383         INIT_TP_WR(req, 0);
2384         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2385         req->local_port = sport;
2386         req->peer_port = htons(0);
2387         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2388         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2389         req->peer_ip_hi = cpu_to_be64(0);
2390         req->peer_ip_lo = cpu_to_be64(0);
2391         chan = rxq_to_chan(&adap->sge, queue);
2392         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2393         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2394                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2395         return t4_mgmt_tx(adap, skb);
2396 }
2397 EXPORT_SYMBOL(cxgb4_create_server6);
2398
2399 /**
2400  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2401  *      @mtus: the HW MTU table
2402  *      @mtu: the target MTU
2403  *      @idx: index of selected entry in the MTU table
2404  *
2405  *      Returns the index and the value in the HW MTU table that is closest to
2406  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2407  *      table, in which case that smallest available value is selected.
2408  */
2409 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2410                             unsigned int *idx)
2411 {
2412         unsigned int i = 0;
2413
2414         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2415                 ++i;
2416         if (idx)
2417                 *idx = i;
2418         return mtus[i];
2419 }
2420 EXPORT_SYMBOL(cxgb4_best_mtu);
2421
2422 /**
2423  *      cxgb4_port_chan - get the HW channel of a port
2424  *      @dev: the net device for the port
2425  *
2426  *      Return the HW Tx channel of the given port.
2427  */
2428 unsigned int cxgb4_port_chan(const struct net_device *dev)
2429 {
2430         return netdev2pinfo(dev)->tx_chan;
2431 }
2432 EXPORT_SYMBOL(cxgb4_port_chan);
2433
2434 /**
2435  *      cxgb4_port_viid - get the VI id of a port
2436  *      @dev: the net device for the port
2437  *
2438  *      Return the VI id of the given port.
2439  */
2440 unsigned int cxgb4_port_viid(const struct net_device *dev)
2441 {
2442         return netdev2pinfo(dev)->viid;
2443 }
2444 EXPORT_SYMBOL(cxgb4_port_viid);
2445
2446 /**
2447  *      cxgb4_port_idx - get the index of a port
2448  *      @dev: the net device for the port
2449  *
2450  *      Return the index of the given port.
2451  */
2452 unsigned int cxgb4_port_idx(const struct net_device *dev)
2453 {
2454         return netdev2pinfo(dev)->port_id;
2455 }
2456 EXPORT_SYMBOL(cxgb4_port_idx);
2457
2458 /**
2459  *      cxgb4_netdev_by_hwid - return the net device of a HW port
2460  *      @pdev: identifies the adapter
2461  *      @id: the HW port id
2462  *
2463  *      Return the net device associated with the interface with the given HW
2464  *      id.
2465  */
2466 struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2467 {
2468         const struct adapter *adap = pci_get_drvdata(pdev);
2469
2470         if (!adap || id >= NCHAN)
2471                 return NULL;
2472         id = adap->chan_map[id];
2473         return id < MAX_NPORTS ? adap->port[id] : NULL;
2474 }
2475 EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2476
2477 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2478                          struct tp_tcp_stats *v6)
2479 {
2480         struct adapter *adap = pci_get_drvdata(pdev);
2481
2482         spin_lock(&adap->stats_lock);
2483         t4_tp_get_tcp_stats(adap, v4, v6);
2484         spin_unlock(&adap->stats_lock);
2485 }
2486 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2487
2488 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2489                       const unsigned int *pgsz_order)
2490 {
2491         struct adapter *adap = netdev2adap(dev);
2492
2493         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2494         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2495                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2496                      HPZ3(pgsz_order[3]));
2497 }
2498 EXPORT_SYMBOL(cxgb4_iscsi_init);
2499
2500 static struct pci_driver cxgb4_driver;
2501
2502 static void check_neigh_update(struct neighbour *neigh)
2503 {
2504         const struct device *parent;
2505         const struct net_device *netdev = neigh->dev;
2506
2507         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2508                 netdev = vlan_dev_real_dev(netdev);
2509         parent = netdev->dev.parent;
2510         if (parent && parent->driver == &cxgb4_driver.driver)
2511                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2512 }
2513
2514 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2515                        void *data)
2516 {
2517         switch (event) {
2518         case NETEVENT_NEIGH_UPDATE:
2519                 check_neigh_update(data);
2520                 break;
2521         case NETEVENT_PMTU_UPDATE:
2522         case NETEVENT_REDIRECT:
2523         default:
2524                 break;
2525         }
2526         return 0;
2527 }
2528
2529 static bool netevent_registered;
2530 static struct notifier_block cxgb4_netevent_nb = {
2531         .notifier_call = netevent_cb
2532 };
2533
2534 static void uld_attach(struct adapter *adap, unsigned int uld)
2535 {
2536         void *handle;
2537         struct cxgb4_lld_info lli;
2538
2539         lli.pdev = adap->pdev;
2540         lli.l2t = adap->l2t;
2541         lli.tids = &adap->tids;
2542         lli.ports = adap->port;
2543         lli.vr = &adap->vres;
2544         lli.mtus = adap->params.mtus;
2545         if (uld == CXGB4_ULD_RDMA) {
2546                 lli.rxq_ids = adap->sge.rdma_rxq;
2547                 lli.nrxq = adap->sge.rdmaqs;
2548         } else if (uld == CXGB4_ULD_ISCSI) {
2549                 lli.rxq_ids = adap->sge.ofld_rxq;
2550                 lli.nrxq = adap->sge.ofldqsets;
2551         }
2552         lli.ntxq = adap->sge.ofldqsets;
2553         lli.nchan = adap->params.nports;
2554         lli.nports = adap->params.nports;
2555         lli.wr_cred = adap->params.ofldq_wr_cred;
2556         lli.adapter_type = adap->params.rev;
2557         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2558         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2559                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2560                         (adap->fn * 4));
2561         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2562                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2563                         (adap->fn * 4));
2564         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2565         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2566         lli.fw_vers = adap->params.fw_vers;
2567
2568         handle = ulds[uld].add(&lli);
2569         if (IS_ERR(handle)) {
2570                 dev_warn(adap->pdev_dev,
2571                          "could not attach to the %s driver, error %ld\n",
2572                          uld_str[uld], PTR_ERR(handle));
2573                 return;
2574         }
2575
2576         adap->uld_handle[uld] = handle;
2577
2578         if (!netevent_registered) {
2579                 register_netevent_notifier(&cxgb4_netevent_nb);
2580                 netevent_registered = true;
2581         }
2582
2583         if (adap->flags & FULL_INIT_DONE)
2584                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2585 }
2586
2587 static void attach_ulds(struct adapter *adap)
2588 {
2589         unsigned int i;
2590
2591         mutex_lock(&uld_mutex);
2592         list_add_tail(&adap->list_node, &adapter_list);
2593         for (i = 0; i < CXGB4_ULD_MAX; i++)
2594                 if (ulds[i].add)
2595                         uld_attach(adap, i);
2596         mutex_unlock(&uld_mutex);
2597 }
2598
2599 static void detach_ulds(struct adapter *adap)
2600 {
2601         unsigned int i;
2602
2603         mutex_lock(&uld_mutex);
2604         list_del(&adap->list_node);
2605         for (i = 0; i < CXGB4_ULD_MAX; i++)
2606                 if (adap->uld_handle[i]) {
2607                         ulds[i].state_change(adap->uld_handle[i],
2608                                              CXGB4_STATE_DETACH);
2609                         adap->uld_handle[i] = NULL;
2610                 }
2611         if (netevent_registered && list_empty(&adapter_list)) {
2612                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2613                 netevent_registered = false;
2614         }
2615         mutex_unlock(&uld_mutex);
2616 }
2617
2618 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2619 {
2620         unsigned int i;
2621
2622         mutex_lock(&uld_mutex);
2623         for (i = 0; i < CXGB4_ULD_MAX; i++)
2624                 if (adap->uld_handle[i])
2625                         ulds[i].state_change(adap->uld_handle[i], new_state);
2626         mutex_unlock(&uld_mutex);
2627 }
2628
2629 /**
2630  *      cxgb4_register_uld - register an upper-layer driver
2631  *      @type: the ULD type
2632  *      @p: the ULD methods
2633  *
2634  *      Registers an upper-layer driver with this driver and notifies the ULD
2635  *      about any presently available devices that support its type.  Returns
2636  *      %-EBUSY if a ULD of the same type is already registered.
2637  */
2638 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2639 {
2640         int ret = 0;
2641         struct adapter *adap;
2642
2643         if (type >= CXGB4_ULD_MAX)
2644                 return -EINVAL;
2645         mutex_lock(&uld_mutex);
2646         if (ulds[type].add) {
2647                 ret = -EBUSY;
2648                 goto out;
2649         }
2650         ulds[type] = *p;
2651         list_for_each_entry(adap, &adapter_list, list_node)
2652                 uld_attach(adap, type);
2653 out:    mutex_unlock(&uld_mutex);
2654         return ret;
2655 }
2656 EXPORT_SYMBOL(cxgb4_register_uld);
2657
2658 /**
2659  *      cxgb4_unregister_uld - unregister an upper-layer driver
2660  *      @type: the ULD type
2661  *
2662  *      Unregisters an existing upper-layer driver.
2663  */
2664 int cxgb4_unregister_uld(enum cxgb4_uld type)
2665 {
2666         struct adapter *adap;
2667
2668         if (type >= CXGB4_ULD_MAX)
2669                 return -EINVAL;
2670         mutex_lock(&uld_mutex);
2671         list_for_each_entry(adap, &adapter_list, list_node)
2672                 adap->uld_handle[type] = NULL;
2673         ulds[type].add = NULL;
2674         mutex_unlock(&uld_mutex);
2675         return 0;
2676 }
2677 EXPORT_SYMBOL(cxgb4_unregister_uld);
2678
2679 /**
2680  *      cxgb_up - enable the adapter
2681  *      @adap: adapter being enabled
2682  *
2683  *      Called when the first port is enabled, this function performs the
2684  *      actions necessary to make an adapter operational, such as completing
2685  *      the initialization of HW modules, and enabling interrupts.
2686  *
2687  *      Must be called with the rtnl lock held.
2688  */
2689 static int cxgb_up(struct adapter *adap)
2690 {
2691         int err;
2692
2693         err = setup_sge_queues(adap);
2694         if (err)
2695                 goto out;
2696         err = setup_rss(adap);
2697         if (err)
2698                 goto freeq;
2699
2700         if (adap->flags & USING_MSIX) {
2701                 name_msix_vecs(adap);
2702                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2703                                   adap->msix_info[0].desc, adap);
2704                 if (err)
2705                         goto irq_err;
2706
2707                 err = request_msix_queue_irqs(adap);
2708                 if (err) {
2709                         free_irq(adap->msix_info[0].vec, adap);
2710                         goto irq_err;
2711                 }
2712         } else {
2713                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2714                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2715                                   adap->name, adap);
2716                 if (err)
2717                         goto irq_err;
2718         }
2719         enable_rx(adap);
2720         t4_sge_start(adap);
2721         t4_intr_enable(adap);
2722         adap->flags |= FULL_INIT_DONE;
2723         notify_ulds(adap, CXGB4_STATE_UP);
2724  out:
2725         return err;
2726  irq_err:
2727         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2728  freeq:
2729         t4_free_sge_resources(adap);
2730         goto out;
2731 }
2732
2733 static void cxgb_down(struct adapter *adapter)
2734 {
2735         t4_intr_disable(adapter);
2736         cancel_work_sync(&adapter->tid_release_task);
2737         adapter->tid_release_task_busy = false;
2738         adapter->tid_release_head = NULL;
2739
2740         if (adapter->flags & USING_MSIX) {
2741                 free_msix_queue_irqs(adapter);
2742                 free_irq(adapter->msix_info[0].vec, adapter);
2743         } else
2744                 free_irq(adapter->pdev->irq, adapter);
2745         quiesce_rx(adapter);
2746         t4_sge_stop(adapter);
2747         t4_free_sge_resources(adapter);
2748         adapter->flags &= ~FULL_INIT_DONE;
2749 }
2750
2751 /*
2752  * net_device operations
2753  */
2754 static int cxgb_open(struct net_device *dev)
2755 {
2756         int err;
2757         struct port_info *pi = netdev_priv(dev);
2758         struct adapter *adapter = pi->adapter;
2759
2760         if (!(adapter->flags & FULL_INIT_DONE)) {
2761                 err = cxgb_up(adapter);
2762                 if (err < 0)
2763                         return err;
2764         }
2765
2766         netif_set_real_num_tx_queues(dev, pi->nqsets);
2767         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
2768         if (err)
2769                 return err;
2770         err = link_start(dev);
2771         if (!err)
2772                 netif_tx_start_all_queues(dev);
2773         return err;
2774 }
2775
2776 static int cxgb_close(struct net_device *dev)
2777 {
2778         struct port_info *pi = netdev_priv(dev);
2779         struct adapter *adapter = pi->adapter;
2780
2781         netif_tx_stop_all_queues(dev);
2782         netif_carrier_off(dev);
2783         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2784 }
2785
2786 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2787                                                 struct rtnl_link_stats64 *ns)
2788 {
2789         struct port_stats stats;
2790         struct port_info *p = netdev_priv(dev);
2791         struct adapter *adapter = p->adapter;
2792
2793         spin_lock(&adapter->stats_lock);
2794         t4_get_port_stats(adapter, p->tx_chan, &stats);
2795         spin_unlock(&adapter->stats_lock);
2796
2797         ns->tx_bytes   = stats.tx_octets;
2798         ns->tx_packets = stats.tx_frames;
2799         ns->rx_bytes   = stats.rx_octets;
2800         ns->rx_packets = stats.rx_frames;
2801         ns->multicast  = stats.rx_mcast_frames;
2802
2803         /* detailed rx_errors */
2804         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2805                                stats.rx_runt;
2806         ns->rx_over_errors   = 0;
2807         ns->rx_crc_errors    = stats.rx_fcs_err;
2808         ns->rx_frame_errors  = stats.rx_symbol_err;
2809         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
2810                                stats.rx_ovflow2 + stats.rx_ovflow3 +
2811                                stats.rx_trunc0 + stats.rx_trunc1 +
2812                                stats.rx_trunc2 + stats.rx_trunc3;
2813         ns->rx_missed_errors = 0;
2814
2815         /* detailed tx_errors */
2816         ns->tx_aborted_errors   = 0;
2817         ns->tx_carrier_errors   = 0;
2818         ns->tx_fifo_errors      = 0;
2819         ns->tx_heartbeat_errors = 0;
2820         ns->tx_window_errors    = 0;
2821
2822         ns->tx_errors = stats.tx_error_frames;
2823         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2824                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2825         return ns;
2826 }
2827
2828 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2829 {
2830         unsigned int mbox;
2831         int ret = 0, prtad, devad;
2832         struct port_info *pi = netdev_priv(dev);
2833         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2834
2835         switch (cmd) {
2836         case SIOCGMIIPHY:
2837                 if (pi->mdio_addr < 0)
2838                         return -EOPNOTSUPP;
2839                 data->phy_id = pi->mdio_addr;
2840                 break;
2841         case SIOCGMIIREG:
2842         case SIOCSMIIREG:
2843                 if (mdio_phy_id_is_c45(data->phy_id)) {
2844                         prtad = mdio_phy_id_prtad(data->phy_id);
2845                         devad = mdio_phy_id_devad(data->phy_id);
2846                 } else if (data->phy_id < 32) {
2847                         prtad = data->phy_id;
2848                         devad = 0;
2849                         data->reg_num &= 0x1f;
2850                 } else
2851                         return -EINVAL;
2852
2853                 mbox = pi->adapter->fn;
2854                 if (cmd == SIOCGMIIREG)
2855                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2856                                          data->reg_num, &data->val_out);
2857                 else
2858                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2859                                          data->reg_num, data->val_in);
2860                 break;
2861         default:
2862                 return -EOPNOTSUPP;
2863         }
2864         return ret;
2865 }
2866
2867 static void cxgb_set_rxmode(struct net_device *dev)
2868 {
2869         /* unfortunately we can't return errors to the stack */
2870         set_rxmode(dev, -1, false);
2871 }
2872
2873 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2874 {
2875         int ret;
2876         struct port_info *pi = netdev_priv(dev);
2877
2878         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
2879                 return -EINVAL;
2880         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2881                             -1, -1, -1, true);
2882         if (!ret)
2883                 dev->mtu = new_mtu;
2884         return ret;
2885 }
2886
2887 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2888 {
2889         int ret;
2890         struct sockaddr *addr = p;
2891         struct port_info *pi = netdev_priv(dev);
2892
2893         if (!is_valid_ether_addr(addr->sa_data))
2894                 return -EINVAL;
2895
2896         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2897                             pi->xact_addr_filt, addr->sa_data, true, true);
2898         if (ret < 0)
2899                 return ret;
2900
2901         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2902         pi->xact_addr_filt = ret;
2903         return 0;
2904 }
2905
2906 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2907 {
2908         struct port_info *pi = netdev_priv(dev);
2909
2910         pi->vlan_grp = grp;
2911         t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1, -1, -1, -1,
2912                       grp != NULL, true);
2913 }
2914
2915 #ifdef CONFIG_NET_POLL_CONTROLLER
2916 static void cxgb_netpoll(struct net_device *dev)
2917 {
2918         struct port_info *pi = netdev_priv(dev);
2919         struct adapter *adap = pi->adapter;
2920
2921         if (adap->flags & USING_MSIX) {
2922                 int i;
2923                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2924
2925                 for (i = pi->nqsets; i; i--, rx++)
2926                         t4_sge_intr_msix(0, &rx->rspq);
2927         } else
2928                 t4_intr_handler(adap)(0, adap);
2929 }
2930 #endif
2931
2932 static const struct net_device_ops cxgb4_netdev_ops = {
2933         .ndo_open             = cxgb_open,
2934         .ndo_stop             = cxgb_close,
2935         .ndo_start_xmit       = t4_eth_xmit,
2936         .ndo_get_stats64      = cxgb_get_stats,
2937         .ndo_set_rx_mode      = cxgb_set_rxmode,
2938         .ndo_set_mac_address  = cxgb_set_mac_addr,
2939         .ndo_validate_addr    = eth_validate_addr,
2940         .ndo_do_ioctl         = cxgb_ioctl,
2941         .ndo_change_mtu       = cxgb_change_mtu,
2942         .ndo_vlan_rx_register = vlan_rx_register,
2943 #ifdef CONFIG_NET_POLL_CONTROLLER
2944         .ndo_poll_controller  = cxgb_netpoll,
2945 #endif
2946 };
2947
2948 void t4_fatal_err(struct adapter *adap)
2949 {
2950         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2951         t4_intr_disable(adap);
2952         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2953 }
2954
2955 static void setup_memwin(struct adapter *adap)
2956 {
2957         u32 bar0;
2958
2959         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
2960         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2961                      (bar0 + MEMWIN0_BASE) | BIR(0) |
2962                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2963         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2964                      (bar0 + MEMWIN1_BASE) | BIR(0) |
2965                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2966         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2967                      (bar0 + MEMWIN2_BASE) | BIR(0) |
2968                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2969         if (adap->vres.ocq.size) {
2970                 unsigned int start, sz_kb;
2971
2972                 start = pci_resource_start(adap->pdev, 2) +
2973                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2974                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2975                 t4_write_reg(adap,
2976                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2977                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
2978                 t4_write_reg(adap,
2979                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2980                              adap->vres.ocq.start);
2981                 t4_read_reg(adap,
2982                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2983         }
2984 }
2985
2986 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2987 {
2988         u32 v;
2989         int ret;
2990
2991         /* get device capabilities */
2992         memset(c, 0, sizeof(*c));
2993         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2994                                FW_CMD_REQUEST | FW_CMD_READ);
2995         c->retval_len16 = htonl(FW_LEN16(*c));
2996         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
2997         if (ret < 0)
2998                 return ret;
2999
3000         /* select capabilities we'll be using */
3001         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3002                 if (!vf_acls)
3003                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3004                 else
3005                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3006         } else if (vf_acls) {
3007                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
3008                 return ret;
3009         }
3010         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3011                                FW_CMD_REQUEST | FW_CMD_WRITE);
3012         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
3013         if (ret < 0)
3014                 return ret;
3015
3016         ret = t4_config_glbl_rss(adap, adap->fn,
3017                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3018                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3019                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
3020         if (ret < 0)
3021                 return ret;
3022
3023         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
3024                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
3025         if (ret < 0)
3026                 return ret;
3027
3028         t4_sge_init(adap);
3029
3030         /* tweak some settings */
3031         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
3032         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
3033         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
3034         v = t4_read_reg(adap, TP_PIO_DATA);
3035         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
3036
3037         /* get basic stuff going */
3038         return t4_early_init(adap, adap->fn);
3039 }
3040
3041 /*
3042  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
3043  */
3044 #define MAX_ATIDS 8192U
3045
3046 /*
3047  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3048  */
3049 static int adap_init0(struct adapter *adap)
3050 {
3051         int ret;
3052         u32 v, port_vec;
3053         enum dev_state state;
3054         u32 params[7], val[7];
3055         struct fw_caps_config_cmd c;
3056
3057         ret = t4_check_fw_version(adap);
3058         if (ret == -EINVAL || ret > 0) {
3059                 if (upgrade_fw(adap) >= 0)             /* recache FW version */
3060                         ret = t4_check_fw_version(adap);
3061         }
3062         if (ret < 0)
3063                 return ret;
3064
3065         /* contact FW, request master */
3066         ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
3067         if (ret < 0) {
3068                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3069                         ret);
3070                 return ret;
3071         }
3072
3073         /* reset device */
3074         ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
3075         if (ret < 0)
3076                 goto bye;
3077
3078         for (v = 0; v < SGE_NTIMERS - 1; v++)
3079                 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3080         adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3081         adap->sge.counter_val[0] = 1;
3082         for (v = 1; v < SGE_NCOUNTERS; v++)
3083                 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3084                                                THRESHOLD_3_MASK);
3085 #define FW_PARAM_DEV(param) \
3086         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3087          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3088
3089         params[0] = FW_PARAM_DEV(CCLK);
3090         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
3091         if (ret < 0)
3092                 goto bye;
3093         adap->params.vpd.cclk = val[0];
3094
3095         ret = adap_init1(adap, &c);
3096         if (ret < 0)
3097                 goto bye;
3098
3099 #define FW_PARAM_PFVF(param) \
3100         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3101          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3102          FW_PARAMS_PARAM_Y(adap->fn))
3103
3104         params[0] = FW_PARAM_DEV(PORTVEC);
3105         params[1] = FW_PARAM_PFVF(L2T_START);
3106         params[2] = FW_PARAM_PFVF(L2T_END);
3107         params[3] = FW_PARAM_PFVF(FILTER_START);
3108         params[4] = FW_PARAM_PFVF(FILTER_END);
3109         params[5] = FW_PARAM_PFVF(IQFLINT_START);
3110         params[6] = FW_PARAM_PFVF(EQ_START);
3111         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
3112         if (ret < 0)
3113                 goto bye;
3114         port_vec = val[0];
3115         adap->tids.ftid_base = val[3];
3116         adap->tids.nftids = val[4] - val[3] + 1;
3117         adap->sge.ingr_start = val[5];
3118         adap->sge.egr_start = val[6];
3119
3120         if (c.ofldcaps) {
3121                 /* query offload-related parameters */
3122                 params[0] = FW_PARAM_DEV(NTID);
3123                 params[1] = FW_PARAM_PFVF(SERVER_START);
3124                 params[2] = FW_PARAM_PFVF(SERVER_END);
3125                 params[3] = FW_PARAM_PFVF(TDDP_START);
3126                 params[4] = FW_PARAM_PFVF(TDDP_END);
3127                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3128                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3129                                       val);
3130                 if (ret < 0)
3131                         goto bye;
3132                 adap->tids.ntids = val[0];
3133                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3134                 adap->tids.stid_base = val[1];
3135                 adap->tids.nstids = val[2] - val[1] + 1;
3136                 adap->vres.ddp.start = val[3];
3137                 adap->vres.ddp.size = val[4] - val[3] + 1;
3138                 adap->params.ofldq_wr_cred = val[5];
3139                 adap->params.offload = 1;
3140         }
3141         if (c.rdmacaps) {
3142                 params[0] = FW_PARAM_PFVF(STAG_START);
3143                 params[1] = FW_PARAM_PFVF(STAG_END);
3144                 params[2] = FW_PARAM_PFVF(RQ_START);
3145                 params[3] = FW_PARAM_PFVF(RQ_END);
3146                 params[4] = FW_PARAM_PFVF(PBL_START);
3147                 params[5] = FW_PARAM_PFVF(PBL_END);
3148                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3149                                       val);
3150                 if (ret < 0)
3151                         goto bye;
3152                 adap->vres.stag.start = val[0];
3153                 adap->vres.stag.size = val[1] - val[0] + 1;
3154                 adap->vres.rq.start = val[2];
3155                 adap->vres.rq.size = val[3] - val[2] + 1;
3156                 adap->vres.pbl.start = val[4];
3157                 adap->vres.pbl.size = val[5] - val[4] + 1;
3158
3159                 params[0] = FW_PARAM_PFVF(SQRQ_START);
3160                 params[1] = FW_PARAM_PFVF(SQRQ_END);
3161                 params[2] = FW_PARAM_PFVF(CQ_START);
3162                 params[3] = FW_PARAM_PFVF(CQ_END);
3163                 params[4] = FW_PARAM_PFVF(OCQ_START);
3164                 params[5] = FW_PARAM_PFVF(OCQ_END);
3165                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3166                                       val);
3167                 if (ret < 0)
3168                         goto bye;
3169                 adap->vres.qp.start = val[0];
3170                 adap->vres.qp.size = val[1] - val[0] + 1;
3171                 adap->vres.cq.start = val[2];
3172                 adap->vres.cq.size = val[3] - val[2] + 1;
3173                 adap->vres.ocq.start = val[4];
3174                 adap->vres.ocq.size = val[5] - val[4] + 1;
3175         }
3176         if (c.iscsicaps) {
3177                 params[0] = FW_PARAM_PFVF(ISCSI_START);
3178                 params[1] = FW_PARAM_PFVF(ISCSI_END);
3179                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3180                                       val);
3181                 if (ret < 0)
3182                         goto bye;
3183                 adap->vres.iscsi.start = val[0];
3184                 adap->vres.iscsi.size = val[1] - val[0] + 1;
3185         }
3186 #undef FW_PARAM_PFVF
3187 #undef FW_PARAM_DEV
3188
3189         adap->params.nports = hweight32(port_vec);
3190         adap->params.portvec = port_vec;
3191         adap->flags |= FW_OK;
3192
3193         /* These are finalized by FW initialization, load their values now */
3194         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3195         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3196         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3197         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3198                      adap->params.b_wnd);
3199
3200 #ifdef CONFIG_PCI_IOV
3201         /*
3202          * Provision resource limits for Virtual Functions.  We currently
3203          * grant them all the same static resource limits except for the Port
3204          * Access Rights Mask which we're assigning based on the PF.  All of
3205          * the static provisioning stuff for both the PF and VF really needs
3206          * to be managed in a persistent manner for each device which the
3207          * firmware controls.
3208          */
3209         {
3210                 int pf, vf;
3211
3212                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3213                         if (num_vf[pf] <= 0)
3214                                 continue;
3215
3216                         /* VF numbering starts at 1! */
3217                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3218                                 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3219                                                   VFRES_NEQ, VFRES_NETHCTRL,
3220                                                   VFRES_NIQFLINT, VFRES_NIQ,
3221                                                   VFRES_TC, VFRES_NVI,
3222                                                   FW_PFVF_CMD_CMASK_MASK,
3223                                                   pfvfres_pmask(adap, pf, vf),
3224                                                   VFRES_NEXACTF,
3225                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3226                                 if (ret < 0)
3227                                         dev_warn(adap->pdev_dev, "failed to "
3228                                                  "provision pf/vf=%d/%d; "
3229                                                  "err=%d\n", pf, vf, ret);
3230                         }
3231                 }
3232         }
3233 #endif
3234
3235         setup_memwin(adap);
3236         return 0;
3237
3238         /*
3239          * If a command timed out or failed with EIO FW does not operate within
3240          * its spec or something catastrophic happened to HW/FW, stop issuing
3241          * commands.
3242          */
3243 bye:    if (ret != -ETIMEDOUT && ret != -EIO)
3244                 t4_fw_bye(adap, adap->fn);
3245         return ret;
3246 }
3247
3248 /* EEH callbacks */
3249
3250 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3251                                          pci_channel_state_t state)
3252 {
3253         int i;
3254         struct adapter *adap = pci_get_drvdata(pdev);
3255
3256         if (!adap)
3257                 goto out;
3258
3259         rtnl_lock();
3260         adap->flags &= ~FW_OK;
3261         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3262         for_each_port(adap, i) {
3263                 struct net_device *dev = adap->port[i];
3264
3265                 netif_device_detach(dev);
3266                 netif_carrier_off(dev);
3267         }
3268         if (adap->flags & FULL_INIT_DONE)
3269                 cxgb_down(adap);
3270         rtnl_unlock();
3271         pci_disable_device(pdev);
3272 out:    return state == pci_channel_io_perm_failure ?
3273                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3274 }
3275
3276 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3277 {
3278         int i, ret;
3279         struct fw_caps_config_cmd c;
3280         struct adapter *adap = pci_get_drvdata(pdev);
3281
3282         if (!adap) {
3283                 pci_restore_state(pdev);
3284                 pci_save_state(pdev);
3285                 return PCI_ERS_RESULT_RECOVERED;
3286         }
3287
3288         if (pci_enable_device(pdev)) {
3289                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3290                 return PCI_ERS_RESULT_DISCONNECT;
3291         }
3292
3293         pci_set_master(pdev);
3294         pci_restore_state(pdev);
3295         pci_save_state(pdev);
3296         pci_cleanup_aer_uncorrect_error_status(pdev);
3297
3298         if (t4_wait_dev_ready(adap) < 0)
3299                 return PCI_ERS_RESULT_DISCONNECT;
3300         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3301                 return PCI_ERS_RESULT_DISCONNECT;
3302         adap->flags |= FW_OK;
3303         if (adap_init1(adap, &c))
3304                 return PCI_ERS_RESULT_DISCONNECT;
3305
3306         for_each_port(adap, i) {
3307                 struct port_info *p = adap2pinfo(adap, i);
3308
3309                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3310                                   NULL, NULL);
3311                 if (ret < 0)
3312                         return PCI_ERS_RESULT_DISCONNECT;
3313                 p->viid = ret;
3314                 p->xact_addr_filt = -1;
3315         }
3316
3317         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3318                      adap->params.b_wnd);
3319         setup_memwin(adap);
3320         if (cxgb_up(adap))
3321                 return PCI_ERS_RESULT_DISCONNECT;
3322         return PCI_ERS_RESULT_RECOVERED;
3323 }
3324
3325 static void eeh_resume(struct pci_dev *pdev)
3326 {
3327         int i;
3328         struct adapter *adap = pci_get_drvdata(pdev);
3329
3330         if (!adap)
3331                 return;
3332
3333         rtnl_lock();
3334         for_each_port(adap, i) {
3335                 struct net_device *dev = adap->port[i];
3336
3337                 if (netif_running(dev)) {
3338                         link_start(dev);
3339                         cxgb_set_rxmode(dev);
3340                 }
3341                 netif_device_attach(dev);
3342         }
3343         rtnl_unlock();
3344 }
3345
3346 static struct pci_error_handlers cxgb4_eeh = {
3347         .error_detected = eeh_err_detected,
3348         .slot_reset     = eeh_slot_reset,
3349         .resume         = eeh_resume,
3350 };
3351
3352 static inline bool is_10g_port(const struct link_config *lc)
3353 {
3354         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3355 }
3356
3357 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3358                              unsigned int size, unsigned int iqe_size)
3359 {
3360         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3361                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3362         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3363         q->iqe_len = iqe_size;
3364         q->size = size;
3365 }
3366
3367 /*
3368  * Perform default configuration of DMA queues depending on the number and type
3369  * of ports we found and the number of available CPUs.  Most settings can be
3370  * modified by the admin prior to actual use.
3371  */
3372 static void __devinit cfg_queues(struct adapter *adap)
3373 {
3374         struct sge *s = &adap->sge;
3375         int i, q10g = 0, n10g = 0, qidx = 0;
3376
3377         for_each_port(adap, i)
3378                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3379
3380         /*
3381          * We default to 1 queue per non-10G port and up to # of cores queues
3382          * per 10G port.
3383          */
3384         if (n10g)
3385                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3386         if (q10g > num_online_cpus())
3387                 q10g = num_online_cpus();
3388
3389         for_each_port(adap, i) {
3390                 struct port_info *pi = adap2pinfo(adap, i);
3391
3392                 pi->first_qset = qidx;
3393                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3394                 qidx += pi->nqsets;
3395         }
3396
3397         s->ethqsets = qidx;
3398         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
3399
3400         if (is_offload(adap)) {
3401                 /*
3402                  * For offload we use 1 queue/channel if all ports are up to 1G,
3403                  * otherwise we divide all available queues amongst the channels
3404                  * capped by the number of available cores.
3405                  */
3406                 if (n10g) {
3407                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3408                                   num_online_cpus());
3409                         s->ofldqsets = roundup(i, adap->params.nports);
3410                 } else
3411                         s->ofldqsets = adap->params.nports;
3412                 /* For RDMA one Rx queue per channel suffices */
3413                 s->rdmaqs = adap->params.nports;
3414         }
3415
3416         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3417                 struct sge_eth_rxq *r = &s->ethrxq[i];
3418
3419                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3420                 r->fl.size = 72;
3421         }
3422
3423         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3424                 s->ethtxq[i].q.size = 1024;
3425
3426         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3427                 s->ctrlq[i].q.size = 512;
3428
3429         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3430                 s->ofldtxq[i].q.size = 1024;
3431
3432         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3433                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3434
3435                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3436                 r->rspq.uld = CXGB4_ULD_ISCSI;
3437                 r->fl.size = 72;
3438         }
3439
3440         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3441                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3442
3443                 init_rspq(&r->rspq, 0, 0, 511, 64);
3444                 r->rspq.uld = CXGB4_ULD_RDMA;
3445                 r->fl.size = 72;
3446         }
3447
3448         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3449         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3450 }
3451
3452 /*
3453  * Reduce the number of Ethernet queues across all ports to at most n.
3454  * n provides at least one queue per port.
3455  */
3456 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3457 {
3458         int i;
3459         struct port_info *pi;
3460
3461         while (n < adap->sge.ethqsets)
3462                 for_each_port(adap, i) {
3463                         pi = adap2pinfo(adap, i);
3464                         if (pi->nqsets > 1) {
3465                                 pi->nqsets--;
3466                                 adap->sge.ethqsets--;
3467                                 if (adap->sge.ethqsets <= n)
3468                                         break;
3469                         }
3470                 }
3471
3472         n = 0;
3473         for_each_port(adap, i) {
3474                 pi = adap2pinfo(adap, i);
3475                 pi->first_qset = n;
3476                 n += pi->nqsets;
3477         }
3478 }
3479
3480 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3481 #define EXTRA_VECS 2
3482
3483 static int __devinit enable_msix(struct adapter *adap)
3484 {
3485         int ofld_need = 0;
3486         int i, err, want, need;
3487         struct sge *s = &adap->sge;
3488         unsigned int nchan = adap->params.nports;
3489         struct msix_entry entries[MAX_INGQ + 1];
3490
3491         for (i = 0; i < ARRAY_SIZE(entries); ++i)
3492                 entries[i].entry = i;
3493
3494         want = s->max_ethqsets + EXTRA_VECS;
3495         if (is_offload(adap)) {
3496                 want += s->rdmaqs + s->ofldqsets;
3497                 /* need nchan for each possible ULD */
3498                 ofld_need = 2 * nchan;
3499         }
3500         need = adap->params.nports + EXTRA_VECS + ofld_need;
3501
3502         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3503                 want = err;
3504
3505         if (!err) {
3506                 /*
3507                  * Distribute available vectors to the various queue groups.
3508                  * Every group gets its minimum requirement and NIC gets top
3509                  * priority for leftovers.
3510                  */
3511                 i = want - EXTRA_VECS - ofld_need;
3512                 if (i < s->max_ethqsets) {
3513                         s->max_ethqsets = i;
3514                         if (i < s->ethqsets)
3515                                 reduce_ethqs(adap, i);
3516                 }
3517                 if (is_offload(adap)) {
3518                         i = want - EXTRA_VECS - s->max_ethqsets;
3519                         i -= ofld_need - nchan;
3520                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
3521                 }
3522                 for (i = 0; i < want; ++i)
3523                         adap->msix_info[i].vec = entries[i].vector;
3524         } else if (err > 0)
3525                 dev_info(adap->pdev_dev,
3526                          "only %d MSI-X vectors left, not using MSI-X\n", err);
3527         return err;
3528 }
3529
3530 #undef EXTRA_VECS
3531
3532 static int __devinit init_rss(struct adapter *adap)
3533 {
3534         unsigned int i, j;
3535
3536         for_each_port(adap, i) {
3537                 struct port_info *pi = adap2pinfo(adap, i);
3538
3539                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3540                 if (!pi->rss)
3541                         return -ENOMEM;
3542                 for (j = 0; j < pi->rss_size; j++)
3543                         pi->rss[j] = j % pi->nqsets;
3544         }
3545         return 0;
3546 }
3547
3548 static void __devinit print_port_info(struct adapter *adap)
3549 {
3550         static const char *base[] = {
3551                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3552                 "KX", "KR", "KR SFP+", "KR FEC"
3553         };
3554
3555         int i;
3556         char buf[80];
3557         const char *spd = "";
3558
3559         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3560                 spd = " 2.5 GT/s";
3561         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3562                 spd = " 5 GT/s";
3563
3564         for_each_port(adap, i) {
3565                 struct net_device *dev = adap->port[i];
3566                 const struct port_info *pi = netdev_priv(dev);
3567                 char *bufp = buf;
3568
3569                 if (!test_bit(i, &adap->registered_device_map))
3570                         continue;
3571
3572                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3573                         bufp += sprintf(bufp, "100/");
3574                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3575                         bufp += sprintf(bufp, "1000/");
3576                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3577                         bufp += sprintf(bufp, "10G/");
3578                 if (bufp != buf)
3579                         --bufp;
3580                 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3581
3582                 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3583                             adap->params.vpd.id, adap->params.rev,
3584                             buf, is_offload(adap) ? "R" : "",
3585                             adap->params.pci.width, spd,
3586                             (adap->flags & USING_MSIX) ? " MSI-X" :
3587                             (adap->flags & USING_MSI) ? " MSI" : "");
3588                 if (adap->name == dev->name)
3589                         netdev_info(dev, "S/N: %s, E/C: %s\n",
3590                                     adap->params.vpd.sn, adap->params.vpd.ec);
3591         }
3592 }
3593
3594 /*
3595  * Free the following resources:
3596  * - memory used for tables
3597  * - MSI/MSI-X
3598  * - net devices
3599  * - resources FW is holding for us
3600  */
3601 static void free_some_resources(struct adapter *adapter)
3602 {
3603         unsigned int i;
3604
3605         t4_free_mem(adapter->l2t);
3606         t4_free_mem(adapter->tids.tid_tab);
3607         disable_msi(adapter);
3608
3609         for_each_port(adapter, i)
3610                 if (adapter->port[i]) {
3611                         kfree(adap2pinfo(adapter, i)->rss);
3612                         free_netdev(adapter->port[i]);
3613                 }
3614         if (adapter->flags & FW_OK)
3615                 t4_fw_bye(adapter, adapter->fn);
3616 }
3617
3618 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3619                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3620
3621 static int __devinit init_one(struct pci_dev *pdev,
3622                               const struct pci_device_id *ent)
3623 {
3624         int func, i, err;
3625         struct port_info *pi;
3626         unsigned int highdma = 0;
3627         struct adapter *adapter = NULL;
3628
3629         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3630
3631         err = pci_request_regions(pdev, KBUILD_MODNAME);
3632         if (err) {
3633                 /* Just info, some other driver may have claimed the device. */
3634                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3635                 return err;
3636         }
3637
3638         /* We control everything through one PF */
3639         func = PCI_FUNC(pdev->devfn);
3640         if (func != ent->driver_data) {
3641                 pci_save_state(pdev);        /* to restore SR-IOV later */
3642                 goto sriov;
3643         }
3644
3645         err = pci_enable_device(pdev);
3646         if (err) {
3647                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3648                 goto out_release_regions;
3649         }
3650
3651         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3652                 highdma = NETIF_F_HIGHDMA;
3653                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3654                 if (err) {
3655                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3656                                 "coherent allocations\n");
3657                         goto out_disable_device;
3658                 }
3659         } else {
3660                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3661                 if (err) {
3662                         dev_err(&pdev->dev, "no usable DMA configuration\n");
3663                         goto out_disable_device;
3664                 }
3665         }
3666
3667         pci_enable_pcie_error_reporting(pdev);
3668         pci_set_master(pdev);
3669         pci_save_state(pdev);
3670
3671         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3672         if (!adapter) {
3673                 err = -ENOMEM;
3674                 goto out_disable_device;
3675         }
3676
3677         adapter->regs = pci_ioremap_bar(pdev, 0);
3678         if (!adapter->regs) {
3679                 dev_err(&pdev->dev, "cannot map device registers\n");
3680                 err = -ENOMEM;
3681                 goto out_free_adapter;
3682         }
3683
3684         adapter->pdev = pdev;
3685         adapter->pdev_dev = &pdev->dev;
3686         adapter->fn = func;
3687         adapter->name = pci_name(pdev);
3688         adapter->msg_enable = dflt_msg_enable;
3689         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3690
3691         spin_lock_init(&adapter->stats_lock);
3692         spin_lock_init(&adapter->tid_release_lock);
3693
3694         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3695
3696         err = t4_prep_adapter(adapter);
3697         if (err)
3698                 goto out_unmap_bar;
3699         err = adap_init0(adapter);
3700         if (err)
3701                 goto out_unmap_bar;
3702
3703         for_each_port(adapter, i) {
3704                 struct net_device *netdev;
3705
3706                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3707                                            MAX_ETH_QSETS);
3708                 if (!netdev) {
3709                         err = -ENOMEM;
3710                         goto out_free_dev;
3711                 }
3712
3713                 SET_NETDEV_DEV(netdev, &pdev->dev);
3714
3715                 adapter->port[i] = netdev;
3716                 pi = netdev_priv(netdev);
3717                 pi->adapter = adapter;
3718                 pi->xact_addr_filt = -1;
3719                 pi->rx_offload = RX_CSO;
3720                 pi->port_id = i;
3721                 netif_carrier_off(netdev);
3722                 netif_tx_stop_all_queues(netdev);
3723                 netdev->irq = pdev->irq;
3724
3725                 netdev->features |= NETIF_F_SG | TSO_FLAGS;
3726                 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3727                 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
3728                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3729                 netdev->vlan_features = netdev->features & VLAN_FEAT;
3730
3731                 netdev->netdev_ops = &cxgb4_netdev_ops;
3732                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3733         }
3734
3735         pci_set_drvdata(pdev, adapter);
3736
3737         if (adapter->flags & FW_OK) {
3738                 err = t4_port_init(adapter, func, func, 0);
3739                 if (err)
3740                         goto out_free_dev;
3741         }
3742
3743         /*
3744          * Configure queues and allocate tables now, they can be needed as
3745          * soon as the first register_netdev completes.
3746          */
3747         cfg_queues(adapter);
3748
3749         adapter->l2t = t4_init_l2t();
3750         if (!adapter->l2t) {
3751                 /* We tolerate a lack of L2T, giving up some functionality */
3752                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3753                 adapter->params.offload = 0;
3754         }
3755
3756         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3757                 dev_warn(&pdev->dev, "could not allocate TID table, "
3758                          "continuing\n");
3759                 adapter->params.offload = 0;
3760         }
3761
3762         /* See what interrupts we'll be using */
3763         if (msi > 1 && enable_msix(adapter) == 0)
3764                 adapter->flags |= USING_MSIX;
3765         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3766                 adapter->flags |= USING_MSI;
3767
3768         err = init_rss(adapter);
3769         if (err)
3770                 goto out_free_dev;
3771
3772         /*
3773          * The card is now ready to go.  If any errors occur during device
3774          * registration we do not fail the whole card but rather proceed only
3775          * with the ports we manage to register successfully.  However we must
3776          * register at least one net device.
3777          */
3778         for_each_port(adapter, i) {
3779                 err = register_netdev(adapter->port[i]);
3780                 if (err)
3781                         dev_warn(&pdev->dev,
3782                                  "cannot register net device %s, skipping\n",
3783                                  adapter->port[i]->name);
3784                 else {
3785                         /*
3786                          * Change the name we use for messages to the name of
3787                          * the first successfully registered interface.
3788                          */
3789                         if (!adapter->registered_device_map)
3790                                 adapter->name = adapter->port[i]->name;
3791
3792                         __set_bit(i, &adapter->registered_device_map);
3793                         adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3794                 }
3795         }
3796         if (!adapter->registered_device_map) {
3797                 dev_err(&pdev->dev, "could not register any net devices\n");
3798                 goto out_free_dev;
3799         }
3800
3801         if (cxgb4_debugfs_root) {
3802                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3803                                                            cxgb4_debugfs_root);
3804                 setup_debugfs(adapter);
3805         }
3806
3807         if (is_offload(adapter))
3808                 attach_ulds(adapter);
3809
3810         print_port_info(adapter);
3811
3812 sriov:
3813 #ifdef CONFIG_PCI_IOV
3814         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3815                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3816                         dev_info(&pdev->dev,
3817                                  "instantiated %u virtual functions\n",
3818                                  num_vf[func]);
3819 #endif
3820         return 0;
3821
3822  out_free_dev:
3823         free_some_resources(adapter);
3824  out_unmap_bar:
3825         iounmap(adapter->regs);
3826  out_free_adapter:
3827         kfree(adapter);
3828  out_disable_device:
3829         pci_disable_pcie_error_reporting(pdev);
3830         pci_disable_device(pdev);
3831  out_release_regions:
3832         pci_release_regions(pdev);
3833         pci_set_drvdata(pdev, NULL);
3834         return err;
3835 }
3836
3837 static void __devexit remove_one(struct pci_dev *pdev)
3838 {
3839         struct adapter *adapter = pci_get_drvdata(pdev);
3840
3841         pci_disable_sriov(pdev);
3842
3843         if (adapter) {
3844                 int i;
3845
3846                 if (is_offload(adapter))
3847                         detach_ulds(adapter);
3848
3849                 for_each_port(adapter, i)
3850                         if (test_bit(i, &adapter->registered_device_map))
3851                                 unregister_netdev(adapter->port[i]);
3852
3853                 if (adapter->debugfs_root)
3854                         debugfs_remove_recursive(adapter->debugfs_root);
3855
3856                 if (adapter->flags & FULL_INIT_DONE)
3857                         cxgb_down(adapter);
3858
3859                 free_some_resources(adapter);
3860                 iounmap(adapter->regs);
3861                 kfree(adapter);
3862                 pci_disable_pcie_error_reporting(pdev);
3863                 pci_disable_device(pdev);
3864                 pci_release_regions(pdev);
3865                 pci_set_drvdata(pdev, NULL);
3866         } else
3867                 pci_release_regions(pdev);
3868 }
3869
3870 static struct pci_driver cxgb4_driver = {
3871         .name     = KBUILD_MODNAME,
3872         .id_table = cxgb4_pci_tbl,
3873         .probe    = init_one,
3874         .remove   = __devexit_p(remove_one),
3875         .err_handler = &cxgb4_eeh,
3876 };
3877
3878 static int __init cxgb4_init_module(void)
3879 {
3880         int ret;
3881
3882         /* Debugfs support is optional, just warn if this fails */
3883         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3884         if (!cxgb4_debugfs_root)
3885                 pr_warning("could not create debugfs entry, continuing\n");
3886
3887         ret = pci_register_driver(&cxgb4_driver);
3888         if (ret < 0)
3889                 debugfs_remove(cxgb4_debugfs_root);
3890         return ret;
3891 }
3892
3893 static void __exit cxgb4_cleanup_module(void)
3894 {
3895         pci_unregister_driver(&cxgb4_driver);
3896         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
3897 }
3898
3899 module_init(cxgb4_init_module);
3900 module_exit(cxgb4_cleanup_module);