drivers/net/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2010 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@serverengines.com
  12  *
  13  * ServerEngines
  14  * 209 N. Fair Oaks Ave
  15  * Sunnyvale, CA 94085
  16  */
  17
  18 #include "be.h"
  19 #include "be_cmds.h"
  20 #include <asm/div64.h>
  21
  22 MODULE_VERSION(DRV_VER);
  23 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  24 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  25 MODULE_AUTHOR("ServerEngines Corporation");
  26 MODULE_LICENSE("GPL");
  27
  28 static unsigned int rx_frag_size = 2048;
  29 static unsigned int num_vfs;
  30 module_param(rx_frag_size, uint, S_IRUGO);
  31 module_param(num_vfs, uint, S_IRUGO);
  32 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  33 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  34
  35 static DEFINE_PCI_DEVICE_TABLE(be_dev_ids) = {
  36         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  37         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  38         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  39         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  40         { 0 }
  41 };
  42 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  43 /* UE Status Low CSR */
  44 static char *ue_status_low_desc[] = {
  45         "CEV",
  46         "CTX",
  47         "DBUF",
  48         "ERX",
  49         "Host",
  50         "MPU",
  51         "NDMA",
  52         "PTC ",
  53         "RDMA ",
  54         "RXF ",
  55         "RXIPS ",
  56         "RXULP0 ",
  57         "RXULP1 ",
  58         "RXULP2 ",
  59         "TIM ",
  60         "TPOST ",
  61         "TPRE ",
  62         "TXIPS ",
  63         "TXULP0 ",
  64         "TXULP1 ",
  65         "UC ",
  66         "WDMA ",
  67         "TXULP2 ",
  68         "HOST1 ",
  69         "P0_OB_LINK ",
  70         "P1_OB_LINK ",
  71         "HOST_GPIO ",
  72         "MBOX ",
  73         "AXGMAC0",
  74         "AXGMAC1",
  75         "JTAG",
  76         "MPU_INTPEND"
  77 };
  78 /* UE Status High CSR */
  79 static char *ue_status_hi_desc[] = {
  80         "LPCMEMHOST",
  81         "MGMT_MAC",
  82         "PCS0ONLINE",
  83         "MPU_IRAM",
  84         "PCS1ONLINE",
  85         "PCTL0",
  86         "PCTL1",
  87         "PMEM",
  88         "RR",
  89         "TXPB",
  90         "RXPP",
  91         "XAUI",
  92         "TXP",
  93         "ARM",
  94         "IPC",
  95         "HOST2",
  96         "HOST3",
  97         "HOST4",
  98         "HOST5",
  99         "HOST6",
 100         "HOST7",
 101         "HOST8",
 102         "HOST9",
 103         "NETC"
 104         "Unknown",
 105         "Unknown",
 106         "Unknown",
 107         "Unknown",
 108         "Unknown",
 109         "Unknown",
 110         "Unknown",
 111         "Unknown"
 112 };
 113
 114 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 115 {
 116         struct be_dma_mem *mem = &q->dma_mem;
 117         if (mem->va)
 118                 pci_free_consistent(adapter->pdev, mem->size,
 119                         mem->va, mem->dma);
 120 }
 121
 122 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 123                 u16 len, u16 entry_size)
 124 {
 125         struct be_dma_mem *mem = &q->dma_mem;
 126
 127         memset(q, 0, sizeof(*q));
 128         q->len = len;
 129         q->entry_size = entry_size;
 130         mem->size = len * entry_size;
 131         mem->va = pci_alloc_consistent(adapter->pdev, mem->size, &mem->dma);
 132         if (!mem->va)
 133                 return -1;
 134         memset(mem->va, 0, mem->size);
 135         return 0;
 136 }
 137
 138 static void be_intr_set(struct be_adapter *adapter, bool enable)
 139 {
 140         u8 __iomem *addr = adapter->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
 141         u32 reg = ioread32(addr);
 142         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 143
 144         if (adapter->eeh_err)
 145                 return;
 146
 147         if (!enabled && enable)
 148                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 149         else if (enabled && !enable)
 150                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 151         else
 152                 return;
 153
 154         iowrite32(reg, addr);
 155 }
 156
 157 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 158 {
 159         u32 val = 0;
 160         val |= qid & DB_RQ_RING_ID_MASK;
 161         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 162
 163         wmb();
 164         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 165 }
 166
 167 static void be_txq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 168 {
 169         u32 val = 0;
 170         val |= qid & DB_TXULP_RING_ID_MASK;
 171         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 172
 173         wmb();
 174         iowrite32(val, adapter->db + DB_TXULP1_OFFSET);
 175 }
 176
 177 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 178                 bool arm, bool clear_int, u16 num_popped)
 179 {
 180         u32 val = 0;
 181         val |= qid & DB_EQ_RING_ID_MASK;
 182
 183         if (adapter->eeh_err)
 184                 return;
 185
 186         if (arm)
 187                 val |= 1 << DB_EQ_REARM_SHIFT;
 188         if (clear_int)
 189                 val |= 1 << DB_EQ_CLR_SHIFT;
 190         val |= 1 << DB_EQ_EVNT_SHIFT;
 191         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 192         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 193 }
 194
 195 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 196 {
 197         u32 val = 0;
 198         val |= qid & DB_CQ_RING_ID_MASK;
 199
 200         if (adapter->eeh_err)
 201                 return;
 202
 203         if (arm)
 204                 val |= 1 << DB_CQ_REARM_SHIFT;
 205         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 206         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 207 }
 208
 209 static int be_mac_addr_set(struct net_device *netdev, void *p)
 210 {
 211         struct be_adapter *adapter = netdev_priv(netdev);
 212         struct sockaddr *addr = p;
 213         int status = 0;
 214
 215         if (!is_valid_ether_addr(addr->sa_data))
 216                 return -EADDRNOTAVAIL;
 217
 218         /* MAC addr configuration will be done in hardware for VFs
 219          * by their corresponding PFs. Just copy to netdev addr here
 220          */
 221         if (!be_physfn(adapter))
 222                 goto netdev_addr;
 223
 224         status = be_cmd_pmac_del(adapter, adapter->if_handle, adapter->pmac_id);
 225         if (status)
 226                 return status;
 227
 228         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 229                         adapter->if_handle, &adapter->pmac_id);
 230 netdev_addr:
 231         if (!status)
 232                 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 233
 234         return status;
 235 }
 236
 237 void netdev_stats_update(struct be_adapter *adapter)
 238 {
 239         struct be_hw_stats *hw_stats = hw_stats_from_cmd(adapter->stats.cmd.va);
 240         struct be_rxf_stats *rxf_stats = &hw_stats->rxf;
 241         struct be_port_rxf_stats *port_stats =
 242                         &rxf_stats->port[adapter->port_num];
 243         struct net_device_stats *dev_stats = &adapter->netdev->stats;
 244         struct be_erx_stats *erx_stats = &hw_stats->erx;
 245
 246         dev_stats->rx_packets = drvr_stats(adapter)->be_rx_pkts;
 247         dev_stats->tx_packets = drvr_stats(adapter)->be_tx_pkts;
 248         dev_stats->rx_bytes = drvr_stats(adapter)->be_rx_bytes;
 249         dev_stats->tx_bytes = drvr_stats(adapter)->be_tx_bytes;
 250
 251         /* bad pkts received */
 252         dev_stats->rx_errors = port_stats->rx_crc_errors +
 253                 port_stats->rx_alignment_symbol_errors +
 254                 port_stats->rx_in_range_errors +
 255                 port_stats->rx_out_range_errors +
 256                 port_stats->rx_frame_too_long +
 257                 port_stats->rx_dropped_too_small +
 258                 port_stats->rx_dropped_too_short +
 259                 port_stats->rx_dropped_header_too_small +
 260                 port_stats->rx_dropped_tcp_length +
 261                 port_stats->rx_dropped_runt +
 262                 port_stats->rx_tcp_checksum_errs +
 263                 port_stats->rx_ip_checksum_errs +
 264                 port_stats->rx_udp_checksum_errs;
 265
 266         /*  no space in linux buffers: best possible approximation */
 267         dev_stats->rx_dropped =
 268                 erx_stats->rx_drops_no_fragments[adapter->rx_obj.q.id];
 269
 270         /* detailed rx errors */
 271         dev_stats->rx_length_errors = port_stats->rx_in_range_errors +
 272                 port_stats->rx_out_range_errors +
 273                 port_stats->rx_frame_too_long;
 274
 275         /* receive ring buffer overflow */
 276         dev_stats->rx_over_errors = 0;
 277
 278         dev_stats->rx_crc_errors = port_stats->rx_crc_errors;
 279
 280         /* frame alignment errors */
 281         dev_stats->rx_frame_errors = port_stats->rx_alignment_symbol_errors;
 282
 283         /* receiver fifo overrun */
 284         /* drops_no_pbuf is no per i/f, it's per BE card */
 285         dev_stats->rx_fifo_errors = port_stats->rx_fifo_overflow +
 286                                         port_stats->rx_input_fifo_overflow +
 287                                         rxf_stats->rx_drops_no_pbuf;
 288         /* receiver missed packetd */
 289         dev_stats->rx_missed_errors = 0;
 290
 291         /*  packet transmit problems */
 292         dev_stats->tx_errors = 0;
 293
 294         /* no space available in linux */
 295         dev_stats->tx_dropped = 0;
 296
 297         dev_stats->multicast = port_stats->rx_multicast_frames;
 298         dev_stats->collisions = 0;
 299
 300         /* detailed tx_errors */
 301         dev_stats->tx_aborted_errors = 0;
 302         dev_stats->tx_carrier_errors = 0;
 303         dev_stats->tx_fifo_errors = 0;
 304         dev_stats->tx_heartbeat_errors = 0;
 305         dev_stats->tx_window_errors = 0;
 306 }
 307
 308 void be_link_status_update(struct be_adapter *adapter, bool link_up)
 309 {
 310         struct net_device *netdev = adapter->netdev;
 311
 312         /* If link came up or went down */
 313         if (adapter->link_up != link_up) {
 314                 adapter->link_speed = -1;
 315                 if (link_up) {
 316                         netif_start_queue(netdev);
 317                         netif_carrier_on(netdev);
 318                         printk(KERN_INFO "%s: Link up\n", netdev->name);
 319                 } else {
 320                         netif_stop_queue(netdev);
 321                         netif_carrier_off(netdev);
 322                         printk(KERN_INFO "%s: Link down\n", netdev->name);
 323                 }
 324                 adapter->link_up = link_up;
 325         }
 326 }
 327
 328 /* Update the EQ delay n BE based on the RX frags consumed / sec */
 329 static void be_rx_eqd_update(struct be_adapter *adapter)
 330 {
 331         struct be_eq_obj *rx_eq = &adapter->rx_eq;
 332         struct be_drvr_stats *stats = &adapter->stats.drvr_stats;
 333         ulong now = jiffies;
 334         u32 eqd;
 335
 336         if (!rx_eq->enable_aic)
 337                 return;
 338
 339         /* Wrapped around */
 340         if (time_before(now, stats->rx_fps_jiffies)) {
 341                 stats->rx_fps_jiffies = now;
 342                 return;
 343         }
 344
 345         /* Update once a second */
 346         if ((now - stats->rx_fps_jiffies) < HZ)
 347                 return;
 348
 349         stats->be_rx_fps = (stats->be_rx_frags - stats->be_prev_rx_frags) /
 350                         ((now - stats->rx_fps_jiffies) / HZ);
 351
 352         stats->rx_fps_jiffies = now;
 353         stats->be_prev_rx_frags = stats->be_rx_frags;
 354         eqd = stats->be_rx_fps / 110000;
 355         eqd = eqd << 3;
 356         if (eqd > rx_eq->max_eqd)
 357                 eqd = rx_eq->max_eqd;
 358         if (eqd < rx_eq->min_eqd)
 359                 eqd = rx_eq->min_eqd;
 360         if (eqd < 10)
 361                 eqd = 0;
 362         if (eqd != rx_eq->cur_eqd)
 363                 be_cmd_modify_eqd(adapter, rx_eq->q.id, eqd);
 364
 365         rx_eq->cur_eqd = eqd;
 366 }
 367
 368 static u32 be_calc_rate(u64 bytes, unsigned long ticks)
 369 {
 370         u64 rate = bytes;
 371
 372         do_div(rate, ticks / HZ);
 373         rate <<= 3;                     /* bytes/sec -> bits/sec */
 374         do_div(rate, 1000000ul);        /* MB/Sec */
 375
 376         return rate;
 377 }
 378
 379 static void be_tx_rate_update(struct be_adapter *adapter)
 380 {
 381         struct be_drvr_stats *stats = drvr_stats(adapter);
 382         ulong now = jiffies;
 383
 384         /* Wrapped around? */
 385         if (time_before(now, stats->be_tx_jiffies)) {
 386                 stats->be_tx_jiffies = now;
 387                 return;
 388         }
 389
 390         /* Update tx rate once in two seconds */
 391         if ((now - stats->be_tx_jiffies) > 2 * HZ) {
 392                 stats->be_tx_rate = be_calc_rate(stats->be_tx_bytes
 393                                                   - stats->be_tx_bytes_prev,
 394                                                  now - stats->be_tx_jiffies);
 395                 stats->be_tx_jiffies = now;
 396                 stats->be_tx_bytes_prev = stats->be_tx_bytes;
 397         }
 398 }
 399
 400 static void be_tx_stats_update(struct be_adapter *adapter,
 401                         u32 wrb_cnt, u32 copied, u32 gso_segs, bool stopped)
 402 {
 403         struct be_drvr_stats *stats = drvr_stats(adapter);
 404         stats->be_tx_reqs++;
 405         stats->be_tx_wrbs += wrb_cnt;
 406         stats->be_tx_bytes += copied;
 407         stats->be_tx_pkts += (gso_segs ? gso_segs : 1);
 408         if (stopped)
 409                 stats->be_tx_stops++;
 410 }
 411
 412 /* Determine number of WRB entries needed to xmit data in an skb */
 413 static u32 wrb_cnt_for_skb(struct sk_buff *skb, bool *dummy)
 414 {
 415         int cnt = (skb->len > skb->data_len);
 416
 417         cnt += skb_shinfo(skb)->nr_frags;
 418
 419         /* to account for hdr wrb */
 420         cnt++;
 421         if (cnt & 1) {
 422                 /* add a dummy to make it an even num */
 423                 cnt++;
 424                 *dummy = true;
 425         } else
 426                 *dummy = false;
 427         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 428         return cnt;
 429 }
 430
 431 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 432 {
 433         wrb->frag_pa_hi = upper_32_bits(addr);
 434         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 435         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 436 }
 437
 438 static void wrb_fill_hdr(struct be_eth_hdr_wrb *hdr, struct sk_buff *skb,
 439                 bool vlan, u32 wrb_cnt, u32 len)
 440 {
 441         memset(hdr, 0, sizeof(*hdr));
 442
 443         AMAP_SET_BITS(struct amap_eth_hdr_wrb, crc, hdr, 1);
 444
 445         if (skb_is_gso(skb)) {
 446                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso, hdr, 1);
 447                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso_mss,
 448                         hdr, skb_shinfo(skb)->gso_size);
 449                 if (skb_is_gso_v6(skb))
 450                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso6, hdr, 1);
 451         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 452                 if (is_tcp_pkt(skb))
 453                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, tcpcs, hdr, 1);
 454                 else if (is_udp_pkt(skb))
 455                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, udpcs, hdr, 1);
 456         }
 457
 458         if (vlan && vlan_tx_tag_present(skb)) {
 459                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan, hdr, 1);
 460                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan_tag,
 461                         hdr, vlan_tx_tag_get(skb));
 462         }
 463
 464         AMAP_SET_BITS(struct amap_eth_hdr_wrb, event, hdr, 1);
 465         AMAP_SET_BITS(struct amap_eth_hdr_wrb, complete, hdr, 1);
 466         AMAP_SET_BITS(struct amap_eth_hdr_wrb, num_wrb, hdr, wrb_cnt);
 467         AMAP_SET_BITS(struct amap_eth_hdr_wrb, len, hdr, len);
 468 }
 469
 470 static void unmap_tx_frag(struct pci_dev *pdev, struct be_eth_wrb *wrb,
 471                 bool unmap_single)
 472 {
 473         dma_addr_t dma;
 474
 475         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 476
 477         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 478         if (wrb->frag_len) {
 479                 if (unmap_single)
 480                         pci_unmap_single(pdev, dma, wrb->frag_len,
 481                                 PCI_DMA_TODEVICE);
 482                 else
 483                         pci_unmap_page(pdev, dma, wrb->frag_len,
 484                                 PCI_DMA_TODEVICE);
 485         }
 486 }
 487
 488 static int make_tx_wrbs(struct be_adapter *adapter,
 489                 struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb)
 490 {
 491         dma_addr_t busaddr;
 492         int i, copied = 0;
 493         struct pci_dev *pdev = adapter->pdev;
 494         struct sk_buff *first_skb = skb;
 495         struct be_queue_info *txq = &adapter->tx_obj.q;
 496         struct be_eth_wrb *wrb;
 497         struct be_eth_hdr_wrb *hdr;
 498         bool map_single = false;
 499         u16 map_head;
 500
 501         hdr = queue_head_node(txq);
 502         queue_head_inc(txq);
 503         map_head = txq->head;
 504
 505         if (skb->len > skb->data_len) {
 506                 int len = skb_headlen(skb);
 507                 busaddr = pci_map_single(pdev, skb->data, len,
 508                                          PCI_DMA_TODEVICE);
 509                 if (pci_dma_mapping_error(pdev, busaddr))
 510                         goto dma_err;
 511                 map_single = true;
 512                 wrb = queue_head_node(txq);
 513                 wrb_fill(wrb, busaddr, len);
 514                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 515                 queue_head_inc(txq);
 516                 copied += len;
 517         }
 518
 519         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 520                 struct skb_frag_struct *frag =
 521                         &skb_shinfo(skb)->frags[i];
 522                 busaddr = pci_map_page(pdev, frag->page,
 523                                        frag->page_offset,
 524                                        frag->size, PCI_DMA_TODEVICE);
 525                 if (pci_dma_mapping_error(pdev, busaddr))
 526                         goto dma_err;
 527                 wrb = queue_head_node(txq);
 528                 wrb_fill(wrb, busaddr, frag->size);
 529                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 530                 queue_head_inc(txq);
 531                 copied += frag->size;
 532         }
 533
 534         if (dummy_wrb) {
 535                 wrb = queue_head_node(txq);
 536                 wrb_fill(wrb, 0, 0);
 537                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 538                 queue_head_inc(txq);
 539         }
 540
 541         wrb_fill_hdr(hdr, first_skb, adapter->vlan_grp ? true : false,
 542                 wrb_cnt, copied);
 543         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 544
 545         return copied;
 546 dma_err:
 547         txq->head = map_head;
 548         while (copied) {
 549                 wrb = queue_head_node(txq);
 550                 unmap_tx_frag(pdev, wrb, map_single);
 551                 map_single = false;
 552                 copied -= wrb->frag_len;
 553                 queue_head_inc(txq);
 554         }
 555         return 0;
 556 }
 557
 558 static netdev_tx_t be_xmit(struct sk_buff *skb,
 559                         struct net_device *netdev)
 560 {
 561         struct be_adapter *adapter = netdev_priv(netdev);
 562         struct be_tx_obj *tx_obj = &adapter->tx_obj;
 563         struct be_queue_info *txq = &tx_obj->q;
 564         u32 wrb_cnt = 0, copied = 0;
 565         u32 start = txq->head;
 566         bool dummy_wrb, stopped = false;
 567
 568         wrb_cnt = wrb_cnt_for_skb(skb, &dummy_wrb);
 569
 570         copied = make_tx_wrbs(adapter, skb, wrb_cnt, dummy_wrb);
 571         if (copied) {
 572                 /* record the sent skb in the sent_skb table */
 573                 BUG_ON(tx_obj->sent_skb_list[start]);
 574                 tx_obj->sent_skb_list[start] = skb;
 575
 576                 /* Ensure txq has space for the next skb; Else stop the queue
 577                  * *BEFORE* ringing the tx doorbell, so that we serialze the
 578                  * tx compls of the current transmit which'll wake up the queue
 579                  */
 580                 atomic_add(wrb_cnt, &txq->used);
 581                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
 582                                                                 txq->len) {
 583                         netif_stop_queue(netdev);
 584                         stopped = true;
 585                 }
 586
 587                 be_txq_notify(adapter, txq->id, wrb_cnt);
 588
 589                 be_tx_stats_update(adapter, wrb_cnt, copied,
 590                                 skb_shinfo(skb)->gso_segs, stopped);
 591         } else {
 592                 txq->head = start;
 593                 dev_kfree_skb_any(skb);
 594         }
 595         return NETDEV_TX_OK;
 596 }
 597
 598 static int be_change_mtu(struct net_device *netdev, int new_mtu)
 599 {
 600         struct be_adapter *adapter = netdev_priv(netdev);
 601         if (new_mtu < BE_MIN_MTU ||
 602                         new_mtu > (BE_MAX_JUMBO_FRAME_SIZE -
 603                                         (ETH_HLEN + ETH_FCS_LEN))) {
 604                 dev_info(&adapter->pdev->dev,
 605                         "MTU must be between %d and %d bytes\n",
 606                         BE_MIN_MTU,
 607                         (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN)));
 608                 return -EINVAL;
 609         }
 610         dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n",
 611                         netdev->mtu, new_mtu);
 612         netdev->mtu = new_mtu;
 613         return 0;
 614 }
 615
 616 /*
 617  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
 618  * If the user configures more, place BE in vlan promiscuous mode.
 619  */
 620 static int be_vid_config(struct be_adapter *adapter, bool vf, u32 vf_num)
 621 {
 622         u16 vtag[BE_NUM_VLANS_SUPPORTED];
 623         u16 ntags = 0, i;
 624         int status = 0;
 625         u32 if_handle;
 626
 627         if (vf) {
 628                 if_handle = adapter->vf_cfg[vf_num].vf_if_handle;
 629                 vtag[0] = cpu_to_le16(adapter->vf_cfg[vf_num].vf_vlan_tag);
 630                 status = be_cmd_vlan_config(adapter, if_handle, vtag, 1, 1, 0);
 631         }
 632
 633         if (adapter->vlans_added <= adapter->max_vlans)  {
 634                 /* Construct VLAN Table to give to HW */
 635                 for (i = 0; i < VLAN_GROUP_ARRAY_LEN; i++) {
 636                         if (adapter->vlan_tag[i]) {
 637                                 vtag[ntags] = cpu_to_le16(i);
 638                                 ntags++;
 639                         }
 640                 }
 641                 status = be_cmd_vlan_config(adapter, adapter->if_handle,
 642                                         vtag, ntags, 1, 0);
 643         } else {
 644                 status = be_cmd_vlan_config(adapter, adapter->if_handle,
 645                                         NULL, 0, 1, 1);
 646         }
 647
 648         return status;
 649 }
 650
 651 static void be_vlan_register(struct net_device *netdev, struct vlan_group *grp)
 652 {
 653         struct be_adapter *adapter = netdev_priv(netdev);
 654         struct be_eq_obj *rx_eq = &adapter->rx_eq;
 655         struct be_eq_obj *tx_eq = &adapter->tx_eq;
 656
 657         be_eq_notify(adapter, rx_eq->q.id, false, false, 0);
 658         be_eq_notify(adapter, tx_eq->q.id, false, false, 0);
 659         adapter->vlan_grp = grp;
 660         be_eq_notify(adapter, rx_eq->q.id, true, false, 0);
 661         be_eq_notify(adapter, tx_eq->q.id, true, false, 0);
 662 }
 663
 664 static void be_vlan_add_vid(struct net_device *netdev, u16 vid)
 665 {
 666         struct be_adapter *adapter = netdev_priv(netdev);
 667
 668         adapter->vlans_added++;
 669         if (!be_physfn(adapter))
 670                 return;
 671
 672         adapter->vlan_tag[vid] = 1;
 673         if (adapter->vlans_added <= (adapter->max_vlans + 1))
 674                 be_vid_config(adapter, false, 0);
 675 }
 676
 677 static void be_vlan_rem_vid(struct net_device *netdev, u16 vid)
 678 {
 679         struct be_adapter *adapter = netdev_priv(netdev);
 680
 681         adapter->vlans_added--;
 682         vlan_group_set_device(adapter->vlan_grp, vid, NULL);
 683
 684         if (!be_physfn(adapter))
 685                 return;
 686
 687         adapter->vlan_tag[vid] = 0;
 688         if (adapter->vlans_added <= adapter->max_vlans)
 689                 be_vid_config(adapter, false, 0);
 690 }
 691
 692 static void be_set_multicast_list(struct net_device *netdev)
 693 {
 694         struct be_adapter *adapter = netdev_priv(netdev);
 695
 696         if (netdev->flags & IFF_PROMISC) {
 697                 be_cmd_promiscuous_config(adapter, adapter->port_num, 1);
 698                 adapter->promiscuous = true;
 699                 goto done;
 700         }
 701
 702         /* BE was previously in promiscous mode; disable it */
 703         if (adapter->promiscuous) {
 704                 adapter->promiscuous = false;
 705                 be_cmd_promiscuous_config(adapter, adapter->port_num, 0);
 706         }
 707
 708         /* Enable multicast promisc if num configured exceeds what we support */
 709         if (netdev->flags & IFF_ALLMULTI ||
 710             netdev_mc_count(netdev) > BE_MAX_MC) {
 711                 be_cmd_multicast_set(adapter, adapter->if_handle, NULL,
 712                                 &adapter->mc_cmd_mem);
 713                 goto done;
 714         }
 715
 716         be_cmd_multicast_set(adapter, adapter->if_handle, netdev,
 717                 &adapter->mc_cmd_mem);
 718 done:
 719         return;
 720 }
 721
 722 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
 723 {
 724         struct be_adapter *adapter = netdev_priv(netdev);
 725         int status;
 726
 727         if (!adapter->sriov_enabled)
 728                 return -EPERM;
 729
 730         if (!is_valid_ether_addr(mac) || (vf >= num_vfs))
 731                 return -EINVAL;
 732
 733         if (adapter->vf_cfg[vf].vf_pmac_id != BE_INVALID_PMAC_ID)
 734                 status = be_cmd_pmac_del(adapter,
 735                                         adapter->vf_cfg[vf].vf_if_handle,
 736                                         adapter->vf_cfg[vf].vf_pmac_id);
 737
 738         status = be_cmd_pmac_add(adapter, mac,
 739                                 adapter->vf_cfg[vf].vf_if_handle,
 740                                 &adapter->vf_cfg[vf].vf_pmac_id);
 741
 742         if (status)
 743                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed\n",
 744                                 mac, vf);
 745         else
 746                 memcpy(adapter->vf_cfg[vf].vf_mac_addr, mac, ETH_ALEN);
 747
 748         return status;
 749 }
 750
 751 static int be_get_vf_config(struct net_device *netdev, int vf,
 752                         struct ifla_vf_info *vi)
 753 {
 754         struct be_adapter *adapter = netdev_priv(netdev);
 755
 756         if (!adapter->sriov_enabled)
 757                 return -EPERM;
 758
 759         if (vf >= num_vfs)
 760                 return -EINVAL;
 761
 762         vi->vf = vf;
 763         vi->tx_rate = adapter->vf_cfg[vf].vf_tx_rate;
 764         vi->vlan = adapter->vf_cfg[vf].vf_vlan_tag;
 765         vi->qos = 0;
 766         memcpy(&vi->mac, adapter->vf_cfg[vf].vf_mac_addr, ETH_ALEN);
 767
 768         return 0;
 769 }
 770
 771 static int be_set_vf_vlan(struct net_device *netdev,
 772                         int vf, u16 vlan, u8 qos)
 773 {
 774         struct be_adapter *adapter = netdev_priv(netdev);
 775         int status = 0;
 776
 777         if (!adapter->sriov_enabled)
 778                 return -EPERM;
 779
 780         if ((vf >= num_vfs) || (vlan > 4095))
 781                 return -EINVAL;
 782
 783         if (vlan) {
 784                 adapter->vf_cfg[vf].vf_vlan_tag = vlan;
 785                 adapter->vlans_added++;
 786         } else {
 787                 adapter->vf_cfg[vf].vf_vlan_tag = 0;
 788                 adapter->vlans_added--;
 789         }
 790
 791         status = be_vid_config(adapter, true, vf);
 792
 793         if (status)
 794                 dev_info(&adapter->pdev->dev,
 795                                 "VLAN %d config on VF %d failed\n", vlan, vf);
 796         return status;
 797 }
 798
 799 static int be_set_vf_tx_rate(struct net_device *netdev,
 800                         int vf, int rate)
 801 {
 802         struct be_adapter *adapter = netdev_priv(netdev);
 803         int status = 0;
 804
 805         if (!adapter->sriov_enabled)
 806                 return -EPERM;
 807
 808         if ((vf >= num_vfs) || (rate < 0))
 809                 return -EINVAL;
 810
 811         if (rate > 10000)
 812                 rate = 10000;
 813
 814         adapter->vf_cfg[vf].vf_tx_rate = rate;
 815         status = be_cmd_set_qos(adapter, rate / 10, vf);
 816
 817         if (status)
 818                 dev_info(&adapter->pdev->dev,
 819                                 "tx rate %d on VF %d failed\n", rate, vf);
 820         return status;
 821 }
 822
 823 static void be_rx_rate_update(struct be_adapter *adapter)
 824 {
 825         struct be_drvr_stats *stats = drvr_stats(adapter);
 826         ulong now = jiffies;
 827
 828         /* Wrapped around */
 829         if (time_before(now, stats->be_rx_jiffies)) {
 830                 stats->be_rx_jiffies = now;
 831                 return;
 832         }
 833
 834         /* Update the rate once in two seconds */
 835         if ((now - stats->be_rx_jiffies) < 2 * HZ)
 836                 return;
 837
 838         stats->be_rx_rate = be_calc_rate(stats->be_rx_bytes
 839                                           - stats->be_rx_bytes_prev,
 840                                          now - stats->be_rx_jiffies);
 841         stats->be_rx_jiffies = now;
 842         stats->be_rx_bytes_prev = stats->be_rx_bytes;
 843 }
 844
 845 static void be_rx_stats_update(struct be_adapter *adapter,
 846                 u32 pktsize, u16 numfrags)
 847 {
 848         struct be_drvr_stats *stats = drvr_stats(adapter);
 849
 850         stats->be_rx_compl++;
 851         stats->be_rx_frags += numfrags;
 852         stats->be_rx_bytes += pktsize;
 853         stats->be_rx_pkts++;
 854 }
 855
 856 static inline bool do_pkt_csum(struct be_eth_rx_compl *rxcp, bool cso)
 857 {
 858         u8 l4_cksm, ip_version, ipcksm, tcpf = 0, udpf = 0, ipv6_chk;
 859
 860         l4_cksm = AMAP_GET_BITS(struct amap_eth_rx_compl, l4_cksm, rxcp);
 861         ipcksm = AMAP_GET_BITS(struct amap_eth_rx_compl, ipcksm, rxcp);
 862         ip_version = AMAP_GET_BITS(struct amap_eth_rx_compl, ip_version, rxcp);
 863         if (ip_version) {
 864                 tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl, tcpf, rxcp);
 865                 udpf = AMAP_GET_BITS(struct amap_eth_rx_compl, udpf, rxcp);
 866         }
 867         ipv6_chk = (ip_version && (tcpf || udpf));
 868
 869         return ((l4_cksm && ipv6_chk && ipcksm) && cso) ? false : true;
 870 }
 871
 872 static struct be_rx_page_info *
 873 get_rx_page_info(struct be_adapter *adapter, u16 frag_idx)
 874 {
 875         struct be_rx_page_info *rx_page_info;
 876         struct be_queue_info *rxq = &adapter->rx_obj.q;
 877
 878         rx_page_info = &adapter->rx_obj.page_info_tbl[frag_idx];
 879         BUG_ON(!rx_page_info->page);
 880
 881         if (rx_page_info->last_page_user) {
 882                 pci_unmap_page(adapter->pdev, dma_unmap_addr(rx_page_info, bus),
 883                         adapter->big_page_size, PCI_DMA_FROMDEVICE);
 884                 rx_page_info->last_page_user = false;
 885         }
 886
 887         atomic_dec(&rxq->used);
 888         return rx_page_info;
 889 }
 890
 891 /* Throwaway the data in the Rx completion */
 892 static void be_rx_compl_discard(struct be_adapter *adapter,
 893                         struct be_eth_rx_compl *rxcp)
 894 {
 895         struct be_queue_info *rxq = &adapter->rx_obj.q;
 896         struct be_rx_page_info *page_info;
 897         u16 rxq_idx, i, num_rcvd;
 898
 899         rxq_idx = AMAP_GET_BITS(struct amap_eth_rx_compl, fragndx, rxcp);
 900         num_rcvd = AMAP_GET_BITS(struct amap_eth_rx_compl, numfrags, rxcp);
 901
 902         for (i = 0; i < num_rcvd; i++) {
 903                 page_info = get_rx_page_info(adapter, rxq_idx);
 904                 put_page(page_info->page);
 905                 memset(page_info, 0, sizeof(*page_info));
 906                 index_inc(&rxq_idx, rxq->len);
 907         }
 908 }
 909
 910 /*
 911  * skb_fill_rx_data forms a complete skb for an ether frame
 912  * indicated by rxcp.
 913  */
 914 static void skb_fill_rx_data(struct be_adapter *adapter,
 915                         struct sk_buff *skb, struct be_eth_rx_compl *rxcp,
 916                         u16 num_rcvd)
 917 {
 918         struct be_queue_info *rxq = &adapter->rx_obj.q;
 919         struct be_rx_page_info *page_info;
 920         u16 rxq_idx, i, j;
 921         u32 pktsize, hdr_len, curr_frag_len, size;
 922         u8 *start;
 923
 924         rxq_idx = AMAP_GET_BITS(struct amap_eth_rx_compl, fragndx, rxcp);
 925         pktsize = AMAP_GET_BITS(struct amap_eth_rx_compl, pktsize, rxcp);
 926
 927         page_info = get_rx_page_info(adapter, rxq_idx);
 928
 929         start = page_address(page_info->page) + page_info->page_offset;
 930         prefetch(start);
 931
 932         /* Copy data in the first descriptor of this completion */
 933         curr_frag_len = min(pktsize, rx_frag_size);
 934
 935         /* Copy the header portion into skb_data */
 936         hdr_len = min((u32)BE_HDR_LEN, curr_frag_len);
 937         memcpy(skb->data, start, hdr_len);
 938         skb->len = curr_frag_len;
 939         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
 940                 /* Complete packet has now been moved to data */
 941                 put_page(page_info->page);
 942                 skb->data_len = 0;
 943                 skb->tail += curr_frag_len;
 944         } else {
 945                 skb_shinfo(skb)->nr_frags = 1;
 946                 skb_shinfo(skb)->frags[0].page = page_info->page;
 947                 skb_shinfo(skb)->frags[0].page_offset =
 948                                         page_info->page_offset + hdr_len;
 949                 skb_shinfo(skb)->frags[0].size = curr_frag_len - hdr_len;
 950                 skb->data_len = curr_frag_len - hdr_len;
 951                 skb->tail += hdr_len;
 952         }
 953         page_info->page = NULL;
 954
 955         if (pktsize <= rx_frag_size) {
 956                 BUG_ON(num_rcvd != 1);
 957                 goto done;
 958         }
 959
 960         /* More frags present for this completion */
 961         size = pktsize;
 962         for (i = 1, j = 0; i < num_rcvd; i++) {
 963                 size -= curr_frag_len;
 964                 index_inc(&rxq_idx, rxq->len);
 965                 page_info = get_rx_page_info(adapter, rxq_idx);
 966
 967                 curr_frag_len = min(size, rx_frag_size);
 968
 969                 /* Coalesce all frags from the same physical page in one slot */
 970                 if (page_info->page_offset == 0) {
 971                         /* Fresh page */
 972                         j++;
 973                         skb_shinfo(skb)->frags[j].page = page_info->page;
 974                         skb_shinfo(skb)->frags[j].page_offset =
 975                                                         page_info->page_offset;
 976                         skb_shinfo(skb)->frags[j].size = 0;
 977                         skb_shinfo(skb)->nr_frags++;
 978                 } else {
 979                         put_page(page_info->page);
 980                 }
 981
 982                 skb_shinfo(skb)->frags[j].size += curr_frag_len;
 983                 skb->len += curr_frag_len;
 984                 skb->data_len += curr_frag_len;
 985
 986                 page_info->page = NULL;
 987         }
 988         BUG_ON(j > MAX_SKB_FRAGS);
 989
 990 done:
 991         be_rx_stats_update(adapter, pktsize, num_rcvd);
 992 }
 993
 994 /* Process the RX completion indicated by rxcp when GRO is disabled */
 995 static void be_rx_compl_process(struct be_adapter *adapter,
 996                         struct be_eth_rx_compl *rxcp)
 997 {
 998         struct sk_buff *skb;
 999         u32 vlanf, vid;
1000         u16 num_rcvd;
1001         u8 vtm;
1002
1003         num_rcvd = AMAP_GET_BITS(struct amap_eth_rx_compl, numfrags, rxcp);
1004         /* Is it a flush compl that has no data */
1005         if (unlikely(num_rcvd == 0))
1006                 return;
1007
1008         skb = netdev_alloc_skb_ip_align(adapter->netdev, BE_HDR_LEN);
1009         if (unlikely(!skb)) {
1010                 if (net_ratelimit())
1011                         dev_warn(&adapter->pdev->dev, "skb alloc failed\n");
1012                 be_rx_compl_discard(adapter, rxcp);
1013                 return;
1014         }
1015
1016         skb_fill_rx_data(adapter, skb, rxcp, num_rcvd);
1017
1018         if (do_pkt_csum(rxcp, adapter->rx_csum))
1019                 skb->ip_summed = CHECKSUM_NONE;
1020         else
1021                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1022
1023         skb->truesize = skb->len + sizeof(struct sk_buff);
1024         skb->protocol = eth_type_trans(skb, adapter->netdev);
1025
1026         vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl, vtp, rxcp);
1027         vtm = AMAP_GET_BITS(struct amap_eth_rx_compl, vtm, rxcp);
1028
1029         /* vlanf could be wrongly set in some cards.
1030          * ignore if vtm is not set */
1031         if ((adapter->function_mode & 0x400) && !vtm)
1032                 vlanf = 0;
1033
1034         if (unlikely(vlanf)) {
1035                 if (!adapter->vlan_grp || adapter->vlans_added == 0) {
1036                         kfree_skb(skb);
1037                         return;
1038                 }
1039                 vid = AMAP_GET_BITS(struct amap_eth_rx_compl, vlan_tag, rxcp);
1040                 vid = swab16(vid);
1041                 vlan_hwaccel_receive_skb(skb, adapter->vlan_grp, vid);
1042         } else {
1043                 netif_receive_skb(skb);
1044         }
1045 }
1046
1047 /* Process the RX completion indicated by rxcp when GRO is enabled */
1048 static void be_rx_compl_process_gro(struct be_adapter *adapter,
1049                         struct be_eth_rx_compl *rxcp)
1050 {
1051         struct be_rx_page_info *page_info;
1052         struct sk_buff *skb = NULL;
1053         struct be_queue_info *rxq = &adapter->rx_obj.q;
1054         struct be_eq_obj *eq_obj =  &adapter->rx_eq;
1055         u32 num_rcvd, pkt_size, remaining, vlanf, curr_frag_len;
1056         u16 i, rxq_idx = 0, vid, j;
1057         u8 vtm;
1058
1059         num_rcvd = AMAP_GET_BITS(struct amap_eth_rx_compl, numfrags, rxcp);
1060         /* Is it a flush compl that has no data */
1061         if (unlikely(num_rcvd == 0))
1062                 return;
1063
1064         pkt_size = AMAP_GET_BITS(struct amap_eth_rx_compl, pktsize, rxcp);
1065         vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl, vtp, rxcp);
1066         rxq_idx = AMAP_GET_BITS(struct amap_eth_rx_compl, fragndx, rxcp);
1067         vtm = AMAP_GET_BITS(struct amap_eth_rx_compl, vtm, rxcp);
1068
1069         /* vlanf could be wrongly set in some cards.
1070          * ignore if vtm is not set */
1071         if ((adapter->function_mode & 0x400) && !vtm)
1072                 vlanf = 0;
1073
1074         skb = napi_get_frags(&eq_obj->napi);
1075         if (!skb) {
1076                 be_rx_compl_discard(adapter, rxcp);
1077                 return;
1078         }
1079
1080         remaining = pkt_size;
1081         for (i = 0, j = -1; i < num_rcvd; i++) {
1082                 page_info = get_rx_page_info(adapter, rxq_idx);
1083
1084                 curr_frag_len = min(remaining, rx_frag_size);
1085
1086                 /* Coalesce all frags from the same physical page in one slot */
1087                 if (i == 0 || page_info->page_offset == 0) {
1088                         /* First frag or Fresh page */
1089                         j++;
1090                         skb_shinfo(skb)->frags[j].page = page_info->page;
1091                         skb_shinfo(skb)->frags[j].page_offset =
1092                                                         page_info->page_offset;
1093                         skb_shinfo(skb)->frags[j].size = 0;
1094                 } else {
1095                         put_page(page_info->page);
1096                 }
1097                 skb_shinfo(skb)->frags[j].size += curr_frag_len;
1098
1099                 remaining -= curr_frag_len;
1100                 index_inc(&rxq_idx, rxq->len);
1101                 memset(page_info, 0, sizeof(*page_info));
1102         }
1103         BUG_ON(j > MAX_SKB_FRAGS);
1104
1105         skb_shinfo(skb)->nr_frags = j + 1;
1106         skb->len = pkt_size;
1107         skb->data_len = pkt_size;
1108         skb->truesize += pkt_size;
1109         skb->ip_summed = CHECKSUM_UNNECESSARY;
1110
1111         if (likely(!vlanf)) {
1112                 napi_gro_frags(&eq_obj->napi);
1113         } else {
1114                 vid = AMAP_GET_BITS(struct amap_eth_rx_compl, vlan_tag, rxcp);
1115                 vid = swab16(vid);
1116
1117                 if (!adapter->vlan_grp || adapter->vlans_added == 0)
1118                         return;
1119
1120                 vlan_gro_frags(&eq_obj->napi, adapter->vlan_grp, vid);
1121         }
1122
1123         be_rx_stats_update(adapter, pkt_size, num_rcvd);
1124 }
1125
1126 static struct be_eth_rx_compl *be_rx_compl_get(struct be_adapter *adapter)
1127 {
1128         struct be_eth_rx_compl *rxcp = queue_tail_node(&adapter->rx_obj.cq);
1129
1130         if (rxcp->dw[offsetof(struct amap_eth_rx_compl, valid) / 32] == 0)
1131                 return NULL;
1132
1133         rmb();
1134         be_dws_le_to_cpu(rxcp, sizeof(*rxcp));
1135
1136         queue_tail_inc(&adapter->rx_obj.cq);
1137         return rxcp;
1138 }
1139
1140 /* To reset the valid bit, we need to reset the whole word as
1141  * when walking the queue the valid entries are little-endian
1142  * and invalid entries are host endian
1143  */
1144 static inline void be_rx_compl_reset(struct be_eth_rx_compl *rxcp)
1145 {
1146         rxcp->dw[offsetof(struct amap_eth_rx_compl, valid) / 32] = 0;
1147 }
1148
1149 static inline struct page *be_alloc_pages(u32 size)
1150 {
1151         gfp_t alloc_flags = GFP_ATOMIC;
1152         u32 order = get_order(size);
1153         if (order > 0)
1154                 alloc_flags |= __GFP_COMP;
1155         return  alloc_pages(alloc_flags, order);
1156 }
1157
1158 /*
1159  * Allocate a page, split it to fragments of size rx_frag_size and post as
1160  * receive buffers to BE
1161  */
1162 static void be_post_rx_frags(struct be_adapter *adapter)
1163 {
1164         struct be_rx_page_info *page_info_tbl = adapter->rx_obj.page_info_tbl;
1165         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1166         struct be_queue_info *rxq = &adapter->rx_obj.q;
1167         struct page *pagep = NULL;
1168         struct be_eth_rx_d *rxd;
1169         u64 page_dmaaddr = 0, frag_dmaaddr;
1170         u32 posted, page_offset = 0;
1171
1172         page_info = &page_info_tbl[rxq->head];
1173         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1174                 if (!pagep) {
1175                         pagep = be_alloc_pages(adapter->big_page_size);
1176                         if (unlikely(!pagep)) {
1177                                 drvr_stats(adapter)->be_ethrx_post_fail++;
1178                                 break;
1179                         }
1180                         page_dmaaddr = pci_map_page(adapter->pdev, pagep, 0,
1181                                                 adapter->big_page_size,
1182                                                 PCI_DMA_FROMDEVICE);
1183                         page_info->page_offset = 0;
1184                 } else {
1185                         get_page(pagep);
1186                         page_info->page_offset = page_offset + rx_frag_size;
1187                 }
1188                 page_offset = page_info->page_offset;
1189                 page_info->page = pagep;
1190                 dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1191                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1192
1193                 rxd = queue_head_node(rxq);
1194                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1195                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1196
1197                 /* Any space left in the current big page for another frag? */
1198                 if ((page_offset + rx_frag_size + rx_frag_size) >
1199                                         adapter->big_page_size) {
1200                         pagep = NULL;
1201                         page_info->last_page_user = true;
1202                 }
1203
1204                 prev_page_info = page_info;
1205                 queue_head_inc(rxq);
1206                 page_info = &page_info_tbl[rxq->head];
1207         }
1208         if (pagep)
1209                 prev_page_info->last_page_user = true;
1210
1211         if (posted) {
1212                 atomic_add(posted, &rxq->used);
1213                 be_rxq_notify(adapter, rxq->id, posted);
1214         } else if (atomic_read(&rxq->used) == 0) {
1215                 /* Let be_worker replenish when memory is available */
1216                 adapter->rx_post_starved = true;
1217         }
1218 }
1219
1220 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1221 {
1222         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1223
1224         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1225                 return NULL;
1226
1227         rmb();
1228         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1229
1230         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1231
1232         queue_tail_inc(tx_cq);
1233         return txcp;
1234 }
1235
1236 static void be_tx_compl_process(struct be_adapter *adapter, u16 last_index)
1237 {
1238         struct be_queue_info *txq = &adapter->tx_obj.q;
1239         struct be_eth_wrb *wrb;
1240         struct sk_buff **sent_skbs = adapter->tx_obj.sent_skb_list;
1241         struct sk_buff *sent_skb;
1242         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1243         bool unmap_skb_hdr = true;
1244
1245         sent_skb = sent_skbs[txq->tail];
1246         BUG_ON(!sent_skb);
1247         sent_skbs[txq->tail] = NULL;
1248
1249         /* skip header wrb */
1250         queue_tail_inc(txq);
1251
1252         do {
1253                 cur_index = txq->tail;
1254                 wrb = queue_tail_node(txq);
1255                 unmap_tx_frag(adapter->pdev, wrb, (unmap_skb_hdr &&
1256                                         skb_headlen(sent_skb)));
1257                 unmap_skb_hdr = false;
1258
1259                 num_wrbs++;
1260                 queue_tail_inc(txq);
1261         } while (cur_index != last_index);
1262
1263         atomic_sub(num_wrbs, &txq->used);
1264
1265         kfree_skb(sent_skb);
1266 }
1267
1268 static inline struct be_eq_entry *event_get(struct be_eq_obj *eq_obj)
1269 {
1270         struct be_eq_entry *eqe = queue_tail_node(&eq_obj->q);
1271
1272         if (!eqe->evt)
1273                 return NULL;
1274
1275         rmb();
1276         eqe->evt = le32_to_cpu(eqe->evt);
1277         queue_tail_inc(&eq_obj->q);
1278         return eqe;
1279 }
1280
1281 static int event_handle(struct be_adapter *adapter,
1282                         struct be_eq_obj *eq_obj)
1283 {
1284         struct be_eq_entry *eqe;
1285         u16 num = 0;
1286
1287         while ((eqe = event_get(eq_obj)) != NULL) {
1288                 eqe->evt = 0;
1289                 num++;
1290         }
1291
1292         /* Deal with any spurious interrupts that come
1293          * without events
1294          */
1295         be_eq_notify(adapter, eq_obj->q.id, true, true, num);
1296         if (num)
1297                 napi_schedule(&eq_obj->napi);
1298
1299         return num;
1300 }
1301
1302 /* Just read and notify events without processing them.
1303  * Used at the time of destroying event queues */
1304 static void be_eq_clean(struct be_adapter *adapter,
1305                         struct be_eq_obj *eq_obj)
1306 {
1307         struct be_eq_entry *eqe;
1308         u16 num = 0;
1309
1310         while ((eqe = event_get(eq_obj)) != NULL) {
1311                 eqe->evt = 0;
1312                 num++;
1313         }
1314
1315         if (num)
1316                 be_eq_notify(adapter, eq_obj->q.id, false, true, num);
1317 }
1318
1319 static void be_rx_q_clean(struct be_adapter *adapter)
1320 {
1321         struct be_rx_page_info *page_info;
1322         struct be_queue_info *rxq = &adapter->rx_obj.q;
1323         struct be_queue_info *rx_cq = &adapter->rx_obj.cq;
1324         struct be_eth_rx_compl *rxcp;
1325         u16 tail;
1326
1327         /* First cleanup pending rx completions */
1328         while ((rxcp = be_rx_compl_get(adapter)) != NULL) {
1329                 be_rx_compl_discard(adapter, rxcp);
1330                 be_rx_compl_reset(rxcp);
1331                 be_cq_notify(adapter, rx_cq->id, true, 1);
1332         }
1333
1334         /* Then free posted rx buffer that were not used */
1335         tail = (rxq->head + rxq->len - atomic_read(&rxq->used)) % rxq->len;
1336         for (; atomic_read(&rxq->used) > 0; index_inc(&tail, rxq->len)) {
1337                 page_info = get_rx_page_info(adapter, tail);
1338                 put_page(page_info->page);
1339                 memset(page_info, 0, sizeof(*page_info));
1340         }
1341         BUG_ON(atomic_read(&rxq->used));
1342 }
1343
1344 static void be_tx_compl_clean(struct be_adapter *adapter)
1345 {
1346         struct be_queue_info *tx_cq = &adapter->tx_obj.cq;
1347         struct be_queue_info *txq = &adapter->tx_obj.q;
1348         struct be_eth_tx_compl *txcp;
1349         u16 end_idx, cmpl = 0, timeo = 0;
1350         struct sk_buff **sent_skbs = adapter->tx_obj.sent_skb_list;
1351         struct sk_buff *sent_skb;
1352         bool dummy_wrb;
1353
1354         /* Wait for a max of 200ms for all the tx-completions to arrive. */
1355         do {
1356                 while ((txcp = be_tx_compl_get(tx_cq))) {
1357                         end_idx = AMAP_GET_BITS(struct amap_eth_tx_compl,
1358                                         wrb_index, txcp);
1359                         be_tx_compl_process(adapter, end_idx);
1360                         cmpl++;
1361                 }
1362                 if (cmpl) {
1363                         be_cq_notify(adapter, tx_cq->id, false, cmpl);
1364                         cmpl = 0;
1365                 }
1366
1367                 if (atomic_read(&txq->used) == 0 || ++timeo > 200)
1368                         break;
1369
1370                 mdelay(1);
1371         } while (true);
1372
1373         if (atomic_read(&txq->used))
1374                 dev_err(&adapter->pdev->dev, "%d pending tx-completions\n",
1375                         atomic_read(&txq->used));
1376
1377         /* free posted tx for which compls will never arrive */
1378         while (atomic_read(&txq->used)) {
1379                 sent_skb = sent_skbs[txq->tail];
1380                 end_idx = txq->tail;
1381                 index_adv(&end_idx,
1382                         wrb_cnt_for_skb(sent_skb, &dummy_wrb) - 1, txq->len);
1383                 be_tx_compl_process(adapter, end_idx);
1384         }
1385 }
1386
1387 static void be_mcc_queues_destroy(struct be_adapter *adapter)
1388 {
1389         struct be_queue_info *q;
1390
1391         q = &adapter->mcc_obj.q;
1392         if (q->created)
1393                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
1394         be_queue_free(adapter, q);
1395
1396         q = &adapter->mcc_obj.cq;
1397         if (q->created)
1398                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
1399         be_queue_free(adapter, q);
1400 }
1401
1402 /* Must be called only after TX qs are created as MCC shares TX EQ */
1403 static int be_mcc_queues_create(struct be_adapter *adapter)
1404 {
1405         struct be_queue_info *q, *cq;
1406
1407         /* Alloc MCC compl queue */
1408         cq = &adapter->mcc_obj.cq;
1409         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
1410                         sizeof(struct be_mcc_compl)))
1411                 goto err;
1412
1413         /* Ask BE to create MCC compl queue; share TX's eq */
1414         if (be_cmd_cq_create(adapter, cq, &adapter->tx_eq.q, false, true, 0))
1415                 goto mcc_cq_free;
1416
1417         /* Alloc MCC queue */
1418         q = &adapter->mcc_obj.q;
1419         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
1420                 goto mcc_cq_destroy;
1421
1422         /* Ask BE to create MCC queue */
1423         if (be_cmd_mccq_create(adapter, q, cq))
1424                 goto mcc_q_free;
1425
1426         return 0;
1427
1428 mcc_q_free:
1429         be_queue_free(adapter, q);
1430 mcc_cq_destroy:
1431         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
1432 mcc_cq_free:
1433         be_queue_free(adapter, cq);
1434 err:
1435         return -1;
1436 }
1437
1438 static void be_tx_queues_destroy(struct be_adapter *adapter)
1439 {
1440         struct be_queue_info *q;
1441
1442         q = &adapter->tx_obj.q;
1443         if (q->created)
1444                 be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
1445         be_queue_free(adapter, q);
1446
1447         q = &adapter->tx_obj.cq;
1448         if (q->created)
1449                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
1450         be_queue_free(adapter, q);
1451
1452         /* Clear any residual events */
1453         be_eq_clean(adapter, &adapter->tx_eq);
1454
1455         q = &adapter->tx_eq.q;
1456         if (q->created)
1457                 be_cmd_q_destroy(adapter, q, QTYPE_EQ);
1458         be_queue_free(adapter, q);
1459 }
1460
1461 static int be_tx_queues_create(struct be_adapter *adapter)
1462 {
1463         struct be_queue_info *eq, *q, *cq;
1464
1465         adapter->tx_eq.max_eqd = 0;
1466         adapter->tx_eq.min_eqd = 0;
1467         adapter->tx_eq.cur_eqd = 96;
1468         adapter->tx_eq.enable_aic = false;
1469         /* Alloc Tx Event queue */
1470         eq = &adapter->tx_eq.q;
1471         if (be_queue_alloc(adapter, eq, EVNT_Q_LEN, sizeof(struct be_eq_entry)))
1472                 return -1;
1473
1474         /* Ask BE to create Tx Event queue */
1475         if (be_cmd_eq_create(adapter, eq, adapter->tx_eq.cur_eqd))
1476                 goto tx_eq_free;
1477         adapter->base_eq_id = adapter->tx_eq.q.id;
1478
1479         /* Alloc TX eth compl queue */
1480         cq = &adapter->tx_obj.cq;
1481         if (be_queue_alloc(adapter, cq, TX_CQ_LEN,
1482                         sizeof(struct be_eth_tx_compl)))
1483                 goto tx_eq_destroy;
1484
1485         /* Ask BE to create Tx eth compl queue */
1486         if (be_cmd_cq_create(adapter, cq, eq, false, false, 3))
1487                 goto tx_cq_free;
1488
1489         /* Alloc TX eth queue */
1490         q = &adapter->tx_obj.q;
1491         if (be_queue_alloc(adapter, q, TX_Q_LEN, sizeof(struct be_eth_wrb)))
1492                 goto tx_cq_destroy;
1493
1494         /* Ask BE to create Tx eth queue */
1495         if (be_cmd_txq_create(adapter, q, cq))
1496                 goto tx_q_free;
1497         return 0;
1498
1499 tx_q_free:
1500         be_queue_free(adapter, q);
1501 tx_cq_destroy:
1502         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
1503 tx_cq_free:
1504         be_queue_free(adapter, cq);
1505 tx_eq_destroy:
1506         be_cmd_q_destroy(adapter, eq, QTYPE_EQ);
1507 tx_eq_free:
1508         be_queue_free(adapter, eq);
1509         return -1;
1510 }
1511
1512 static void be_rx_queues_destroy(struct be_adapter *adapter)
1513 {
1514         struct be_queue_info *q;
1515
1516         q = &adapter->rx_obj.q;
1517         if (q->created) {
1518                 be_cmd_q_destroy(adapter, q, QTYPE_RXQ);
1519
1520                 /* After the rxq is invalidated, wait for a grace time
1521                  * of 1ms for all dma to end and the flush compl to arrive
1522                  */
1523                 mdelay(1);
1524                 be_rx_q_clean(adapter);
1525         }
1526         be_queue_free(adapter, q);
1527
1528         q = &adapter->rx_obj.cq;
1529         if (q->created)
1530                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
1531         be_queue_free(adapter, q);
1532
1533         /* Clear any residual events */
1534         be_eq_clean(adapter, &adapter->rx_eq);
1535
1536         q = &adapter->rx_eq.q;
1537         if (q->created)
1538                 be_cmd_q_destroy(adapter, q, QTYPE_EQ);
1539         be_queue_free(adapter, q);
1540 }
1541
1542 static int be_rx_queues_create(struct be_adapter *adapter)
1543 {
1544         struct be_queue_info *eq, *q, *cq;
1545         int rc;
1546
1547         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
1548         adapter->rx_eq.max_eqd = BE_MAX_EQD;
1549         adapter->rx_eq.min_eqd = 0;
1550         adapter->rx_eq.cur_eqd = 0;
1551         adapter->rx_eq.enable_aic = true;
1552
1553         /* Alloc Rx Event queue */
1554         eq = &adapter->rx_eq.q;
1555         rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
1556                                 sizeof(struct be_eq_entry));
1557         if (rc)
1558                 return rc;
1559
1560         /* Ask BE to create Rx Event queue */
1561         rc = be_cmd_eq_create(adapter, eq, adapter->rx_eq.cur_eqd);
1562         if (rc)
1563                 goto rx_eq_free;
1564
1565         /* Alloc RX eth compl queue */
1566         cq = &adapter->rx_obj.cq;
1567         rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
1568                         sizeof(struct be_eth_rx_compl));
1569         if (rc)
1570                 goto rx_eq_destroy;
1571
1572         /* Ask BE to create Rx eth compl queue */
1573         rc = be_cmd_cq_create(adapter, cq, eq, false, false, 3);
1574         if (rc)
1575                 goto rx_cq_free;
1576
1577         /* Alloc RX eth queue */
1578         q = &adapter->rx_obj.q;
1579         rc = be_queue_alloc(adapter, q, RX_Q_LEN, sizeof(struct be_eth_rx_d));
1580         if (rc)
1581                 goto rx_cq_destroy;
1582
1583         /* Ask BE to create Rx eth queue */
1584         rc = be_cmd_rxq_create(adapter, q, cq->id, rx_frag_size,
1585                 BE_MAX_JUMBO_FRAME_SIZE, adapter->if_handle, false);
1586         if (rc)
1587                 goto rx_q_free;
1588
1589         return 0;
1590 rx_q_free:
1591         be_queue_free(adapter, q);
1592 rx_cq_destroy:
1593         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
1594 rx_cq_free:
1595         be_queue_free(adapter, cq);
1596 rx_eq_destroy:
1597         be_cmd_q_destroy(adapter, eq, QTYPE_EQ);
1598 rx_eq_free:
1599         be_queue_free(adapter, eq);
1600         return rc;
1601 }
1602
1603 /* There are 8 evt ids per func. Retruns the evt id's bit number */
1604 static inline int be_evt_bit_get(struct be_adapter *adapter, u32 eq_id)
1605 {
1606         return eq_id - adapter->base_eq_id;
1607 }
1608
1609 static irqreturn_t be_intx(int irq, void *dev)
1610 {
1611         struct be_adapter *adapter = dev;
1612         int isr;
1613
1614         isr = ioread32(adapter->csr + CEV_ISR0_OFFSET +
1615                 (adapter->tx_eq.q.id/ 8) * CEV_ISR_SIZE);
1616         if (!isr)
1617                 return IRQ_NONE;
1618
1619         event_handle(adapter, &adapter->tx_eq);
1620         event_handle(adapter, &adapter->rx_eq);
1621
1622         return IRQ_HANDLED;
1623 }
1624
1625 static irqreturn_t be_msix_rx(int irq, void *dev)
1626 {
1627         struct be_adapter *adapter = dev;
1628
1629         event_handle(adapter, &adapter->rx_eq);
1630
1631         return IRQ_HANDLED;
1632 }
1633
1634 static irqreturn_t be_msix_tx_mcc(int irq, void *dev)
1635 {
1636         struct be_adapter *adapter = dev;
1637
1638         event_handle(adapter, &adapter->tx_eq);
1639
1640         return IRQ_HANDLED;
1641 }
1642
1643 static inline bool do_gro(struct be_adapter *adapter,
1644                         struct be_eth_rx_compl *rxcp)
1645 {
1646         int err = AMAP_GET_BITS(struct amap_eth_rx_compl, err, rxcp);
1647         int tcp_frame = AMAP_GET_BITS(struct amap_eth_rx_compl, tcpf, rxcp);
1648
1649         if (err)
1650                 drvr_stats(adapter)->be_rxcp_err++;
1651
1652         return (tcp_frame && !err) ? true : false;
1653 }
1654
1655 int be_poll_rx(struct napi_struct *napi, int budget)
1656 {
1657         struct be_eq_obj *rx_eq = container_of(napi, struct be_eq_obj, napi);
1658         struct be_adapter *adapter =
1659                 container_of(rx_eq, struct be_adapter, rx_eq);
1660         struct be_queue_info *rx_cq = &adapter->rx_obj.cq;
1661         struct be_eth_rx_compl *rxcp;
1662         u32 work_done;
1663
1664         adapter->stats.drvr_stats.be_rx_polls++;
1665         for (work_done = 0; work_done < budget; work_done++) {
1666                 rxcp = be_rx_compl_get(adapter);
1667                 if (!rxcp)
1668                         break;
1669
1670                 if (do_gro(adapter, rxcp))
1671                         be_rx_compl_process_gro(adapter, rxcp);
1672                 else
1673                         be_rx_compl_process(adapter, rxcp);
1674
1675                 be_rx_compl_reset(rxcp);
1676         }
1677
1678         /* Refill the queue */
1679         if (atomic_read(&adapter->rx_obj.q.used) < RX_FRAGS_REFILL_WM)
1680                 be_post_rx_frags(adapter);
1681
1682         /* All consumed */
1683         if (work_done < budget) {
1684                 napi_complete(napi);
1685                 be_cq_notify(adapter, rx_cq->id, true, work_done);
1686         } else {
1687                 /* More to be consumed; continue with interrupts disabled */
1688                 be_cq_notify(adapter, rx_cq->id, false, work_done);
1689         }
1690         return work_done;
1691 }
1692
1693 /* As TX and MCC share the same EQ check for both TX and MCC completions.
1694  * For TX/MCC we don't honour budget; consume everything
1695  */
1696 static int be_poll_tx_mcc(struct napi_struct *napi, int budget)
1697 {
1698         struct be_eq_obj *tx_eq = container_of(napi, struct be_eq_obj, napi);
1699         struct be_adapter *adapter =
1700                 container_of(tx_eq, struct be_adapter, tx_eq);
1701         struct be_queue_info *txq = &adapter->tx_obj.q;
1702         struct be_queue_info *tx_cq = &adapter->tx_obj.cq;
1703         struct be_eth_tx_compl *txcp;
1704         int tx_compl = 0, mcc_compl, status = 0;
1705         u16 end_idx;
1706
1707         while ((txcp = be_tx_compl_get(tx_cq))) {
1708                 end_idx = AMAP_GET_BITS(struct amap_eth_tx_compl,
1709                                 wrb_index, txcp);
1710                 be_tx_compl_process(adapter, end_idx);
1711                 tx_compl++;
1712         }
1713
1714         mcc_compl = be_process_mcc(adapter, &status);
1715
1716         napi_complete(napi);
1717
1718         if (mcc_compl) {
1719                 struct be_mcc_obj *mcc_obj = &adapter->mcc_obj;
1720                 be_cq_notify(adapter, mcc_obj->cq.id, true, mcc_compl);
1721         }
1722
1723         if (tx_compl) {
1724                 be_cq_notify(adapter, adapter->tx_obj.cq.id, true, tx_compl);
1725
1726                 /* As Tx wrbs have been freed up, wake up netdev queue if
1727                  * it was stopped due to lack of tx wrbs.
1728                  */
1729                 if (netif_queue_stopped(adapter->netdev) &&
1730                         atomic_read(&txq->used) < txq->len / 2) {
1731                         netif_wake_queue(adapter->netdev);
1732                 }
1733
1734                 drvr_stats(adapter)->be_tx_events++;
1735                 drvr_stats(adapter)->be_tx_compl += tx_compl;
1736         }
1737
1738         return 1;
1739 }
1740
1741 static inline bool be_detect_ue(struct be_adapter *adapter)
1742 {
1743         u32 online0 = 0, online1 = 0;
1744
1745         pci_read_config_dword(adapter->pdev, PCICFG_ONLINE0, &online0);
1746
1747         pci_read_config_dword(adapter->pdev, PCICFG_ONLINE1, &online1);
1748
1749         if (!online0 || !online1) {
1750                 adapter->ue_detected = true;
1751                 dev_err(&adapter->pdev->dev,
1752                         "UE Detected!! online0=%d online1=%d\n",
1753                         online0, online1);
1754                 return true;
1755         }
1756
1757         return false;
1758 }
1759
1760 void be_dump_ue(struct be_adapter *adapter)
1761 {
1762         u32 ue_status_lo, ue_status_hi, ue_status_lo_mask, ue_status_hi_mask;
1763         u32 i;
1764
1765         pci_read_config_dword(adapter->pdev,
1766                                 PCICFG_UE_STATUS_LOW, &ue_status_lo);
1767         pci_read_config_dword(adapter->pdev,
1768                                 PCICFG_UE_STATUS_HIGH, &ue_status_hi);
1769         pci_read_config_dword(adapter->pdev,
1770                                 PCICFG_UE_STATUS_LOW_MASK, &ue_status_lo_mask);
1771         pci_read_config_dword(adapter->pdev,
1772                                 PCICFG_UE_STATUS_HI_MASK, &ue_status_hi_mask);
1773
1774         ue_status_lo = (ue_status_lo & (~ue_status_lo_mask));
1775         ue_status_hi = (ue_status_hi & (~ue_status_hi_mask));
1776
1777         if (ue_status_lo) {
1778                 for (i = 0; ue_status_lo; ue_status_lo >>= 1, i++) {
1779                         if (ue_status_lo & 1)
1780                                 dev_err(&adapter->pdev->dev,
1781                                 "UE: %s bit set\n", ue_status_low_desc[i]);
1782                 }
1783         }
1784         if (ue_status_hi) {
1785                 for (i = 0; ue_status_hi; ue_status_hi >>= 1, i++) {
1786                         if (ue_status_hi & 1)
1787                                 dev_err(&adapter->pdev->dev,
1788                                 "UE: %s bit set\n", ue_status_hi_desc[i]);
1789                 }
1790         }
1791
1792 }
1793
1794 static void be_worker(struct work_struct *work)
1795 {
1796         struct be_adapter *adapter =
1797                 container_of(work, struct be_adapter, work.work);
1798
1799         if (!adapter->stats_ioctl_sent)
1800                 be_cmd_get_stats(adapter, &adapter->stats.cmd);
1801
1802         /* Set EQ delay */
1803         be_rx_eqd_update(adapter);
1804
1805         be_tx_rate_update(adapter);
1806         be_rx_rate_update(adapter);
1807
1808         if (adapter->rx_post_starved) {
1809                 adapter->rx_post_starved = false;
1810                 be_post_rx_frags(adapter);
1811         }
1812         if (!adapter->ue_detected) {
1813                 if (be_detect_ue(adapter))
1814                         be_dump_ue(adapter);
1815         }
1816
1817         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
1818 }
1819
1820 static void be_msix_disable(struct be_adapter *adapter)
1821 {
1822         if (adapter->msix_enabled) {
1823                 pci_disable_msix(adapter->pdev);
1824                 adapter->msix_enabled = false;
1825         }
1826 }
1827
1828 static void be_msix_enable(struct be_adapter *adapter)
1829 {
1830         int i, status;
1831
1832         for (i = 0; i < BE_NUM_MSIX_VECTORS; i++)
1833                 adapter->msix_entries[i].entry = i;
1834
1835         status = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1836                 BE_NUM_MSIX_VECTORS);
1837         if (status == 0)
1838                 adapter->msix_enabled = true;
1839 }
1840
1841 static void be_sriov_enable(struct be_adapter *adapter)
1842 {
1843         be_check_sriov_fn_type(adapter);
1844 #ifdef CONFIG_PCI_IOV
1845         if (be_physfn(adapter) && num_vfs) {
1846                 int status;
1847
1848                 status = pci_enable_sriov(adapter->pdev, num_vfs);
1849                 adapter->sriov_enabled = status ? false : true;
1850         }
1851 #endif
1852 }
1853
1854 static void be_sriov_disable(struct be_adapter *adapter)
1855 {
1856 #ifdef CONFIG_PCI_IOV
1857         if (adapter->sriov_enabled) {
1858                 pci_disable_sriov(adapter->pdev);
1859                 adapter->sriov_enabled = false;
1860         }
1861 #endif
1862 }
1863
1864 static inline int be_msix_vec_get(struct be_adapter *adapter, u32 eq_id)
1865 {
1866         return adapter->msix_entries[
1867                         be_evt_bit_get(adapter, eq_id)].vector;
1868 }
1869
1870 static int be_request_irq(struct be_adapter *adapter,
1871                 struct be_eq_obj *eq_obj,
1872                 void *handler, char *desc)
1873 {
1874         struct net_device *netdev = adapter->netdev;
1875         int vec;
1876
1877         sprintf(eq_obj->desc, "%s-%s", netdev->name, desc);
1878         vec = be_msix_vec_get(adapter, eq_obj->q.id);
1879         return request_irq(vec, handler, 0, eq_obj->desc, adapter);
1880 }
1881
1882 static void be_free_irq(struct be_adapter *adapter, struct be_eq_obj *eq_obj)
1883 {
1884         int vec = be_msix_vec_get(adapter, eq_obj->q.id);
1885         free_irq(vec, adapter);
1886 }
1887
1888 static int be_msix_register(struct be_adapter *adapter)
1889 {
1890         int status;
1891
1892         status = be_request_irq(adapter, &adapter->tx_eq, be_msix_tx_mcc, "tx");
1893         if (status)
1894                 goto err;
1895
1896         status = be_request_irq(adapter, &adapter->rx_eq, be_msix_rx, "rx");
1897         if (status)
1898                 goto free_tx_irq;
1899
1900         return 0;
1901
1902 free_tx_irq:
1903         be_free_irq(adapter, &adapter->tx_eq);
1904 err:
1905         dev_warn(&adapter->pdev->dev,
1906                 "MSIX Request IRQ failed - err %d\n", status);
1907         pci_disable_msix(adapter->pdev);
1908         adapter->msix_enabled = false;
1909         return status;
1910 }
1911
1912 static int be_irq_register(struct be_adapter *adapter)
1913 {
1914         struct net_device *netdev = adapter->netdev;
1915         int status;
1916
1917         if (adapter->msix_enabled) {
1918                 status = be_msix_register(adapter);
1919                 if (status == 0)
1920                         goto done;
1921                 /* INTx is not supported for VF */
1922                 if (!be_physfn(adapter))
1923                         return status;
1924         }
1925
1926         /* INTx */
1927         netdev->irq = adapter->pdev->irq;
1928         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
1929                         adapter);
1930         if (status) {
1931                 dev_err(&adapter->pdev->dev,
1932                         "INTx request IRQ failed - err %d\n", status);
1933                 return status;
1934         }
1935 done:
1936         adapter->isr_registered = true;
1937         return 0;
1938 }
1939
1940 static void be_irq_unregister(struct be_adapter *adapter)
1941 {
1942         struct net_device *netdev = adapter->netdev;
1943
1944         if (!adapter->isr_registered)
1945                 return;
1946
1947         /* INTx */
1948         if (!adapter->msix_enabled) {
1949                 free_irq(netdev->irq, adapter);
1950                 goto done;
1951         }
1952
1953         /* MSIx */
1954         be_free_irq(adapter, &adapter->tx_eq);
1955         be_free_irq(adapter, &adapter->rx_eq);
1956 done:
1957         adapter->isr_registered = false;
1958 }
1959
1960 static int be_close(struct net_device *netdev)
1961 {
1962         struct be_adapter *adapter = netdev_priv(netdev);
1963         struct be_eq_obj *rx_eq = &adapter->rx_eq;
1964         struct be_eq_obj *tx_eq = &adapter->tx_eq;
1965         int vec;
1966
1967         cancel_delayed_work_sync(&adapter->work);
1968
1969         be_async_mcc_disable(adapter);
1970
1971         netif_stop_queue(netdev);
1972         netif_carrier_off(netdev);
1973         adapter->link_up = false;
1974
1975         be_intr_set(adapter, false);
1976
1977         if (adapter->msix_enabled) {
1978                 vec = be_msix_vec_get(adapter, tx_eq->q.id);
1979                 synchronize_irq(vec);
1980                 vec = be_msix_vec_get(adapter, rx_eq->q.id);
1981                 synchronize_irq(vec);
1982         } else {
1983                 synchronize_irq(netdev->irq);
1984         }
1985         be_irq_unregister(adapter);
1986
1987         napi_disable(&rx_eq->napi);
1988         napi_disable(&tx_eq->napi);
1989
1990         /* Wait for all pending tx completions to arrive so that
1991          * all tx skbs are freed.
1992          */
1993         be_tx_compl_clean(adapter);
1994
1995         return 0;
1996 }
1997
1998 static int be_open(struct net_device *netdev)
1999 {
2000         struct be_adapter *adapter = netdev_priv(netdev);
2001         struct be_eq_obj *rx_eq = &adapter->rx_eq;
2002         struct be_eq_obj *tx_eq = &adapter->tx_eq;
2003         bool link_up;
2004         int status;
2005         u8 mac_speed;
2006         u16 link_speed;
2007
2008         /* First time posting */
2009         be_post_rx_frags(adapter);
2010
2011         napi_enable(&rx_eq->napi);
2012         napi_enable(&tx_eq->napi);
2013
2014         be_irq_register(adapter);
2015
2016         be_intr_set(adapter, true);
2017
2018         /* The evt queues are created in unarmed state; arm them */
2019         be_eq_notify(adapter, rx_eq->q.id, true, false, 0);
2020         be_eq_notify(adapter, tx_eq->q.id, true, false, 0);
2021
2022         /* Rx compl queue may be in unarmed state; rearm it */
2023         be_cq_notify(adapter, adapter->rx_obj.cq.id, true, 0);
2024
2025         /* Now that interrupts are on we can process async mcc */
2026         be_async_mcc_enable(adapter);
2027
2028         schedule_delayed_work(&adapter->work, msecs_to_jiffies(100));
2029
2030         status = be_cmd_link_status_query(adapter, &link_up, &mac_speed,
2031                         &link_speed);
2032         if (status)
2033                 goto err;
2034         be_link_status_update(adapter, link_up);
2035
2036         if (be_physfn(adapter)) {
2037                 status = be_vid_config(adapter, false, 0);
2038                 if (status)
2039                         goto err;
2040
2041                 status = be_cmd_set_flow_control(adapter,
2042                                 adapter->tx_fc, adapter->rx_fc);
2043                 if (status)
2044                         goto err;
2045         }
2046
2047         return 0;
2048 err:
2049         be_close(adapter->netdev);
2050         return -EIO;
2051 }
2052
2053 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2054 {
2055         struct be_dma_mem cmd;
2056         int status = 0;
2057         u8 mac[ETH_ALEN];
2058
2059         memset(mac, 0, ETH_ALEN);
2060
2061         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2062         cmd.va = pci_alloc_consistent(adapter->pdev, cmd.size, &cmd.dma);
2063         if (cmd.va == NULL)
2064                 return -1;
2065         memset(cmd.va, 0, cmd.size);
2066
2067         if (enable) {
2068                 status = pci_write_config_dword(adapter->pdev,
2069                         PCICFG_PM_CONTROL_OFFSET, PCICFG_PM_CONTROL_MASK);
2070                 if (status) {
2071                         dev_err(&adapter->pdev->dev,
2072                                 "Could not enable Wake-on-lan\n");
2073                         pci_free_consistent(adapter->pdev, cmd.size, cmd.va,
2074                                         cmd.dma);
2075                         return status;
2076                 }
2077                 status = be_cmd_enable_magic_wol(adapter,
2078                                 adapter->netdev->dev_addr, &cmd);
2079                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2080                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2081         } else {
2082                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2083                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2084                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2085         }
2086
2087         pci_free_consistent(adapter->pdev, cmd.size, cmd.va, cmd.dma);
2088         return status;
2089 }
2090
2091 /*
2092  * Generate a seed MAC address from the PF MAC Address using jhash.
2093  * MAC Address for VFs are assigned incrementally starting from the seed.
2094  * These addresses are programmed in the ASIC by the PF and the VF driver
2095  * queries for the MAC address during its probe.
2096  */
2097 static inline int be_vf_eth_addr_config(struct be_adapter *adapter)
2098 {
2099         u32 vf = 0;
2100         int status;
2101         u8 mac[ETH_ALEN];
2102
2103         be_vf_eth_addr_generate(adapter, mac);
2104
2105         for (vf = 0; vf < num_vfs; vf++) {
2106                 status = be_cmd_pmac_add(adapter, mac,
2107                                         adapter->vf_cfg[vf].vf_if_handle,
2108                                         &adapter->vf_cfg[vf].vf_pmac_id);
2109                 if (status)
2110                         dev_err(&adapter->pdev->dev,
2111                                 "Mac address add failed for VF %d\n", vf);
2112                 else
2113                         memcpy(adapter->vf_cfg[vf].vf_mac_addr, mac, ETH_ALEN);
2114
2115                 mac[5] += 1;
2116         }
2117         return status;
2118 }
2119
2120 static inline void be_vf_eth_addr_rem(struct be_adapter *adapter)
2121 {
2122         u32 vf;
2123
2124         for (vf = 0; vf < num_vfs; vf++) {
2125                 if (adapter->vf_cfg[vf].vf_pmac_id != BE_INVALID_PMAC_ID)
2126                         be_cmd_pmac_del(adapter,
2127                                         adapter->vf_cfg[vf].vf_if_handle,
2128                                         adapter->vf_cfg[vf].vf_pmac_id);
2129         }
2130 }
2131
2132 static int be_setup(struct be_adapter *adapter)
2133 {
2134         struct net_device *netdev = adapter->netdev;
2135         u32 cap_flags, en_flags, vf = 0;
2136         int status;
2137         u8 mac[ETH_ALEN];
2138
2139         cap_flags = en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST;
2140
2141         if (be_physfn(adapter)) {
2142                 cap_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS |
2143                                 BE_IF_FLAGS_PROMISCUOUS |
2144                                 BE_IF_FLAGS_PASS_L3L4_ERRORS;
2145                 en_flags |= BE_IF_FLAGS_PASS_L3L4_ERRORS;
2146         }
2147
2148         status = be_cmd_if_create(adapter, cap_flags, en_flags,
2149                         netdev->dev_addr, false/* pmac_invalid */,
2150                         &adapter->if_handle, &adapter->pmac_id, 0);
2151         if (status != 0)
2152                 goto do_none;
2153
2154         if (be_physfn(adapter)) {
2155                 while (vf < num_vfs) {
2156                         cap_flags = en_flags = BE_IF_FLAGS_UNTAGGED
2157                                         | BE_IF_FLAGS_BROADCAST;
2158                         status = be_cmd_if_create(adapter, cap_flags, en_flags,
2159                                         mac, true,
2160                                         &adapter->vf_cfg[vf].vf_if_handle,
2161                                         NULL, vf+1);
2162                         if (status) {
2163                                 dev_err(&adapter->pdev->dev,
2164                                 "Interface Create failed for VF %d\n", vf);
2165                                 goto if_destroy;
2166                         }
2167                         adapter->vf_cfg[vf].vf_pmac_id = BE_INVALID_PMAC_ID;
2168                         vf++;
2169                 }
2170         } else if (!be_physfn(adapter)) {
2171                 status = be_cmd_mac_addr_query(adapter, mac,
2172                         MAC_ADDRESS_TYPE_NETWORK, false, adapter->if_handle);
2173                 if (!status) {
2174                         memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
2175                         memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
2176                 }
2177         }
2178
2179         status = be_tx_queues_create(adapter);
2180         if (status != 0)
2181                 goto if_destroy;
2182
2183         status = be_rx_queues_create(adapter);
2184         if (status != 0)
2185                 goto tx_qs_destroy;
2186
2187         status = be_mcc_queues_create(adapter);
2188         if (status != 0)
2189                 goto rx_qs_destroy;
2190
2191         if (be_physfn(adapter)) {
2192                 status = be_vf_eth_addr_config(adapter);
2193                 if (status)
2194                         goto mcc_q_destroy;
2195         }
2196
2197         adapter->link_speed = -1;
2198
2199         return 0;
2200
2201 mcc_q_destroy:
2202         if (be_physfn(adapter))
2203                 be_vf_eth_addr_rem(adapter);
2204         be_mcc_queues_destroy(adapter);
2205 rx_qs_destroy:
2206         be_rx_queues_destroy(adapter);
2207 tx_qs_destroy:
2208         be_tx_queues_destroy(adapter);
2209 if_destroy:
2210         for (vf = 0; vf < num_vfs; vf++)
2211                 if (adapter->vf_cfg[vf].vf_if_handle)
2212                         be_cmd_if_destroy(adapter,
2213                                         adapter->vf_cfg[vf].vf_if_handle);
2214         be_cmd_if_destroy(adapter, adapter->if_handle);
2215 do_none:
2216         return status;
2217 }
2218
2219 static int be_clear(struct be_adapter *adapter)
2220 {
2221         if (be_physfn(adapter))
2222                 be_vf_eth_addr_rem(adapter);
2223
2224         be_mcc_queues_destroy(adapter);
2225         be_rx_queues_destroy(adapter);
2226         be_tx_queues_destroy(adapter);
2227
2228         be_cmd_if_destroy(adapter, adapter->if_handle);
2229
2230         /* tell fw we're done with firing cmds */
2231         be_cmd_fw_clean(adapter);
2232         return 0;
2233 }
2234
2235
2236 #define FW_FILE_HDR_SIGN        "ServerEngines Corp. "
2237 char flash_cookie[2][16] =      {"*** SE FLAS",
2238                                 "H DIRECTORY *** "};
2239
2240 static bool be_flash_redboot(struct be_adapter *adapter,
2241                         const u8 *p, u32 img_start, int image_size,
2242                         int hdr_size)
2243 {
2244         u32 crc_offset;
2245         u8 flashed_crc[4];
2246         int status;
2247
2248         crc_offset = hdr_size + img_start + image_size - 4;
2249
2250         p += crc_offset;
2251
2252         status = be_cmd_get_flash_crc(adapter, flashed_crc,
2253                         (image_size - 4));
2254         if (status) {
2255                 dev_err(&adapter->pdev->dev,
2256                 "could not get crc from flash, not flashing redboot\n");
2257                 return false;
2258         }
2259
2260         /*update redboot only if crc does not match*/
2261         if (!memcmp(flashed_crc, p, 4))
2262                 return false;
2263         else
2264                 return true;
2265 }
2266
2267 static int be_flash_data(struct be_adapter *adapter,
2268                         const struct firmware *fw,
2269                         struct be_dma_mem *flash_cmd, int num_of_images)
2270
2271 {
2272         int status = 0, i, filehdr_size = 0;
2273         u32 total_bytes = 0, flash_op;
2274         int num_bytes;
2275         const u8 *p = fw->data;
2276         struct be_cmd_write_flashrom *req = flash_cmd->va;
2277         struct flash_comp *pflashcomp;
2278         int num_comp;
2279
2280         struct flash_comp gen3_flash_types[9] = {
2281                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, IMG_TYPE_ISCSI_ACTIVE,
2282                         FLASH_IMAGE_MAX_SIZE_g3},
2283                 { FLASH_REDBOOT_START_g3, IMG_TYPE_REDBOOT,
2284                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3},
2285                 { FLASH_iSCSI_BIOS_START_g3, IMG_TYPE_BIOS,
2286                         FLASH_BIOS_IMAGE_MAX_SIZE_g3},
2287                 { FLASH_PXE_BIOS_START_g3, IMG_TYPE_PXE_BIOS,
2288                         FLASH_BIOS_IMAGE_MAX_SIZE_g3},
2289                 { FLASH_FCoE_BIOS_START_g3, IMG_TYPE_FCOE_BIOS,
2290                         FLASH_BIOS_IMAGE_MAX_SIZE_g3},
2291                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, IMG_TYPE_ISCSI_BACKUP,
2292                         FLASH_IMAGE_MAX_SIZE_g3},
2293                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, IMG_TYPE_FCOE_FW_ACTIVE,
2294                         FLASH_IMAGE_MAX_SIZE_g3},
2295                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, IMG_TYPE_FCOE_FW_BACKUP,
2296                         FLASH_IMAGE_MAX_SIZE_g3},
2297                 { FLASH_NCSI_START_g3, IMG_TYPE_NCSI_FW,
2298                         FLASH_NCSI_IMAGE_MAX_SIZE_g3}
2299         };
2300         struct flash_comp gen2_flash_types[8] = {
2301                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, IMG_TYPE_ISCSI_ACTIVE,
2302                         FLASH_IMAGE_MAX_SIZE_g2},
2303                 { FLASH_REDBOOT_START_g2, IMG_TYPE_REDBOOT,
2304                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2},
2305                 { FLASH_iSCSI_BIOS_START_g2, IMG_TYPE_BIOS,
2306                         FLASH_BIOS_IMAGE_MAX_SIZE_g2},
2307                 { FLASH_PXE_BIOS_START_g2, IMG_TYPE_PXE_BIOS,
2308                         FLASH_BIOS_IMAGE_MAX_SIZE_g2},
2309                 { FLASH_FCoE_BIOS_START_g2, IMG_TYPE_FCOE_BIOS,
2310                         FLASH_BIOS_IMAGE_MAX_SIZE_g2},
2311                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, IMG_TYPE_ISCSI_BACKUP,
2312                         FLASH_IMAGE_MAX_SIZE_g2},
2313                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, IMG_TYPE_FCOE_FW_ACTIVE,
2314                         FLASH_IMAGE_MAX_SIZE_g2},
2315                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, IMG_TYPE_FCOE_FW_BACKUP,
2316                          FLASH_IMAGE_MAX_SIZE_g2}
2317         };
2318
2319         if (adapter->generation == BE_GEN3) {
2320                 pflashcomp = gen3_flash_types;
2321                 filehdr_size = sizeof(struct flash_file_hdr_g3);
2322                 num_comp = 9;
2323         } else {
2324                 pflashcomp = gen2_flash_types;
2325                 filehdr_size = sizeof(struct flash_file_hdr_g2);
2326                 num_comp = 8;
2327         }
2328         for (i = 0; i < num_comp; i++) {
2329                 if ((pflashcomp[i].optype == IMG_TYPE_NCSI_FW) &&
2330                                 memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
2331                         continue;
2332                 if ((pflashcomp[i].optype == IMG_TYPE_REDBOOT) &&
2333                         (!be_flash_redboot(adapter, fw->data,
2334                          pflashcomp[i].offset, pflashcomp[i].size,
2335                          filehdr_size)))
2336                         continue;
2337                 p = fw->data;
2338                 p += filehdr_size + pflashcomp[i].offset
2339                         + (num_of_images * sizeof(struct image_hdr));
2340         if (p + pflashcomp[i].size > fw->data + fw->size)
2341                 return -1;
2342         total_bytes = pflashcomp[i].size;
2343                 while (total_bytes) {
2344                         if (total_bytes > 32*1024)
2345                                 num_bytes = 32*1024;
2346                         else
2347                                 num_bytes = total_bytes;
2348                         total_bytes -= num_bytes;
2349
2350                         if (!total_bytes)
2351                                 flash_op = FLASHROM_OPER_FLASH;
2352                         else
2353                                 flash_op = FLASHROM_OPER_SAVE;
2354                         memcpy(req->params.data_buf, p, num_bytes);
2355                         p += num_bytes;
2356                         status = be_cmd_write_flashrom(adapter, flash_cmd,
2357                                 pflashcomp[i].optype, flash_op, num_bytes);
2358                         if (status) {
2359                                 dev_err(&adapter->pdev->dev,
2360                                         "cmd to write to flash rom failed.\n");
2361                                 return -1;
2362                         }
2363                         yield();
2364                 }
2365         }
2366         return 0;
2367 }
2368
2369 static int get_ufigen_type(struct flash_file_hdr_g2 *fhdr)
2370 {
2371         if (fhdr == NULL)
2372                 return 0;
2373         if (fhdr->build[0] == '3')
2374                 return BE_GEN3;
2375         else if (fhdr->build[0] == '2')
2376                 return BE_GEN2;
2377         else
2378                 return 0;
2379 }
2380
2381 int be_load_fw(struct be_adapter *adapter, u8 *func)
2382 {
2383         char fw_file[ETHTOOL_FLASH_MAX_FILENAME];
2384         const struct firmware *fw;
2385         struct flash_file_hdr_g2 *fhdr;
2386         struct flash_file_hdr_g3 *fhdr3;
2387         struct image_hdr *img_hdr_ptr = NULL;
2388         struct be_dma_mem flash_cmd;
2389         int status, i = 0, num_imgs = 0;
2390         const u8 *p;
2391
2392         strcpy(fw_file, func);
2393
2394         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
2395         if (status)
2396                 goto fw_exit;
2397
2398         p = fw->data;
2399         fhdr = (struct flash_file_hdr_g2 *) p;
2400         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
2401
2402         flash_cmd.size = sizeof(struct be_cmd_write_flashrom) + 32*1024;
2403         flash_cmd.va = pci_alloc_consistent(adapter->pdev, flash_cmd.size,
2404                                         &flash_cmd.dma);
2405         if (!flash_cmd.va) {
2406                 status = -ENOMEM;
2407                 dev_err(&adapter->pdev->dev,
2408                         "Memory allocation failure while flashing\n");
2409                 goto fw_exit;
2410         }
2411
2412         if ((adapter->generation == BE_GEN3) &&
2413                         (get_ufigen_type(fhdr) == BE_GEN3)) {
2414                 fhdr3 = (struct flash_file_hdr_g3 *) fw->data;
2415                 num_imgs = le32_to_cpu(fhdr3->num_imgs);
2416                 for (i = 0; i < num_imgs; i++) {
2417                         img_hdr_ptr = (struct image_hdr *) (fw->data +
2418                                         (sizeof(struct flash_file_hdr_g3) +
2419                                          i * sizeof(struct image_hdr)));
2420                         if (le32_to_cpu(img_hdr_ptr->imageid) == 1)
2421                                 status = be_flash_data(adapter, fw, &flash_cmd,
2422                                                         num_imgs);
2423                 }
2424         } else if ((adapter->generation == BE_GEN2) &&
2425                         (get_ufigen_type(fhdr) == BE_GEN2)) {
2426                 status = be_flash_data(adapter, fw, &flash_cmd, 0);
2427         } else {
2428                 dev_err(&adapter->pdev->dev,
2429                         "UFI and Interface are not compatible for flashing\n");
2430                 status = -1;
2431         }
2432
2433         pci_free_consistent(adapter->pdev, flash_cmd.size, flash_cmd.va,
2434                                 flash_cmd.dma);
2435         if (status) {
2436                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
2437                 goto fw_exit;
2438         }
2439
2440         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
2441
2442 fw_exit:
2443         release_firmware(fw);
2444         return status;
2445 }
2446
2447 static struct net_device_ops be_netdev_ops = {
2448         .ndo_open               = be_open,
2449         .ndo_stop               = be_close,
2450         .ndo_start_xmit         = be_xmit,
2451         .ndo_set_rx_mode        = be_set_multicast_list,
2452         .ndo_set_mac_address    = be_mac_addr_set,
2453         .ndo_change_mtu         = be_change_mtu,
2454         .ndo_validate_addr      = eth_validate_addr,
2455         .ndo_vlan_rx_register   = be_vlan_register,
2456         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
2457         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
2458         .ndo_set_vf_mac         = be_set_vf_mac,
2459         .ndo_set_vf_vlan        = be_set_vf_vlan,
2460         .ndo_set_vf_tx_rate     = be_set_vf_tx_rate,
2461         .ndo_get_vf_config      = be_get_vf_config
2462 };
2463
2464 static void be_netdev_init(struct net_device *netdev)
2465 {
2466         struct be_adapter *adapter = netdev_priv(netdev);
2467
2468         netdev->features |= NETIF_F_SG | NETIF_F_HW_VLAN_RX | NETIF_F_TSO |
2469                 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_FILTER | NETIF_F_HW_CSUM |
2470                 NETIF_F_GRO | NETIF_F_TSO6;
2471
2472         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_CSUM;
2473
2474         netdev->flags |= IFF_MULTICAST;
2475
2476         adapter->rx_csum = true;
2477
2478         /* Default settings for Rx and Tx flow control */
2479         adapter->rx_fc = true;
2480         adapter->tx_fc = true;
2481
2482         netif_set_gso_max_size(netdev, 65535);
2483
2484         BE_SET_NETDEV_OPS(netdev, &be_netdev_ops);
2485
2486         SET_ETHTOOL_OPS(netdev, &be_ethtool_ops);
2487
2488         netif_napi_add(netdev, &adapter->rx_eq.napi, be_poll_rx,
2489                 BE_NAPI_WEIGHT);
2490         netif_napi_add(netdev, &adapter->tx_eq.napi, be_poll_tx_mcc,
2491                 BE_NAPI_WEIGHT);
2492
2493         netif_carrier_off(netdev);
2494         netif_stop_queue(netdev);
2495 }
2496
2497 static void be_unmap_pci_bars(struct be_adapter *adapter)
2498 {
2499         if (adapter->csr)
2500                 iounmap(adapter->csr);
2501         if (adapter->db)
2502                 iounmap(adapter->db);
2503         if (adapter->pcicfg && be_physfn(adapter))
2504                 iounmap(adapter->pcicfg);
2505 }
2506
2507 static int be_map_pci_bars(struct be_adapter *adapter)
2508 {
2509         u8 __iomem *addr;
2510         int pcicfg_reg, db_reg;
2511
2512         if (be_physfn(adapter)) {
2513                 addr = ioremap_nocache(pci_resource_start(adapter->pdev, 2),
2514                                 pci_resource_len(adapter->pdev, 2));
2515                 if (addr == NULL)
2516                         return -ENOMEM;
2517                 adapter->csr = addr;
2518         }
2519
2520         if (adapter->generation == BE_GEN2) {
2521                 pcicfg_reg = 1;
2522                 db_reg = 4;
2523         } else {
2524                 pcicfg_reg = 0;
2525                 if (be_physfn(adapter))
2526                         db_reg = 4;
2527                 else
2528                         db_reg = 0;
2529         }
2530         addr = ioremap_nocache(pci_resource_start(adapter->pdev, db_reg),
2531                                 pci_resource_len(adapter->pdev, db_reg));
2532         if (addr == NULL)
2533                 goto pci_map_err;
2534         adapter->db = addr;
2535
2536         if (be_physfn(adapter)) {
2537                 addr = ioremap_nocache(
2538                                 pci_resource_start(adapter->pdev, pcicfg_reg),
2539                                 pci_resource_len(adapter->pdev, pcicfg_reg));
2540                 if (addr == NULL)
2541                         goto pci_map_err;
2542                 adapter->pcicfg = addr;
2543         } else
2544                 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
2545
2546         return 0;
2547 pci_map_err:
2548         be_unmap_pci_bars(adapter);
2549         return -ENOMEM;
2550 }
2551
2552
2553 static void be_ctrl_cleanup(struct be_adapter *adapter)
2554 {
2555         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
2556
2557         be_unmap_pci_bars(adapter);
2558
2559         if (mem->va)
2560                 pci_free_consistent(adapter->pdev, mem->size,
2561                         mem->va, mem->dma);
2562
2563         mem = &adapter->mc_cmd_mem;
2564         if (mem->va)
2565                 pci_free_consistent(adapter->pdev, mem->size,
2566                         mem->va, mem->dma);
2567 }
2568
2569 static int be_ctrl_init(struct be_adapter *adapter)
2570 {
2571         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
2572         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
2573         struct be_dma_mem *mc_cmd_mem = &adapter->mc_cmd_mem;
2574         int status;
2575
2576         status = be_map_pci_bars(adapter);
2577         if (status)
2578                 goto done;
2579
2580         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
2581         mbox_mem_alloc->va = pci_alloc_consistent(adapter->pdev,
2582                                 mbox_mem_alloc->size, &mbox_mem_alloc->dma);
2583         if (!mbox_mem_alloc->va) {
2584                 status = -ENOMEM;
2585                 goto unmap_pci_bars;
2586         }
2587
2588         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
2589         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
2590         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
2591         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
2592
2593         mc_cmd_mem->size = sizeof(struct be_cmd_req_mcast_mac_config);
2594         mc_cmd_mem->va = pci_alloc_consistent(adapter->pdev, mc_cmd_mem->size,
2595                         &mc_cmd_mem->dma);
2596         if (mc_cmd_mem->va == NULL) {
2597                 status = -ENOMEM;
2598                 goto free_mbox;
2599         }
2600         memset(mc_cmd_mem->va, 0, mc_cmd_mem->size);
2601
2602         spin_lock_init(&adapter->mbox_lock);
2603         spin_lock_init(&adapter->mcc_lock);
2604         spin_lock_init(&adapter->mcc_cq_lock);
2605
2606         init_completion(&adapter->flash_compl);
2607         pci_save_state(adapter->pdev);
2608         return 0;
2609
2610 free_mbox:
2611         pci_free_consistent(adapter->pdev, mbox_mem_alloc->size,
2612                 mbox_mem_alloc->va, mbox_mem_alloc->dma);
2613
2614 unmap_pci_bars:
2615         be_unmap_pci_bars(adapter);
2616
2617 done:
2618         return status;
2619 }
2620
2621 static void be_stats_cleanup(struct be_adapter *adapter)
2622 {
2623         struct be_stats_obj *stats = &adapter->stats;
2624         struct be_dma_mem *cmd = &stats->cmd;
2625
2626         if (cmd->va)
2627                 pci_free_consistent(adapter->pdev, cmd->size,
2628                         cmd->va, cmd->dma);
2629 }
2630
2631 static int be_stats_init(struct be_adapter *adapter)
2632 {
2633         struct be_stats_obj *stats = &adapter->stats;
2634         struct be_dma_mem *cmd = &stats->cmd;
2635
2636         cmd->size = sizeof(struct be_cmd_req_get_stats);
2637         cmd->va = pci_alloc_consistent(adapter->pdev, cmd->size, &cmd->dma);
2638         if (cmd->va == NULL)
2639                 return -1;
2640         memset(cmd->va, 0, cmd->size);
2641         return 0;
2642 }
2643
2644 static void __devexit be_remove(struct pci_dev *pdev)
2645 {
2646         struct be_adapter *adapter = pci_get_drvdata(pdev);
2647
2648         if (!adapter)
2649                 return;
2650
2651         unregister_netdev(adapter->netdev);
2652
2653         be_clear(adapter);
2654
2655         be_stats_cleanup(adapter);
2656
2657         be_ctrl_cleanup(adapter);
2658
2659         be_sriov_disable(adapter);
2660
2661         be_msix_disable(adapter);
2662
2663         pci_set_drvdata(pdev, NULL);
2664         pci_release_regions(pdev);
2665         pci_disable_device(pdev);
2666
2667         free_netdev(adapter->netdev);
2668 }
2669
2670 static int be_get_config(struct be_adapter *adapter)
2671 {
2672         int status;
2673         u8 mac[ETH_ALEN];
2674
2675         status = be_cmd_get_fw_ver(adapter, adapter->fw_ver);
2676         if (status)
2677                 return status;
2678
2679         status = be_cmd_query_fw_cfg(adapter,
2680                                 &adapter->port_num, &adapter->function_mode);
2681         if (status)
2682                 return status;
2683
2684         memset(mac, 0, ETH_ALEN);
2685
2686         if (be_physfn(adapter)) {
2687                 status = be_cmd_mac_addr_query(adapter, mac,
2688                         MAC_ADDRESS_TYPE_NETWORK, true /*permanent */, 0);
2689
2690                 if (status)
2691                         return status;
2692
2693                 if (!is_valid_ether_addr(mac))
2694                         return -EADDRNOTAVAIL;
2695
2696                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
2697                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
2698         }
2699
2700         if (adapter->function_mode & 0x400)
2701                 adapter->max_vlans = BE_NUM_VLANS_SUPPORTED/4;
2702         else
2703                 adapter->max_vlans = BE_NUM_VLANS_SUPPORTED;
2704
2705         return 0;
2706 }
2707
2708 static int __devinit be_probe(struct pci_dev *pdev,
2709                         const struct pci_device_id *pdev_id)
2710 {
2711         int status = 0;
2712         struct be_adapter *adapter;
2713         struct net_device *netdev;
2714
2715
2716         status = pci_enable_device(pdev);
2717         if (status)
2718                 goto do_none;
2719
2720         status = pci_request_regions(pdev, DRV_NAME);
2721         if (status)
2722                 goto disable_dev;
2723         pci_set_master(pdev);
2724
2725         netdev = alloc_etherdev(sizeof(struct be_adapter));
2726         if (netdev == NULL) {
2727                 status = -ENOMEM;
2728                 goto rel_reg;
2729         }
2730         adapter = netdev_priv(netdev);
2731
2732         switch (pdev->device) {
2733         case BE_DEVICE_ID1:
2734         case OC_DEVICE_ID1:
2735                 adapter->generation = BE_GEN2;
2736                 break;
2737         case BE_DEVICE_ID2:
2738         case OC_DEVICE_ID2:
2739                 adapter->generation = BE_GEN3;
2740                 break;
2741         default:
2742                 adapter->generation = 0;
2743         }
2744
2745         adapter->pdev = pdev;
2746         pci_set_drvdata(pdev, adapter);
2747         adapter->netdev = netdev;
2748         be_netdev_init(netdev);
2749         SET_NETDEV_DEV(netdev, &pdev->dev);
2750
2751         be_msix_enable(adapter);
2752
2753         status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2754         if (!status) {
2755                 netdev->features |= NETIF_F_HIGHDMA;
2756         } else {
2757                 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2758                 if (status) {
2759                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
2760                         goto free_netdev;
2761                 }
2762         }
2763
2764         be_sriov_enable(adapter);
2765
2766         status = be_ctrl_init(adapter);
2767         if (status)
2768                 goto free_netdev;
2769
2770         /* sync up with fw's ready state */
2771         if (be_physfn(adapter)) {
2772                 status = be_cmd_POST(adapter);
2773                 if (status)
2774                         goto ctrl_clean;
2775         }
2776
2777         /* tell fw we're ready to fire cmds */
2778         status = be_cmd_fw_init(adapter);
2779         if (status)
2780                 goto ctrl_clean;
2781
2782         if (be_physfn(adapter)) {
2783                 status = be_cmd_reset_function(adapter);
2784                 if (status)
2785                         goto ctrl_clean;
2786         }
2787
2788         status = be_stats_init(adapter);
2789         if (status)
2790                 goto ctrl_clean;
2791
2792         status = be_get_config(adapter);
2793         if (status)
2794                 goto stats_clean;
2795
2796         INIT_DELAYED_WORK(&adapter->work, be_worker);
2797
2798         status = be_setup(adapter);
2799         if (status)
2800                 goto stats_clean;
2801
2802         status = register_netdev(netdev);
2803         if (status != 0)
2804                 goto unsetup;
2805
2806         dev_info(&pdev->dev, "%s port %d\n", nic_name(pdev), adapter->port_num);
2807         return 0;
2808
2809 unsetup:
2810         be_clear(adapter);
2811 stats_clean:
2812         be_stats_cleanup(adapter);
2813 ctrl_clean:
2814         be_ctrl_cleanup(adapter);
2815 free_netdev:
2816         be_msix_disable(adapter);
2817         be_sriov_disable(adapter);
2818         free_netdev(adapter->netdev);
2819         pci_set_drvdata(pdev, NULL);
2820 rel_reg:
2821         pci_release_regions(pdev);
2822 disable_dev:
2823         pci_disable_device(pdev);
2824 do_none:
2825         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
2826         return status;
2827 }
2828
2829 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
2830 {
2831         struct be_adapter *adapter = pci_get_drvdata(pdev);
2832         struct net_device *netdev =  adapter->netdev;
2833
2834         if (adapter->wol)
2835                 be_setup_wol(adapter, true);
2836
2837         netif_device_detach(netdev);
2838         if (netif_running(netdev)) {
2839                 rtnl_lock();
2840                 be_close(netdev);
2841                 rtnl_unlock();
2842         }
2843         be_cmd_get_flow_control(adapter, &adapter->tx_fc, &adapter->rx_fc);
2844         be_clear(adapter);
2845
2846         pci_save_state(pdev);
2847         pci_disable_device(pdev);
2848         pci_set_power_state(pdev, pci_choose_state(pdev, state));
2849         return 0;
2850 }
2851
2852 static int be_resume(struct pci_dev *pdev)
2853 {
2854         int status = 0;
2855         struct be_adapter *adapter = pci_get_drvdata(pdev);
2856         struct net_device *netdev =  adapter->netdev;
2857
2858         netif_device_detach(netdev);
2859
2860         status = pci_enable_device(pdev);
2861         if (status)
2862                 return status;
2863
2864         pci_set_power_state(pdev, 0);
2865         pci_restore_state(pdev);
2866
2867         /* tell fw we're ready to fire cmds */
2868         status = be_cmd_fw_init(adapter);
2869         if (status)
2870                 return status;
2871
2872         be_setup(adapter);
2873         if (netif_running(netdev)) {
2874                 rtnl_lock();
2875                 be_open(netdev);
2876                 rtnl_unlock();
2877         }
2878         netif_device_attach(netdev);
2879
2880         if (adapter->wol)
2881                 be_setup_wol(adapter, false);
2882         return 0;
2883 }
2884
2885 /*
2886  * An FLR will stop BE from DMAing any data.
2887  */
2888 static void be_shutdown(struct pci_dev *pdev)
2889 {
2890         struct be_adapter *adapter = pci_get_drvdata(pdev);
2891         struct net_device *netdev =  adapter->netdev;
2892
2893         netif_device_detach(netdev);
2894
2895         be_cmd_reset_function(adapter);
2896
2897         if (adapter->wol)
2898                 be_setup_wol(adapter, true);
2899
2900         pci_disable_device(pdev);
2901 }
2902
2903 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
2904                                 pci_channel_state_t state)
2905 {
2906         struct be_adapter *adapter = pci_get_drvdata(pdev);
2907         struct net_device *netdev =  adapter->netdev;
2908
2909         dev_err(&adapter->pdev->dev, "EEH error detected\n");
2910
2911         adapter->eeh_err = true;
2912
2913         netif_device_detach(netdev);
2914
2915         if (netif_running(netdev)) {
2916                 rtnl_lock();
2917                 be_close(netdev);
2918                 rtnl_unlock();
2919         }
2920         be_clear(adapter);
2921
2922         if (state == pci_channel_io_perm_failure)
2923                 return PCI_ERS_RESULT_DISCONNECT;
2924
2925         pci_disable_device(pdev);
2926
2927         return PCI_ERS_RESULT_NEED_RESET;
2928 }
2929
2930 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
2931 {
2932         struct be_adapter *adapter = pci_get_drvdata(pdev);
2933         int status;
2934
2935         dev_info(&adapter->pdev->dev, "EEH reset\n");
2936         adapter->eeh_err = false;
2937
2938         status = pci_enable_device(pdev);
2939         if (status)
2940                 return PCI_ERS_RESULT_DISCONNECT;
2941
2942         pci_set_master(pdev);
2943         pci_set_power_state(pdev, 0);
2944         pci_restore_state(pdev);
2945
2946         /* Check if card is ok and fw is ready */
2947         status = be_cmd_POST(adapter);
2948         if (status)
2949                 return PCI_ERS_RESULT_DISCONNECT;
2950
2951         return PCI_ERS_RESULT_RECOVERED;
2952 }
2953
2954 static void be_eeh_resume(struct pci_dev *pdev)
2955 {
2956         int status = 0;
2957         struct be_adapter *adapter = pci_get_drvdata(pdev);
2958         struct net_device *netdev =  adapter->netdev;
2959
2960         dev_info(&adapter->pdev->dev, "EEH resume\n");
2961
2962         pci_save_state(pdev);
2963
2964         /* tell fw we're ready to fire cmds */
2965         status = be_cmd_fw_init(adapter);
2966         if (status)
2967                 goto err;
2968
2969         status = be_setup(adapter);
2970         if (status)
2971                 goto err;
2972
2973         if (netif_running(netdev)) {
2974                 status = be_open(netdev);
2975                 if (status)
2976                         goto err;
2977         }
2978         netif_device_attach(netdev);
2979         return;
2980 err:
2981         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
2982 }
2983
2984 static struct pci_error_handlers be_eeh_handlers = {
2985         .error_detected = be_eeh_err_detected,
2986         .slot_reset = be_eeh_reset,
2987         .resume = be_eeh_resume,
2988 };
2989
2990 static struct pci_driver be_driver = {
2991         .name = DRV_NAME,
2992         .id_table = be_dev_ids,
2993         .probe = be_probe,
2994         .remove = be_remove,
2995         .suspend = be_suspend,
2996         .resume = be_resume,
2997         .shutdown = be_shutdown,
2998         .err_handler = &be_eeh_handlers
2999 };
3000
3001 static int __init be_init_module(void)
3002 {
3003         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
3004             rx_frag_size != 2048) {
3005                 printk(KERN_WARNING DRV_NAME
3006                         " : Module param rx_frag_size must be 2048/4096/8192."
3007                         " Using 2048\n");
3008                 rx_frag_size = 2048;
3009         }
3010
3011         if (num_vfs > 32) {
3012                 printk(KERN_WARNING DRV_NAME
3013                         " : Module param num_vfs must not be greater than 32."
3014                         "Using 32\n");
3015                 num_vfs = 32;
3016         }
3017
3018         return pci_register_driver(&be_driver);
3019 }
3020 module_init(be_init_module);
3021
3022 static void __exit be_exit_module(void)
3023 {
3024         pci_unregister_driver(&be_driver);
3025 }
3026 module_exit(be_exit_module);