drivers/net/bnx2x/bnx2x_main.c

   1 /* bnx2x_main.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <linux/kernel.h>
  21 #include <linux/device.h>  /* for dev_info() */
  22 #include <linux/timer.h>
  23 #include <linux/errno.h>
  24 #include <linux/ioport.h>
  25 #include <linux/slab.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/pci.h>
  29 #include <linux/init.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/bitops.h>
  35 #include <linux/irq.h>
  36 #include <linux/delay.h>
  37 #include <asm/byteorder.h>
  38 #include <linux/time.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/mii.h>
  41 #include <linux/if_vlan.h>
  42 #include <net/ip.h>
  43 #include <net/tcp.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/crc32.h>
  48 #include <linux/crc32c.h>
  49 #include <linux/prefetch.h>
  50 #include <linux/zlib.h>
  51 #include <linux/io.h>
  52 #include <linux/stringify.h>
  53
  54
  55 #include "bnx2x.h"
  56 #include "bnx2x_init.h"
  57 #include "bnx2x_init_ops.h"
  58 #include "bnx2x_dump.h"
  59
  60 #define DRV_MODULE_VERSION      "1.52.53-1"
  61 #define DRV_MODULE_RELDATE      "2010/18/04"
  62 #define BNX2X_BC_VER            0x040200
  63
  64 #include <linux/firmware.h>
  65 #include "bnx2x_fw_file_hdr.h"
  66 /* FW files */
  67 #define FW_FILE_VERSION                                 \
  68         __stringify(BCM_5710_FW_MAJOR_VERSION) "."      \
  69         __stringify(BCM_5710_FW_MINOR_VERSION) "."      \
  70         __stringify(BCM_5710_FW_REVISION_VERSION) "."   \
  71         __stringify(BCM_5710_FW_ENGINEERING_VERSION)
  72 #define FW_FILE_NAME_E1         "bnx2x-e1-" FW_FILE_VERSION ".fw"
  73 #define FW_FILE_NAME_E1H        "bnx2x-e1h-" FW_FILE_VERSION ".fw"
  74
  75 /* Time in jiffies before concluding the transmitter is hung */
  76 #define TX_TIMEOUT              (5*HZ)
  77
  78 static char version[] __devinitdata =
  79         "Broadcom NetXtreme II 5771x 10Gigabit Ethernet Driver "
  80         DRV_MODULE_NAME " " DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  81
  82 MODULE_AUTHOR("Eliezer Tamir");
  83 MODULE_DESCRIPTION("Broadcom NetXtreme II BCM57710/57711/57711E Driver");
  84 MODULE_LICENSE("GPL");
  85 MODULE_VERSION(DRV_MODULE_VERSION);
  86 MODULE_FIRMWARE(FW_FILE_NAME_E1);
  87 MODULE_FIRMWARE(FW_FILE_NAME_E1H);
  88
  89 static int multi_mode = 1;
  90 module_param(multi_mode, int, 0);
  91 MODULE_PARM_DESC(multi_mode, " Multi queue mode "
  92                              "(0 Disable; 1 Enable (default))");
  93
  94 static int num_queues;
  95 module_param(num_queues, int, 0);
  96 MODULE_PARM_DESC(num_queues, " Number of queues for multi_mode=1"
  97                                 " (default is as a number of CPUs)");
  98
  99 static int disable_tpa;
 100 module_param(disable_tpa, int, 0);
 101 MODULE_PARM_DESC(disable_tpa, " Disable the TPA (LRO) feature");
 102
 103 static int int_mode;
 104 module_param(int_mode, int, 0);
 105 MODULE_PARM_DESC(int_mode, " Force interrupt mode other then MSI-X "
 106                                 "(1 INT#x; 2 MSI)");
 107
 108 static int dropless_fc;
 109 module_param(dropless_fc, int, 0);
 110 MODULE_PARM_DESC(dropless_fc, " Pause on exhausted host ring");
 111
 112 static int poll;
 113 module_param(poll, int, 0);
 114 MODULE_PARM_DESC(poll, " Use polling (for debug)");
 115
 116 static int mrrs = -1;
 117 module_param(mrrs, int, 0);
 118 MODULE_PARM_DESC(mrrs, " Force Max Read Req Size (0..3) (for debug)");
 119
 120 static int debug;
 121 module_param(debug, int, 0);
 122 MODULE_PARM_DESC(debug, " Default debug msglevel");
 123
 124 static int load_count[3]; /* 0-common, 1-port0, 2-port1 */
 125
 126 static struct workqueue_struct *bnx2x_wq;
 127
 128 enum bnx2x_board_type {
 129         BCM57710 = 0,
 130         BCM57711 = 1,
 131         BCM57711E = 2,
 132 };
 133
 134 /* indexed by board_type, above */
 135 static struct {
 136         char *name;
 137 } board_info[] __devinitdata = {
 138         { "Broadcom NetXtreme II BCM57710 XGb" },
 139         { "Broadcom NetXtreme II BCM57711 XGb" },
 140         { "Broadcom NetXtreme II BCM57711E XGb" }
 141 };
 142
 143
 144 static DEFINE_PCI_DEVICE_TABLE(bnx2x_pci_tbl) = {
 145         { PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57710), BCM57710 },
 146         { PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711), BCM57711 },
 147         { PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711E), BCM57711E },
 148         { 0 }
 149 };
 150
 151 MODULE_DEVICE_TABLE(pci, bnx2x_pci_tbl);
 152
 153 /****************************************************************************
 154 * General service functions
 155 ****************************************************************************/
 156
 157 /* used only at init
 158  * locking is done by mcp
 159  */
 160 void bnx2x_reg_wr_ind(struct bnx2x *bp, u32 addr, u32 val)
 161 {
 162         pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS, addr);
 163         pci_write_config_dword(bp->pdev, PCICFG_GRC_DATA, val);
 164         pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS,
 165                                PCICFG_VENDOR_ID_OFFSET);
 166 }
 167
 168 static u32 bnx2x_reg_rd_ind(struct bnx2x *bp, u32 addr)
 169 {
 170         u32 val;
 171
 172         pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS, addr);
 173         pci_read_config_dword(bp->pdev, PCICFG_GRC_DATA, &val);
 174         pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS,
 175                                PCICFG_VENDOR_ID_OFFSET);
 176
 177         return val;
 178 }
 179
 180 static const u32 dmae_reg_go_c[] = {
 181         DMAE_REG_GO_C0, DMAE_REG_GO_C1, DMAE_REG_GO_C2, DMAE_REG_GO_C3,
 182         DMAE_REG_GO_C4, DMAE_REG_GO_C5, DMAE_REG_GO_C6, DMAE_REG_GO_C7,
 183         DMAE_REG_GO_C8, DMAE_REG_GO_C9, DMAE_REG_GO_C10, DMAE_REG_GO_C11,
 184         DMAE_REG_GO_C12, DMAE_REG_GO_C13, DMAE_REG_GO_C14, DMAE_REG_GO_C15
 185 };
 186
 187 /* copy command into DMAE command memory and set DMAE command go */
 188 static void bnx2x_post_dmae(struct bnx2x *bp, struct dmae_command *dmae,
 189                             int idx)
 190 {
 191         u32 cmd_offset;
 192         int i;
 193
 194         cmd_offset = (DMAE_REG_CMD_MEM + sizeof(struct dmae_command) * idx);
 195         for (i = 0; i < (sizeof(struct dmae_command)/4); i++) {
 196                 REG_WR(bp, cmd_offset + i*4, *(((u32 *)dmae) + i));
 197
 198                 DP(BNX2X_MSG_OFF, "DMAE cmd[%d].%d (0x%08x) : 0x%08x\n",
 199                    idx, i, cmd_offset + i*4, *(((u32 *)dmae) + i));
 200         }
 201         REG_WR(bp, dmae_reg_go_c[idx], 1);
 202 }
 203
 204 void bnx2x_write_dmae(struct bnx2x *bp, dma_addr_t dma_addr, u32 dst_addr,
 205                       u32 len32)
 206 {
 207         struct dmae_command dmae;
 208         u32 *wb_comp = bnx2x_sp(bp, wb_comp);
 209         int cnt = 200;
 210
 211         if (!bp->dmae_ready) {
 212                 u32 *data = bnx2x_sp(bp, wb_data[0]);
 213
 214                 DP(BNX2X_MSG_OFF, "DMAE is not ready (dst_addr %08x  len32 %d)"
 215                    "  using indirect\n", dst_addr, len32);
 216                 bnx2x_init_ind_wr(bp, dst_addr, data, len32);
 217                 return;
 218         }
 219
 220         memset(&dmae, 0, sizeof(struct dmae_command));
 221
 222         dmae.opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
 223                        DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
 224                        DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
 225 #ifdef __BIG_ENDIAN
 226                        DMAE_CMD_ENDIANITY_B_DW_SWAP |
 227 #else
 228                        DMAE_CMD_ENDIANITY_DW_SWAP |
 229 #endif
 230                        (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
 231                        (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
 232         dmae.src_addr_lo = U64_LO(dma_addr);
 233         dmae.src_addr_hi = U64_HI(dma_addr);
 234         dmae.dst_addr_lo = dst_addr >> 2;
 235         dmae.dst_addr_hi = 0;
 236         dmae.len = len32;
 237         dmae.comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_comp));
 238         dmae.comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_comp));
 239         dmae.comp_val = DMAE_COMP_VAL;
 240
 241         DP(BNX2X_MSG_OFF, "DMAE: opcode 0x%08x\n"
 242            DP_LEVEL "src_addr  [%x:%08x]  len [%d *4]  "
 243                     "dst_addr [%x:%08x (%08x)]\n"
 244            DP_LEVEL "comp_addr [%x:%08x]  comp_val 0x%08x\n",
 245            dmae.opcode, dmae.src_addr_hi, dmae.src_addr_lo,
 246            dmae.len, dmae.dst_addr_hi, dmae.dst_addr_lo, dst_addr,
 247            dmae.comp_addr_hi, dmae.comp_addr_lo, dmae.comp_val);
 248         DP(BNX2X_MSG_OFF, "data [0x%08x 0x%08x 0x%08x 0x%08x]\n",
 249            bp->slowpath->wb_data[0], bp->slowpath->wb_data[1],
 250            bp->slowpath->wb_data[2], bp->slowpath->wb_data[3]);
 251
 252         mutex_lock(&bp->dmae_mutex);
 253
 254         *wb_comp = 0;
 255
 256         bnx2x_post_dmae(bp, &dmae, INIT_DMAE_C(bp));
 257
 258         udelay(5);
 259
 260         while (*wb_comp != DMAE_COMP_VAL) {
 261                 DP(BNX2X_MSG_OFF, "wb_comp 0x%08x\n", *wb_comp);
 262
 263                 if (!cnt) {
 264                         BNX2X_ERR("DMAE timeout!\n");
 265                         break;
 266                 }
 267                 cnt--;
 268                 /* adjust delay for emulation/FPGA */
 269                 if (CHIP_REV_IS_SLOW(bp))
 270                         msleep(100);
 271                 else
 272                         udelay(5);
 273         }
 274
 275         mutex_unlock(&bp->dmae_mutex);
 276 }
 277
 278 void bnx2x_read_dmae(struct bnx2x *bp, u32 src_addr, u32 len32)
 279 {
 280         struct dmae_command dmae;
 281         u32 *wb_comp = bnx2x_sp(bp, wb_comp);
 282         int cnt = 200;
 283
 284         if (!bp->dmae_ready) {
 285                 u32 *data = bnx2x_sp(bp, wb_data[0]);
 286                 int i;
 287
 288                 DP(BNX2X_MSG_OFF, "DMAE is not ready (src_addr %08x  len32 %d)"
 289                    "  using indirect\n", src_addr, len32);
 290                 for (i = 0; i < len32; i++)
 291                         data[i] = bnx2x_reg_rd_ind(bp, src_addr + i*4);
 292                 return;
 293         }
 294
 295         memset(&dmae, 0, sizeof(struct dmae_command));
 296
 297         dmae.opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
 298                        DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
 299                        DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
 300 #ifdef __BIG_ENDIAN
 301                        DMAE_CMD_ENDIANITY_B_DW_SWAP |
 302 #else
 303                        DMAE_CMD_ENDIANITY_DW_SWAP |
 304 #endif
 305                        (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
 306                        (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
 307         dmae.src_addr_lo = src_addr >> 2;
 308         dmae.src_addr_hi = 0;
 309         dmae.dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_data));
 310         dmae.dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_data));
 311         dmae.len = len32;
 312         dmae.comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_comp));
 313         dmae.comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_comp));
 314         dmae.comp_val = DMAE_COMP_VAL;
 315
 316         DP(BNX2X_MSG_OFF, "DMAE: opcode 0x%08x\n"
 317            DP_LEVEL "src_addr  [%x:%08x]  len [%d *4]  "
 318                     "dst_addr [%x:%08x (%08x)]\n"
 319            DP_LEVEL "comp_addr [%x:%08x]  comp_val 0x%08x\n",
 320            dmae.opcode, dmae.src_addr_hi, dmae.src_addr_lo,
 321            dmae.len, dmae.dst_addr_hi, dmae.dst_addr_lo, src_addr,
 322            dmae.comp_addr_hi, dmae.comp_addr_lo, dmae.comp_val);
 323
 324         mutex_lock(&bp->dmae_mutex);
 325
 326         memset(bnx2x_sp(bp, wb_data[0]), 0, sizeof(u32) * 4);
 327         *wb_comp = 0;
 328
 329         bnx2x_post_dmae(bp, &dmae, INIT_DMAE_C(bp));
 330
 331         udelay(5);
 332
 333         while (*wb_comp != DMAE_COMP_VAL) {
 334
 335                 if (!cnt) {
 336                         BNX2X_ERR("DMAE timeout!\n");
 337                         break;
 338                 }
 339                 cnt--;
 340                 /* adjust delay for emulation/FPGA */
 341                 if (CHIP_REV_IS_SLOW(bp))
 342                         msleep(100);
 343                 else
 344                         udelay(5);
 345         }
 346         DP(BNX2X_MSG_OFF, "data [0x%08x 0x%08x 0x%08x 0x%08x]\n",
 347            bp->slowpath->wb_data[0], bp->slowpath->wb_data[1],
 348            bp->slowpath->wb_data[2], bp->slowpath->wb_data[3]);
 349
 350         mutex_unlock(&bp->dmae_mutex);
 351 }
 352
 353 void bnx2x_write_dmae_phys_len(struct bnx2x *bp, dma_addr_t phys_addr,
 354                                u32 addr, u32 len)
 355 {
 356         int dmae_wr_max = DMAE_LEN32_WR_MAX(bp);
 357         int offset = 0;
 358
 359         while (len > dmae_wr_max) {
 360                 bnx2x_write_dmae(bp, phys_addr + offset,
 361                                  addr + offset, dmae_wr_max);
 362                 offset += dmae_wr_max * 4;
 363                 len -= dmae_wr_max;
 364         }
 365
 366         bnx2x_write_dmae(bp, phys_addr + offset, addr + offset, len);
 367 }
 368
 369 /* used only for slowpath so not inlined */
 370 static void bnx2x_wb_wr(struct bnx2x *bp, int reg, u32 val_hi, u32 val_lo)
 371 {
 372         u32 wb_write[2];
 373
 374         wb_write[0] = val_hi;
 375         wb_write[1] = val_lo;
 376         REG_WR_DMAE(bp, reg, wb_write, 2);
 377 }
 378
 379 #ifdef USE_WB_RD
 380 static u64 bnx2x_wb_rd(struct bnx2x *bp, int reg)
 381 {
 382         u32 wb_data[2];
 383
 384         REG_RD_DMAE(bp, reg, wb_data, 2);
 385
 386         return HILO_U64(wb_data[0], wb_data[1]);
 387 }
 388 #endif
 389
 390 static int bnx2x_mc_assert(struct bnx2x *bp)
 391 {
 392         char last_idx;
 393         int i, rc = 0;
 394         u32 row0, row1, row2, row3;
 395
 396         /* XSTORM */
 397         last_idx = REG_RD8(bp, BAR_XSTRORM_INTMEM +
 398                            XSTORM_ASSERT_LIST_INDEX_OFFSET);
 399         if (last_idx)
 400                 BNX2X_ERR("XSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);
 401
 402         /* print the asserts */
 403         for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {
 404
 405                 row0 = REG_RD(bp, BAR_XSTRORM_INTMEM +
 406                               XSTORM_ASSERT_LIST_OFFSET(i));
 407                 row1 = REG_RD(bp, BAR_XSTRORM_INTMEM +
 408                               XSTORM_ASSERT_LIST_OFFSET(i) + 4);
 409                 row2 = REG_RD(bp, BAR_XSTRORM_INTMEM +
 410                               XSTORM_ASSERT_LIST_OFFSET(i) + 8);
 411                 row3 = REG_RD(bp, BAR_XSTRORM_INTMEM +
 412                               XSTORM_ASSERT_LIST_OFFSET(i) + 12);
 413
 414                 if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
 415                         BNX2X_ERR("XSTORM_ASSERT_INDEX 0x%x = 0x%08x"
 416                                   " 0x%08x 0x%08x 0x%08x\n",
 417                                   i, row3, row2, row1, row0);
 418                         rc++;
 419                 } else {
 420                         break;
 421                 }
 422         }
 423
 424         /* TSTORM */
 425         last_idx = REG_RD8(bp, BAR_TSTRORM_INTMEM +
 426                            TSTORM_ASSERT_LIST_INDEX_OFFSET);
 427         if (last_idx)
 428                 BNX2X_ERR("TSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);
 429
 430         /* print the asserts */
 431         for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {
 432
 433                 row0 = REG_RD(bp, BAR_TSTRORM_INTMEM +
 434                               TSTORM_ASSERT_LIST_OFFSET(i));
 435                 row1 = REG_RD(bp, BAR_TSTRORM_INTMEM +
 436                               TSTORM_ASSERT_LIST_OFFSET(i) + 4);
 437                 row2 = REG_RD(bp, BAR_TSTRORM_INTMEM +
 438                               TSTORM_ASSERT_LIST_OFFSET(i) + 8);
 439                 row3 = REG_RD(bp, BAR_TSTRORM_INTMEM +
 440                               TSTORM_ASSERT_LIST_OFFSET(i) + 12);
 441
 442                 if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
 443                         BNX2X_ERR("TSTORM_ASSERT_INDEX 0x%x = 0x%08x"
 444                                   " 0x%08x 0x%08x 0x%08x\n",
 445                                   i, row3, row2, row1, row0);
 446                         rc++;
 447                 } else {
 448                         break;
 449                 }
 450         }
 451
 452         /* CSTORM */
 453         last_idx = REG_RD8(bp, BAR_CSTRORM_INTMEM +
 454                            CSTORM_ASSERT_LIST_INDEX_OFFSET);
 455         if (last_idx)
 456                 BNX2X_ERR("CSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);
 457
 458         /* print the asserts */
 459         for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {
 460
 461                 row0 = REG_RD(bp, BAR_CSTRORM_INTMEM +
 462                               CSTORM_ASSERT_LIST_OFFSET(i));
 463                 row1 = REG_RD(bp, BAR_CSTRORM_INTMEM +
 464                               CSTORM_ASSERT_LIST_OFFSET(i) + 4);
 465                 row2 = REG_RD(bp, BAR_CSTRORM_INTMEM +
 466                               CSTORM_ASSERT_LIST_OFFSET(i) + 8);
 467                 row3 = REG_RD(bp, BAR_CSTRORM_INTMEM +
 468                               CSTORM_ASSERT_LIST_OFFSET(i) + 12);
 469
 470                 if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
 471                         BNX2X_ERR("CSTORM_ASSERT_INDEX 0x%x = 0x%08x"
 472                                   " 0x%08x 0x%08x 0x%08x\n",
 473                                   i, row3, row2, row1, row0);
 474                         rc++;
 475                 } else {
 476                         break;
 477                 }
 478         }
 479
 480         /* USTORM */
 481         last_idx = REG_RD8(bp, BAR_USTRORM_INTMEM +
 482                            USTORM_ASSERT_LIST_INDEX_OFFSET);
 483         if (last_idx)
 484                 BNX2X_ERR("USTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);
 485
 486         /* print the asserts */
 487         for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {
 488
 489                 row0 = REG_RD(bp, BAR_USTRORM_INTMEM +
 490                               USTORM_ASSERT_LIST_OFFSET(i));
 491                 row1 = REG_RD(bp, BAR_USTRORM_INTMEM +
 492                               USTORM_ASSERT_LIST_OFFSET(i) + 4);
 493                 row2 = REG_RD(bp, BAR_USTRORM_INTMEM +
 494                               USTORM_ASSERT_LIST_OFFSET(i) + 8);
 495                 row3 = REG_RD(bp, BAR_USTRORM_INTMEM +
 496                               USTORM_ASSERT_LIST_OFFSET(i) + 12);
 497
 498                 if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
 499                         BNX2X_ERR("USTORM_ASSERT_INDEX 0x%x = 0x%08x"
 500                                   " 0x%08x 0x%08x 0x%08x\n",
 501                                   i, row3, row2, row1, row0);
 502                         rc++;
 503                 } else {
 504                         break;
 505                 }
 506         }
 507
 508         return rc;
 509 }
 510
 511 static void bnx2x_fw_dump(struct bnx2x *bp)
 512 {
 513         u32 addr;
 514         u32 mark, offset;
 515         __be32 data[9];
 516         int word;
 517
 518         if (BP_NOMCP(bp)) {
 519                 BNX2X_ERR("NO MCP - can not dump\n");
 520                 return;
 521         }
 522
 523         addr = bp->common.shmem_base - 0x0800 + 4;
 524         mark = REG_RD(bp, addr);
 525         mark = MCP_REG_MCPR_SCRATCH + ((mark + 0x3) & ~0x3) - 0x08000000;
 526         pr_err("begin fw dump (mark 0x%x)\n", mark);
 527
 528         pr_err("");
 529         for (offset = mark; offset <= bp->common.shmem_base; offset += 0x8*4) {
 530                 for (word = 0; word < 8; word++)
 531                         data[word] = htonl(REG_RD(bp, offset + 4*word));
 532                 data[8] = 0x0;
 533                 pr_cont("%s", (char *)data);
 534         }
 535         for (offset = addr + 4; offset <= mark; offset += 0x8*4) {
 536                 for (word = 0; word < 8; word++)
 537                         data[word] = htonl(REG_RD(bp, offset + 4*word));
 538                 data[8] = 0x0;
 539                 pr_cont("%s", (char *)data);
 540         }
 541         pr_err("end of fw dump\n");
 542 }
 543
 544 static void bnx2x_panic_dump(struct bnx2x *bp)
 545 {
 546         int i;
 547         u16 j, start, end;
 548
 549         bp->stats_state = STATS_STATE_DISABLED;
 550         DP(BNX2X_MSG_STATS, "stats_state - DISABLED\n");
 551
 552         BNX2X_ERR("begin crash dump -----------------\n");
 553
 554         /* Indices */
 555         /* Common */
 556         BNX2X_ERR("def_c_idx(0x%x)  def_u_idx(0x%x)  def_x_idx(0x%x)"
 557                   "  def_t_idx(0x%x)  def_att_idx(0x%x)  attn_state(0x%x)"
 558                   "  spq_prod_idx(0x%x)\n",
 559                   bp->def_c_idx, bp->def_u_idx, bp->def_x_idx, bp->def_t_idx,
 560                   bp->def_att_idx, bp->attn_state, bp->spq_prod_idx);
 561
 562         /* Rx */
 563         for_each_queue(bp, i) {
 564                 struct bnx2x_fastpath *fp = &bp->fp[i];
 565
 566                 BNX2X_ERR("fp%d: rx_bd_prod(0x%x)  rx_bd_cons(0x%x)"
 567                           "  *rx_bd_cons_sb(0x%x)  rx_comp_prod(0x%x)"
 568                           "  rx_comp_cons(0x%x)  *rx_cons_sb(0x%x)\n",
 569                           i, fp->rx_bd_prod, fp->rx_bd_cons,
 570                           le16_to_cpu(*fp->rx_bd_cons_sb), fp->rx_comp_prod,
 571                           fp->rx_comp_cons, le16_to_cpu(*fp->rx_cons_sb));
 572                 BNX2X_ERR("     rx_sge_prod(0x%x)  last_max_sge(0x%x)"
 573                           "  fp_u_idx(0x%x) *sb_u_idx(0x%x)\n",
 574                           fp->rx_sge_prod, fp->last_max_sge,
 575                           le16_to_cpu(fp->fp_u_idx),
 576                           fp->status_blk->u_status_block.status_block_index);
 577         }
 578
 579         /* Tx */
 580         for_each_queue(bp, i) {
 581                 struct bnx2x_fastpath *fp = &bp->fp[i];
 582
 583                 BNX2X_ERR("fp%d: tx_pkt_prod(0x%x)  tx_pkt_cons(0x%x)"
 584                           "  tx_bd_prod(0x%x)  tx_bd_cons(0x%x)"
 585                           "  *tx_cons_sb(0x%x)\n",
 586                           i, fp->tx_pkt_prod, fp->tx_pkt_cons, fp->tx_bd_prod,
 587                           fp->tx_bd_cons, le16_to_cpu(*fp->tx_cons_sb));
 588                 BNX2X_ERR("     fp_c_idx(0x%x)  *sb_c_idx(0x%x)"
 589                           "  tx_db_prod(0x%x)\n", le16_to_cpu(fp->fp_c_idx),
 590                           fp->status_blk->c_status_block.status_block_index,
 591                           fp->tx_db.data.prod);
 592         }
 593
 594         /* Rings */
 595         /* Rx */
 596         for_each_queue(bp, i) {
 597                 struct bnx2x_fastpath *fp = &bp->fp[i];
 598
 599                 start = RX_BD(le16_to_cpu(*fp->rx_cons_sb) - 10);
 600                 end = RX_BD(le16_to_cpu(*fp->rx_cons_sb) + 503);
 601                 for (j = start; j != end; j = RX_BD(j + 1)) {
 602                         u32 *rx_bd = (u32 *)&fp->rx_desc_ring[j];
 603                         struct sw_rx_bd *sw_bd = &fp->rx_buf_ring[j];
 604
 605                         BNX2X_ERR("fp%d: rx_bd[%x]=[%x:%x]  sw_bd=[%p]\n",
 606                                   i, j, rx_bd[1], rx_bd[0], sw_bd->skb);
 607                 }
 608
 609                 start = RX_SGE(fp->rx_sge_prod);
 610                 end = RX_SGE(fp->last_max_sge);
 611                 for (j = start; j != end; j = RX_SGE(j + 1)) {
 612                         u32 *rx_sge = (u32 *)&fp->rx_sge_ring[j];
 613                         struct sw_rx_page *sw_page = &fp->rx_page_ring[j];
 614
 615                         BNX2X_ERR("fp%d: rx_sge[%x]=[%x:%x]  sw_page=[%p]\n",
 616                                   i, j, rx_sge[1], rx_sge[0], sw_page->page);
 617                 }
 618
 619                 start = RCQ_BD(fp->rx_comp_cons - 10);
 620                 end = RCQ_BD(fp->rx_comp_cons + 503);
 621                 for (j = start; j != end; j = RCQ_BD(j + 1)) {
 622                         u32 *cqe = (u32 *)&fp->rx_comp_ring[j];
 623
 624                         BNX2X_ERR("fp%d: cqe[%x]=[%x:%x:%x:%x]\n",
 625                                   i, j, cqe[0], cqe[1], cqe[2], cqe[3]);
 626                 }
 627         }
 628
 629         /* Tx */
 630         for_each_queue(bp, i) {
 631                 struct bnx2x_fastpath *fp = &bp->fp[i];
 632
 633                 start = TX_BD(le16_to_cpu(*fp->tx_cons_sb) - 10);
 634                 end = TX_BD(le16_to_cpu(*fp->tx_cons_sb) + 245);
 635                 for (j = start; j != end; j = TX_BD(j + 1)) {
 636                         struct sw_tx_bd *sw_bd = &fp->tx_buf_ring[j];
 637
 638                         BNX2X_ERR("fp%d: packet[%x]=[%p,%x]\n",
 639                                   i, j, sw_bd->skb, sw_bd->first_bd);
 640                 }
 641
 642                 start = TX_BD(fp->tx_bd_cons - 10);
 643                 end = TX_BD(fp->tx_bd_cons + 254);
 644                 for (j = start; j != end; j = TX_BD(j + 1)) {
 645                         u32 *tx_bd = (u32 *)&fp->tx_desc_ring[j];
 646
 647                         BNX2X_ERR("fp%d: tx_bd[%x]=[%x:%x:%x:%x]\n",
 648                                   i, j, tx_bd[0], tx_bd[1], tx_bd[2], tx_bd[3]);
 649                 }
 650         }
 651
 652         bnx2x_fw_dump(bp);
 653         bnx2x_mc_assert(bp);
 654         BNX2X_ERR("end crash dump -----------------\n");
 655 }
 656
 657 static void bnx2x_int_enable(struct bnx2x *bp)
 658 {
 659         int port = BP_PORT(bp);
 660         u32 addr = port ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0;
 661         u32 val = REG_RD(bp, addr);
 662         int msix = (bp->flags & USING_MSIX_FLAG) ? 1 : 0;
 663         int msi = (bp->flags & USING_MSI_FLAG) ? 1 : 0;
 664
 665         if (msix) {
 666                 val &= ~(HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
 667                          HC_CONFIG_0_REG_INT_LINE_EN_0);
 668                 val |= (HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
 669                         HC_CONFIG_0_REG_ATTN_BIT_EN_0);
 670         } else if (msi) {
 671                 val &= ~HC_CONFIG_0_REG_INT_LINE_EN_0;
 672                 val |= (HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
 673                         HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
 674                         HC_CONFIG_0_REG_ATTN_BIT_EN_0);
 675         } else {
 676                 val |= (HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
 677                         HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
 678                         HC_CONFIG_0_REG_INT_LINE_EN_0 |
 679                         HC_CONFIG_0_REG_ATTN_BIT_EN_0);
 680
 681                 DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)\n",
 682                    val, port, addr);
 683
 684                 REG_WR(bp, addr, val);
 685
 686                 val &= ~HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0;
 687         }
 688
 689         DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)  mode %s\n",
 690            val, port, addr, (msix ? "MSI-X" : (msi ? "MSI" : "INTx")));
 691
 692         REG_WR(bp, addr, val);
 693         /*
 694          * Ensure that HC_CONFIG is written before leading/trailing edge config
 695          */
 696         mmiowb();
 697         barrier();
 698
 699         if (CHIP_IS_E1H(bp)) {
 700                 /* init leading/trailing edge */
 701                 if (IS_E1HMF(bp)) {
 702                         val = (0xee0f | (1 << (BP_E1HVN(bp) + 4)));
 703                         if (bp->port.pmf)
 704                                 /* enable nig and gpio3 attention */
 705                                 val |= 0x1100;
 706                 } else
 707                         val = 0xffff;
 708
 709                 REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, val);
 710                 REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, val);
 711         }
 712
 713         /* Make sure that interrupts are indeed enabled from here on */
 714         mmiowb();
 715 }
 716
 717 static void bnx2x_int_disable(struct bnx2x *bp)
 718 {
 719         int port = BP_PORT(bp);
 720         u32 addr = port ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0;
 721         u32 val = REG_RD(bp, addr);
 722
 723         val &= ~(HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
 724                  HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
 725                  HC_CONFIG_0_REG_INT_LINE_EN_0 |
 726                  HC_CONFIG_0_REG_ATTN_BIT_EN_0);
 727
 728         DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)\n",
 729            val, port, addr);
 730
 731         /* flush all outstanding writes */
 732         mmiowb();
 733
 734         REG_WR(bp, addr, val);
 735         if (REG_RD(bp, addr) != val)
 736                 BNX2X_ERR("BUG! proper val not read from IGU!\n");
 737 }
 738
 739 static void bnx2x_int_disable_sync(struct bnx2x *bp, int disable_hw)
 740 {
 741         int msix = (bp->flags & USING_MSIX_FLAG) ? 1 : 0;
 742         int i, offset;
 743
 744         /* disable interrupt handling */
 745         atomic_inc(&bp->intr_sem);
 746         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
 747
 748         if (disable_hw)
 749                 /* prevent the HW from sending interrupts */
 750                 bnx2x_int_disable(bp);
 751
 752         /* make sure all ISRs are done */
 753         if (msix) {
 754                 synchronize_irq(bp->msix_table[0].vector);
 755                 offset = 1;
 756 #ifdef BCM_CNIC
 757                 offset++;
 758 #endif
 759                 for_each_queue(bp, i)
 760                         synchronize_irq(bp->msix_table[i + offset].vector);
 761         } else
 762                 synchronize_irq(bp->pdev->irq);
 763
 764         /* make sure sp_task is not running */
 765         cancel_delayed_work(&bp->sp_task);
 766         flush_workqueue(bnx2x_wq);
 767 }
 768
 769 /* fast path */
 770
 771 /*
 772  * General service functions
 773  */
 774
 775 /* Return true if succeeded to acquire the lock */
 776 static bool bnx2x_trylock_hw_lock(struct bnx2x *bp, u32 resource)
 777 {
 778         u32 lock_status;
 779         u32 resource_bit = (1 << resource);
 780         int func = BP_FUNC(bp);
 781         u32 hw_lock_control_reg;
 782
 783         DP(NETIF_MSG_HW, "Trying to take a lock on resource %d\n", resource);
 784
 785         /* Validating that the resource is within range */
 786         if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
 787                 DP(NETIF_MSG_HW,
 788                    "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)\n",
 789                    resource, HW_LOCK_MAX_RESOURCE_VALUE);
 790                 return -EINVAL;
 791         }
 792
 793         if (func <= 5)
 794                 hw_lock_control_reg = (MISC_REG_DRIVER_CONTROL_1 + func*8);
 795         else
 796                 hw_lock_control_reg =
 797                                 (MISC_REG_DRIVER_CONTROL_7 + (func - 6)*8);
 798
 799         /* Try to acquire the lock */
 800         REG_WR(bp, hw_lock_control_reg + 4, resource_bit);
 801         lock_status = REG_RD(bp, hw_lock_control_reg);
 802         if (lock_status & resource_bit)
 803                 return true;
 804
 805         DP(NETIF_MSG_HW, "Failed to get a lock on resource %d\n", resource);
 806         return false;
 807 }
 808
 809 static inline void bnx2x_ack_sb(struct bnx2x *bp, u8 sb_id,
 810                                 u8 storm, u16 index, u8 op, u8 update)
 811 {
 812         u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp)*32 +
 813                        COMMAND_REG_INT_ACK);
 814         struct igu_ack_register igu_ack;
 815
 816         igu_ack.status_block_index = index;
 817         igu_ack.sb_id_and_flags =
 818                         ((sb_id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) |
 819                          (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) |
 820                          (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) |
 821                          (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT));
 822
 823         DP(BNX2X_MSG_OFF, "write 0x%08x to HC addr 0x%x\n",
 824            (*(u32 *)&igu_ack), hc_addr);
 825         REG_WR(bp, hc_addr, (*(u32 *)&igu_ack));
 826
 827         /* Make sure that ACK is written */
 828         mmiowb();
 829         barrier();
 830 }
 831
 832 static inline void bnx2x_update_fpsb_idx(struct bnx2x_fastpath *fp)
 833 {
 834         struct host_status_block *fpsb = fp->status_blk;
 835
 836         barrier(); /* status block is written to by the chip */
 837         fp->fp_c_idx = fpsb->c_status_block.status_block_index;
 838         fp->fp_u_idx = fpsb->u_status_block.status_block_index;
 839 }
 840
 841 static u16 bnx2x_ack_int(struct bnx2x *bp)
 842 {
 843         u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp)*32 +
 844                        COMMAND_REG_SIMD_MASK);
 845         u32 result = REG_RD(bp, hc_addr);
 846
 847         DP(BNX2X_MSG_OFF, "read 0x%08x from HC addr 0x%x\n",
 848            result, hc_addr);
 849
 850         return result;
 851 }
 852
 853
 854 /*
 855  * fast path service functions
 856  */
 857
 858 static inline int bnx2x_has_tx_work_unload(struct bnx2x_fastpath *fp)
 859 {
 860         /* Tell compiler that consumer and producer can change */
 861         barrier();
 862         return (fp->tx_pkt_prod != fp->tx_pkt_cons);
 863 }
 864
 865 /* free skb in the packet ring at pos idx
 866  * return idx of last bd freed
 867  */
 868 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 869                              u16 idx)
 870 {
 871         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
 872         struct eth_tx_start_bd *tx_start_bd;
 873         struct eth_tx_bd *tx_data_bd;
 874         struct sk_buff *skb = tx_buf->skb;
 875         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 876         int nbd;
 877
 878         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 879         prefetch(&skb->end);
 880
 881         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
 882            idx, tx_buf, skb);
 883
 884         /* unmap first bd */
 885         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
 886         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
 887         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 888                          BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
 889
 890         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 891 #ifdef BNX2X_STOP_ON_ERROR
 892         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 893                 BNX2X_ERR("BAD nbd!\n");
 894                 bnx2x_panic();
 895         }
 896 #endif
 897         new_cons = nbd + tx_buf->first_bd;
 898
 899         /* Get the next bd */
 900         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 901
 902         /* Skip a parse bd... */
 903         --nbd;
 904         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 905
 906         /* ...and the TSO split header bd since they have no mapping */
 907         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 908                 --nbd;
 909                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 910         }
 911
 912         /* now free frags */
 913         while (nbd > 0) {
 914
 915                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
 916                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
 917                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 918                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 919                 if (--nbd)
 920                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 921         }
 922
 923         /* release skb */
 924         WARN_ON(!skb);
 925         dev_kfree_skb(skb);
 926         tx_buf->first_bd = 0;
 927         tx_buf->skb = NULL;
 928
 929         return new_cons;
 930 }
 931
 932 static inline u16 bnx2x_tx_avail(struct bnx2x_fastpath *fp)
 933 {
 934         s16 used;
 935         u16 prod;
 936         u16 cons;
 937
 938         prod = fp->tx_bd_prod;
 939         cons = fp->tx_bd_cons;
 940
 941         /* NUM_TX_RINGS = number of "next-page" entries
 942            It will be used as a threshold */
 943         used = SUB_S16(prod, cons) + (s16)NUM_TX_RINGS;
 944
 945 #ifdef BNX2X_STOP_ON_ERROR
 946         WARN_ON(used < 0);
 947         WARN_ON(used > fp->bp->tx_ring_size);
 948         WARN_ON((fp->bp->tx_ring_size - used) > MAX_TX_AVAIL);
 949 #endif
 950
 951         return (s16)(fp->bp->tx_ring_size) - used;
 952 }
 953
 954 static inline int bnx2x_has_tx_work(struct bnx2x_fastpath *fp)
 955 {
 956         u16 hw_cons;
 957
 958         /* Tell compiler that status block fields can change */
 959         barrier();
 960         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 961         return hw_cons != fp->tx_pkt_cons;
 962 }
 963
 964 static int bnx2x_tx_int(struct bnx2x_fastpath *fp)
 965 {
 966         struct bnx2x *bp = fp->bp;
 967         struct netdev_queue *txq;
 968         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 969
 970 #ifdef BNX2X_STOP_ON_ERROR
 971         if (unlikely(bp->panic))
 972                 return -1;
 973 #endif
 974
 975         txq = netdev_get_tx_queue(bp->dev, fp->index);
 976         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 977         sw_cons = fp->tx_pkt_cons;
 978
 979         while (sw_cons != hw_cons) {
 980                 u16 pkt_cons;
 981
 982                 pkt_cons = TX_BD(sw_cons);
 983
 984                 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
 985
 986                 DP(NETIF_MSG_TX_DONE, "hw_cons %u  sw_cons %u  pkt_cons %u\n",
 987                    hw_cons, sw_cons, pkt_cons);
 988
 989 /*              if (NEXT_TX_IDX(sw_cons) != hw_cons) {
 990                         rmb();
 991                         prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
 992                 }
 993 */
 994                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 995                 sw_cons++;
 996         }
 997
 998         fp->tx_pkt_cons = sw_cons;
 999         fp->tx_bd_cons = bd_cons;
1000
1001         /* Need to make the tx_bd_cons update visible to start_xmit()
1002          * before checking for netif_tx_queue_stopped().  Without the
1003          * memory barrier, there is a small possibility that
1004          * start_xmit() will miss it and cause the queue to be stopped
1005          * forever.
1006          */
1007         smp_mb();
1008
1009         /* TBD need a thresh? */
1010         if (unlikely(netif_tx_queue_stopped(txq))) {
1011                 /* Taking tx_lock() is needed to prevent reenabling the queue
1012                  * while it's empty. This could have happen if rx_action() gets
1013                  * suspended in bnx2x_tx_int() after the condition before
1014                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
1015                  *
1016                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
1017                  * sends some packets consuming the whole queue again->
1018                  * stops the queue
1019                  */
1020
1021                 __netif_tx_lock(txq, smp_processor_id());
1022
1023                 if ((netif_tx_queue_stopped(txq)) &&
1024                     (bp->state == BNX2X_STATE_OPEN) &&
1025                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
1026                         netif_tx_wake_queue(txq);
1027
1028                 __netif_tx_unlock(txq);
1029         }
1030         return 0;
1031 }
1032
1033 #ifdef BCM_CNIC
1034 static void bnx2x_cnic_cfc_comp(struct bnx2x *bp, int cid);
1035 #endif
1036
1037 static void bnx2x_sp_event(struct bnx2x_fastpath *fp,
1038                            union eth_rx_cqe *rr_cqe)
1039 {
1040         struct bnx2x *bp = fp->bp;
1041         int cid = SW_CID(rr_cqe->ramrod_cqe.conn_and_cmd_data);
1042         int command = CQE_CMD(rr_cqe->ramrod_cqe.conn_and_cmd_data);
1043
1044         DP(BNX2X_MSG_SP,
1045            "fp %d  cid %d  got ramrod #%d  state is %x  type is %d\n",
1046            fp->index, cid, command, bp->state,
1047            rr_cqe->ramrod_cqe.ramrod_type);
1048
1049         bp->spq_left++;
1050
1051         if (fp->index) {
1052                 switch (command | fp->state) {
1053                 case (RAMROD_CMD_ID_ETH_CLIENT_SETUP |
1054                                                 BNX2X_FP_STATE_OPENING):
1055                         DP(NETIF_MSG_IFUP, "got MULTI[%d] setup ramrod\n",
1056                            cid);
1057                         fp->state = BNX2X_FP_STATE_OPEN;
1058                         break;
1059
1060                 case (RAMROD_CMD_ID_ETH_HALT | BNX2X_FP_STATE_HALTING):
1061                         DP(NETIF_MSG_IFDOWN, "got MULTI[%d] halt ramrod\n",
1062                            cid);
1063                         fp->state = BNX2X_FP_STATE_HALTED;
1064                         break;
1065
1066                 default:
1067                         BNX2X_ERR("unexpected MC reply (%d)  "
1068                                   "fp[%d] state is %x\n",
1069                                   command, fp->index, fp->state);
1070                         break;
1071                 }
1072                 mb(); /* force bnx2x_wait_ramrod() to see the change */
1073                 return;
1074         }
1075
1076         switch (command | bp->state) {
1077         case (RAMROD_CMD_ID_ETH_PORT_SETUP | BNX2X_STATE_OPENING_WAIT4_PORT):
1078                 DP(NETIF_MSG_IFUP, "got setup ramrod\n");
1079                 bp->state = BNX2X_STATE_OPEN;
1080                 break;
1081
1082         case (RAMROD_CMD_ID_ETH_HALT | BNX2X_STATE_CLOSING_WAIT4_HALT):
1083                 DP(NETIF_MSG_IFDOWN, "got halt ramrod\n");
1084                 bp->state = BNX2X_STATE_CLOSING_WAIT4_DELETE;
1085                 fp->state = BNX2X_FP_STATE_HALTED;
1086                 break;
1087
1088         case (RAMROD_CMD_ID_ETH_CFC_DEL | BNX2X_STATE_CLOSING_WAIT4_HALT):
1089                 DP(NETIF_MSG_IFDOWN, "got delete ramrod for MULTI[%d]\n", cid);
1090                 bnx2x_fp(bp, cid, state) = BNX2X_FP_STATE_CLOSED;
1091                 break;
1092
1093 #ifdef BCM_CNIC
1094         case (RAMROD_CMD_ID_ETH_CFC_DEL | BNX2X_STATE_OPEN):
1095                 DP(NETIF_MSG_IFDOWN, "got delete ramrod for CID %d\n", cid);
1096                 bnx2x_cnic_cfc_comp(bp, cid);
1097                 break;
1098 #endif
1099
1100         case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_OPEN):
1101         case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_DIAG):
1102                 DP(NETIF_MSG_IFUP, "got set mac ramrod\n");
1103                 bp->set_mac_pending--;
1104                 smp_wmb();
1105                 break;
1106
1107         case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_CLOSING_WAIT4_HALT):
1108                 DP(NETIF_MSG_IFDOWN, "got (un)set mac ramrod\n");
1109                 bp->set_mac_pending--;
1110                 smp_wmb();
1111                 break;
1112
1113         default:
1114                 BNX2X_ERR("unexpected MC reply (%d)  bp->state is %x\n",
1115                           command, bp->state);
1116                 break;
1117         }
1118         mb(); /* force bnx2x_wait_ramrod() to see the change */
1119 }
1120
1121 static inline void bnx2x_free_rx_sge(struct bnx2x *bp,
1122                                      struct bnx2x_fastpath *fp, u16 index)
1123 {
1124         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
1125         struct page *page = sw_buf->page;
1126         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
1127
1128         /* Skip "next page" elements */
1129         if (!page)
1130                 return;
1131
1132         dma_unmap_page(&bp->pdev->dev, dma_unmap_addr(sw_buf, mapping),
1133                        SGE_PAGE_SIZE*PAGES_PER_SGE, PCI_DMA_FROMDEVICE);
1134         __free_pages(page, PAGES_PER_SGE_SHIFT);
1135
1136         sw_buf->page = NULL;
1137         sge->addr_hi = 0;
1138         sge->addr_lo = 0;
1139 }
1140
1141 static inline void bnx2x_free_rx_sge_range(struct bnx2x *bp,
1142                                            struct bnx2x_fastpath *fp, int last)
1143 {
1144         int i;
1145
1146         for (i = 0; i < last; i++)
1147                 bnx2x_free_rx_sge(bp, fp, i);
1148 }
1149
1150 static inline int bnx2x_alloc_rx_sge(struct bnx2x *bp,
1151                                      struct bnx2x_fastpath *fp, u16 index)
1152 {
1153         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
1154         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
1155         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
1156         dma_addr_t mapping;
1157
1158         if (unlikely(page == NULL))
1159                 return -ENOMEM;
1160
1161         mapping = dma_map_page(&bp->pdev->dev, page, 0,
1162                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
1163         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
1164                 __free_pages(page, PAGES_PER_SGE_SHIFT);
1165                 return -ENOMEM;
1166         }
1167
1168         sw_buf->page = page;
1169         dma_unmap_addr_set(sw_buf, mapping, mapping);
1170
1171         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
1172         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
1173
1174         return 0;
1175 }
1176
1177 static inline int bnx2x_alloc_rx_skb(struct bnx2x *bp,
1178                                      struct bnx2x_fastpath *fp, u16 index)
1179 {
1180         struct sk_buff *skb;
1181         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
1182         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
1183         dma_addr_t mapping;
1184
1185         skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
1186         if (unlikely(skb == NULL))
1187                 return -ENOMEM;
1188
1189         mapping = dma_map_single(&bp->pdev->dev, skb->data, bp->rx_buf_size,
1190                                  DMA_FROM_DEVICE);
1191         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
1192                 dev_kfree_skb(skb);
1193                 return -ENOMEM;
1194         }
1195
1196         rx_buf->skb = skb;
1197         dma_unmap_addr_set(rx_buf, mapping, mapping);
1198
1199         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1200         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1201
1202         return 0;
1203 }
1204
1205 /* note that we are not allocating a new skb,
1206  * we are just moving one from cons to prod
1207  * we are not creating a new mapping,
1208  * so there is no need to check for dma_mapping_error().
1209  */
1210 static void bnx2x_reuse_rx_skb(struct bnx2x_fastpath *fp,
1211                                struct sk_buff *skb, u16 cons, u16 prod)
1212 {
1213         struct bnx2x *bp = fp->bp;
1214         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
1215         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
1216         struct eth_rx_bd *cons_bd = &fp->rx_desc_ring[cons];
1217         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
1218
1219         dma_sync_single_for_device(&bp->pdev->dev,
1220                                    dma_unmap_addr(cons_rx_buf, mapping),
1221                                    RX_COPY_THRESH, DMA_FROM_DEVICE);
1222
1223         prod_rx_buf->skb = cons_rx_buf->skb;
1224         dma_unmap_addr_set(prod_rx_buf, mapping,
1225                            dma_unmap_addr(cons_rx_buf, mapping));
1226         *prod_bd = *cons_bd;
1227 }
1228
1229 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
1230                                              u16 idx)
1231 {
1232         u16 last_max = fp->last_max_sge;
1233
1234         if (SUB_S16(idx, last_max) > 0)
1235                 fp->last_max_sge = idx;
1236 }
1237
1238 static void bnx2x_clear_sge_mask_next_elems(struct bnx2x_fastpath *fp)
1239 {
1240         int i, j;
1241
1242         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1243                 int idx = RX_SGE_CNT * i - 1;
1244
1245                 for (j = 0; j < 2; j++) {
1246                         SGE_MASK_CLEAR_BIT(fp, idx);
1247                         idx--;
1248                 }
1249         }
1250 }
1251
1252 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
1253                                   struct eth_fast_path_rx_cqe *fp_cqe)
1254 {
1255         struct bnx2x *bp = fp->bp;
1256         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
1257                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
1258                       SGE_PAGE_SHIFT;
1259         u16 last_max, last_elem, first_elem;
1260         u16 delta = 0;
1261         u16 i;
1262
1263         if (!sge_len)
1264                 return;
1265
1266         /* First mark all used pages */
1267         for (i = 0; i < sge_len; i++)
1268                 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
1269
1270         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
1271            sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
1272
1273         /* Here we assume that the last SGE index is the biggest */
1274         prefetch((void *)(fp->sge_mask));
1275         bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
1276
1277         last_max = RX_SGE(fp->last_max_sge);
1278         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
1279         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
1280
1281         /* If ring is not full */
1282         if (last_elem + 1 != first_elem)
1283                 last_elem++;
1284
1285         /* Now update the prod */
1286         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
1287                 if (likely(fp->sge_mask[i]))
1288                         break;
1289
1290                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
1291                 delta += RX_SGE_MASK_ELEM_SZ;
1292         }
1293
1294         if (delta > 0) {
1295                 fp->rx_sge_prod += delta;
1296                 /* clear page-end entries */
1297                 bnx2x_clear_sge_mask_next_elems(fp);
1298         }
1299
1300         DP(NETIF_MSG_RX_STATUS,
1301            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
1302            fp->last_max_sge, fp->rx_sge_prod);
1303 }
1304
1305 static inline void bnx2x_init_sge_ring_bit_mask(struct bnx2x_fastpath *fp)
1306 {
1307         /* Set the mask to all 1-s: it's faster to compare to 0 than to 0xf-s */
1308         memset(fp->sge_mask, 0xff,
1309                (NUM_RX_SGE >> RX_SGE_MASK_ELEM_SHIFT)*sizeof(u64));
1310
1311         /* Clear the two last indices in the page to 1:
1312            these are the indices that correspond to the "next" element,
1313            hence will never be indicated and should be removed from
1314            the calculations. */
1315         bnx2x_clear_sge_mask_next_elems(fp);
1316 }
1317
1318 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
1319                             struct sk_buff *skb, u16 cons, u16 prod)
1320 {
1321         struct bnx2x *bp = fp->bp;
1322         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
1323         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
1324         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
1325         dma_addr_t mapping;
1326
1327         /* move empty skb from pool to prod and map it */
1328         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
1329         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
1330                                  bp->rx_buf_size, DMA_FROM_DEVICE);
1331         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
1332
1333         /* move partial skb from cons to pool (don't unmap yet) */
1334         fp->tpa_pool[queue] = *cons_rx_buf;
1335
1336         /* mark bin state as start - print error if current state != stop */
1337         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
1338                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
1339
1340         fp->tpa_state[queue] = BNX2X_TPA_START;
1341
1342         /* point prod_bd to new skb */
1343         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1344         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1345
1346 #ifdef BNX2X_STOP_ON_ERROR
1347         fp->tpa_queue_used |= (1 << queue);
1348 #ifdef _ASM_GENERIC_INT_L64_H
1349         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
1350 #else
1351         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
1352 #endif
1353            fp->tpa_queue_used);
1354 #endif
1355 }
1356
1357 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
1358                                struct sk_buff *skb,
1359                                struct eth_fast_path_rx_cqe *fp_cqe,
1360                                u16 cqe_idx)
1361 {
1362         struct sw_rx_page *rx_pg, old_rx_pg;
1363         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
1364         u32 i, frag_len, frag_size, pages;
1365         int err;
1366         int j;
1367
1368         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
1369         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
1370
1371         /* This is needed in order to enable forwarding support */
1372         if (frag_size)
1373                 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
1374                                                max(frag_size, (u32)len_on_bd));
1375
1376 #ifdef BNX2X_STOP_ON_ERROR
1377         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
1378                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
1379                           pages, cqe_idx);
1380                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
1381                           fp_cqe->pkt_len, len_on_bd);
1382                 bnx2x_panic();
1383                 return -EINVAL;
1384         }
1385 #endif
1386
1387         /* Run through the SGL and compose the fragmented skb */
1388         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
1389                 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
1390
1391                 /* FW gives the indices of the SGE as if the ring is an array
1392                    (meaning that "next" element will consume 2 indices) */
1393                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
1394                 rx_pg = &fp->rx_page_ring[sge_idx];
1395                 old_rx_pg = *rx_pg;
1396
1397                 /* If we fail to allocate a substitute page, we simply stop
1398                    where we are and drop the whole packet */
1399                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
1400                 if (unlikely(err)) {
1401                         fp->eth_q_stats.rx_skb_alloc_failed++;
1402                         return err;
1403                 }
1404
1405                 /* Unmap the page as we r going to pass it to the stack */
1406                 dma_unmap_page(&bp->pdev->dev,
1407                                dma_unmap_addr(&old_rx_pg, mapping),
1408                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
1409
1410                 /* Add one frag and update the appropriate fields in the skb */
1411                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
1412
1413                 skb->data_len += frag_len;
1414                 skb->truesize += frag_len;
1415                 skb->len += frag_len;
1416
1417                 frag_size -= frag_len;
1418         }
1419
1420         return 0;
1421 }
1422
1423 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
1424                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
1425                            u16 cqe_idx)
1426 {
1427         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
1428         struct sk_buff *skb = rx_buf->skb;
1429         /* alloc new skb */
1430         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
1431
1432         /* Unmap skb in the pool anyway, as we are going to change
1433            pool entry status to BNX2X_TPA_STOP even if new skb allocation
1434            fails. */
1435         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
1436                          bp->rx_buf_size, DMA_FROM_DEVICE);
1437
1438         if (likely(new_skb)) {
1439                 /* fix ip xsum and give it to the stack */
1440                 /* (no need to map the new skb) */
1441 #ifdef BCM_VLAN
1442                 int is_vlan_cqe =
1443                         (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
1444                          PARSING_FLAGS_VLAN);
1445                 int is_not_hwaccel_vlan_cqe =
1446                         (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
1447 #endif
1448
1449                 prefetch(skb);
1450                 prefetch(((char *)(skb)) + 128);
1451
1452 #ifdef BNX2X_STOP_ON_ERROR
1453                 if (pad + len > bp->rx_buf_size) {
1454                         BNX2X_ERR("skb_put is about to fail...  "
1455                                   "pad %d  len %d  rx_buf_size %d\n",
1456                                   pad, len, bp->rx_buf_size);
1457                         bnx2x_panic();
1458                         return;
1459                 }
1460 #endif
1461
1462                 skb_reserve(skb, pad);
1463                 skb_put(skb, len);
1464
1465                 skb->protocol = eth_type_trans(skb, bp->dev);
1466                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1467
1468                 {
1469                         struct iphdr *iph;
1470
1471                         iph = (struct iphdr *)skb->data;
1472 #ifdef BCM_VLAN
1473                         /* If there is no Rx VLAN offloading -
1474                            take VLAN tag into an account */
1475                         if (unlikely(is_not_hwaccel_vlan_cqe))
1476                                 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
1477 #endif
1478                         iph->check = 0;
1479                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
1480                 }
1481
1482                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
1483                                          &cqe->fast_path_cqe, cqe_idx)) {
1484 #ifdef BCM_VLAN
1485                         if ((bp->vlgrp != NULL) && is_vlan_cqe &&
1486                             (!is_not_hwaccel_vlan_cqe))
1487                                 vlan_gro_receive(&fp->napi, bp->vlgrp,
1488                                                  le16_to_cpu(cqe->fast_path_cqe.
1489                                                              vlan_tag), skb);
1490                         else
1491 #endif
1492                                 napi_gro_receive(&fp->napi, skb);
1493                 } else {
1494                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
1495                            " - dropping packet!\n");
1496                         dev_kfree_skb(skb);
1497                 }
1498
1499
1500                 /* put new skb in bin */
1501                 fp->tpa_pool[queue].skb = new_skb;
1502
1503         } else {
1504                 /* else drop the packet and keep the buffer in the bin */
1505                 DP(NETIF_MSG_RX_STATUS,
1506                    "Failed to allocate new skb - dropping packet!\n");
1507                 fp->eth_q_stats.rx_skb_alloc_failed++;
1508         }
1509
1510         fp->tpa_state[queue] = BNX2X_TPA_STOP;
1511 }
1512
1513 static inline void bnx2x_update_rx_prod(struct bnx2x *bp,
1514                                         struct bnx2x_fastpath *fp,
1515                                         u16 bd_prod, u16 rx_comp_prod,
1516                                         u16 rx_sge_prod)
1517 {
1518         struct ustorm_eth_rx_producers rx_prods = {0};
1519         int i;
1520
1521         /* Update producers */
1522         rx_prods.bd_prod = bd_prod;
1523         rx_prods.cqe_prod = rx_comp_prod;
1524         rx_prods.sge_prod = rx_sge_prod;
1525
1526         /*
1527          * Make sure that the BD and SGE data is updated before updating the
1528          * producers since FW might read the BD/SGE right after the producer
1529          * is updated.
1530          * This is only applicable for weak-ordered memory model archs such
1531          * as IA-64. The following barrier is also mandatory since FW will
1532          * assumes BDs must have buffers.
1533          */
1534         wmb();
1535
1536         for (i = 0; i < sizeof(struct ustorm_eth_rx_producers)/4; i++)
1537                 REG_WR(bp, BAR_USTRORM_INTMEM +
1538                        USTORM_RX_PRODS_OFFSET(BP_PORT(bp), fp->cl_id) + i*4,
1539                        ((u32 *)&rx_prods)[i]);
1540
1541         mmiowb(); /* keep prod updates ordered */
1542
1543         DP(NETIF_MSG_RX_STATUS,
1544            "queue[%d]:  wrote  bd_prod %u  cqe_prod %u  sge_prod %u\n",
1545            fp->index, bd_prod, rx_comp_prod, rx_sge_prod);
1546 }
1547
1548 /* Set Toeplitz hash value in the skb using the value from the
1549  * CQE (calculated by HW).
1550  */
1551 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
1552                                         struct sk_buff *skb)
1553 {
1554         /* Set Toeplitz hash from CQE */
1555         if ((bp->dev->features & NETIF_F_RXHASH) &&
1556             (cqe->fast_path_cqe.status_flags &
1557              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
1558                 skb->rxhash =
1559                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
1560 }
1561
1562 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
1563 {
1564         struct bnx2x *bp = fp->bp;
1565         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
1566         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
1567         int rx_pkt = 0;
1568
1569 #ifdef BNX2X_STOP_ON_ERROR
1570         if (unlikely(bp->panic))
1571                 return 0;
1572 #endif
1573
1574         /* CQ "next element" is of the size of the regular element,
1575            that's why it's ok here */
1576         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
1577         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
1578                 hw_comp_cons++;
1579
1580         bd_cons = fp->rx_bd_cons;
1581         bd_prod = fp->rx_bd_prod;
1582         bd_prod_fw = bd_prod;
1583         sw_comp_cons = fp->rx_comp_cons;
1584         sw_comp_prod = fp->rx_comp_prod;
1585
1586         /* Memory barrier necessary as speculative reads of the rx
1587          * buffer can be ahead of the index in the status block
1588          */
1589         rmb();
1590
1591         DP(NETIF_MSG_RX_STATUS,
1592            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
1593            fp->index, hw_comp_cons, sw_comp_cons);
1594
1595         while (sw_comp_cons != hw_comp_cons) {
1596                 struct sw_rx_bd *rx_buf = NULL;
1597                 struct sk_buff *skb;
1598                 union eth_rx_cqe *cqe;
1599                 u8 cqe_fp_flags;
1600                 u16 len, pad;
1601
1602                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1603                 bd_prod = RX_BD(bd_prod);
1604                 bd_cons = RX_BD(bd_cons);
1605
1606                 /* Prefetch the page containing the BD descriptor
1607                    at producer's index. It will be needed when new skb is
1608                    allocated */
1609                 prefetch((void *)(PAGE_ALIGN((unsigned long)
1610                                              (&fp->rx_desc_ring[bd_prod])) -
1611                                   PAGE_SIZE + 1));
1612
1613                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1614                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
1615
1616                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
1617                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
1618                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
1619                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
1620                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
1621                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
1622
1623                 /* is this a slowpath msg? */
1624                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
1625                         bnx2x_sp_event(fp, cqe);
1626                         goto next_cqe;
1627
1628                 /* this is an rx packet */
1629                 } else {
1630                         rx_buf = &fp->rx_buf_ring[bd_cons];
1631                         skb = rx_buf->skb;
1632                         prefetch(skb);
1633                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
1634                         pad = cqe->fast_path_cqe.placement_offset;
1635
1636                         /* If CQE is marked both TPA_START and TPA_END
1637                            it is a non-TPA CQE */
1638                         if ((!fp->disable_tpa) &&
1639                             (TPA_TYPE(cqe_fp_flags) !=
1640                                         (TPA_TYPE_START | TPA_TYPE_END))) {
1641                                 u16 queue = cqe->fast_path_cqe.queue_index;
1642
1643                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
1644                                         DP(NETIF_MSG_RX_STATUS,
1645                                            "calling tpa_start on queue %d\n",
1646                                            queue);
1647
1648                                         bnx2x_tpa_start(fp, queue, skb,
1649                                                         bd_cons, bd_prod);
1650
1651                                         /* Set Toeplitz hash for an LRO skb */
1652                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
1653
1654                                         goto next_rx;
1655                                 }
1656
1657                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
1658                                         DP(NETIF_MSG_RX_STATUS,
1659                                            "calling tpa_stop on queue %d\n",
1660                                            queue);
1661
1662                                         if (!BNX2X_RX_SUM_FIX(cqe))
1663                                                 BNX2X_ERR("STOP on none TCP "
1664                                                           "data\n");
1665
1666                                         /* This is a size of the linear data
1667                                            on this skb */
1668                                         len = le16_to_cpu(cqe->fast_path_cqe.
1669                                                                 len_on_bd);
1670                                         bnx2x_tpa_stop(bp, fp, queue, pad,
1671                                                     len, cqe, comp_ring_cons);
1672 #ifdef BNX2X_STOP_ON_ERROR
1673                                         if (bp->panic)
1674                                                 return 0;
1675 #endif
1676
1677                                         bnx2x_update_sge_prod(fp,
1678                                                         &cqe->fast_path_cqe);
1679                                         goto next_cqe;
1680                                 }
1681                         }
1682
1683                         dma_sync_single_for_device(&bp->pdev->dev,
1684                                         dma_unmap_addr(rx_buf, mapping),
1685                                                    pad + RX_COPY_THRESH,
1686                                                    DMA_FROM_DEVICE);
1687                         prefetch(((char *)(skb)) + 128);
1688
1689                         /* is this an error packet? */
1690                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1691                                 DP(NETIF_MSG_RX_ERR,
1692                                    "ERROR  flags %x  rx packet %u\n",
1693                                    cqe_fp_flags, sw_comp_cons);
1694                                 fp->eth_q_stats.rx_err_discard_pkt++;
1695                                 goto reuse_rx;
1696                         }
1697
1698                         /* Since we don't have a jumbo ring
1699                          * copy small packets if mtu > 1500
1700                          */
1701                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1702                             (len <= RX_COPY_THRESH)) {
1703                                 struct sk_buff *new_skb;
1704
1705                                 new_skb = netdev_alloc_skb(bp->dev,
1706                                                            len + pad);
1707                                 if (new_skb == NULL) {
1708                                         DP(NETIF_MSG_RX_ERR,
1709                                            "ERROR  packet dropped "
1710                                            "because of alloc failure\n");
1711                                         fp->eth_q_stats.rx_skb_alloc_failed++;
1712                                         goto reuse_rx;
1713                                 }
1714
1715                                 /* aligned copy */
1716                                 skb_copy_from_linear_data_offset(skb, pad,
1717                                                     new_skb->data + pad, len);
1718                                 skb_reserve(new_skb, pad);
1719                                 skb_put(new_skb, len);
1720
1721                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
1722
1723                                 skb = new_skb;
1724
1725                         } else
1726                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
1727                                 dma_unmap_single(&bp->pdev->dev,
1728                                         dma_unmap_addr(rx_buf, mapping),
1729                                                  bp->rx_buf_size,
1730                                                  DMA_FROM_DEVICE);
1731                                 skb_reserve(skb, pad);
1732                                 skb_put(skb, len);
1733
1734                         } else {
1735                                 DP(NETIF_MSG_RX_ERR,
1736                                    "ERROR  packet dropped because "
1737                                    "of alloc failure\n");
1738                                 fp->eth_q_stats.rx_skb_alloc_failed++;
1739 reuse_rx:
1740                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
1741                                 goto next_rx;
1742                         }
1743
1744                         skb->protocol = eth_type_trans(skb, bp->dev);
1745
1746                         /* Set Toeplitz hash for a none-LRO skb */
1747                         bnx2x_set_skb_rxhash(bp, cqe, skb);
1748
1749                         skb->ip_summed = CHECKSUM_NONE;
1750                         if (bp->rx_csum) {
1751                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
1752                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1753                                 else
1754                                         fp->eth_q_stats.hw_csum_err++;
1755                         }
1756                 }
1757
1758                 skb_record_rx_queue(skb, fp->index);
1759
1760 #ifdef BCM_VLAN
1761                 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
1762                     (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
1763                      PARSING_FLAGS_VLAN))
1764                         vlan_gro_receive(&fp->napi, bp->vlgrp,
1765                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
1766                 else
1767 #endif
1768                         napi_gro_receive(&fp->napi, skb);
1769
1770
1771 next_rx:
1772                 rx_buf->skb = NULL;
1773
1774                 bd_cons = NEXT_RX_IDX(bd_cons);
1775                 bd_prod = NEXT_RX_IDX(bd_prod);
1776                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1777                 rx_pkt++;
1778 next_cqe:
1779                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1780                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1781
1782                 if (rx_pkt == budget)
1783                         break;
1784         } /* while */
1785
1786         fp->rx_bd_cons = bd_cons;
1787         fp->rx_bd_prod = bd_prod_fw;
1788         fp->rx_comp_cons = sw_comp_cons;
1789         fp->rx_comp_prod = sw_comp_prod;
1790
1791         /* Update producers */
1792         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1793                              fp->rx_sge_prod);
1794
1795         fp->rx_pkt += rx_pkt;
1796         fp->rx_calls++;
1797
1798         return rx_pkt;
1799 }
1800
1801 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1802 {
1803         struct bnx2x_fastpath *fp = fp_cookie;
1804         struct bnx2x *bp = fp->bp;
1805
1806         /* Return here if interrupt is disabled */
1807         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
1808                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
1809                 return IRQ_HANDLED;
1810         }
1811
1812         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
1813            fp->index, fp->sb_id);
1814         bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1815
1816 #ifdef BNX2X_STOP_ON_ERROR
1817         if (unlikely(bp->panic))
1818                 return IRQ_HANDLED;
1819 #endif
1820
1821         /* Handle Rx and Tx according to MSI-X vector */
1822         prefetch(fp->rx_cons_sb);
1823         prefetch(fp->tx_cons_sb);
1824         prefetch(&fp->status_blk->u_status_block.status_block_index);
1825         prefetch(&fp->status_blk->c_status_block.status_block_index);
1826         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1827
1828         return IRQ_HANDLED;
1829 }
1830
1831 static irqreturn_t bnx2x_interrupt(int irq, void *dev_instance)
1832 {
1833         struct bnx2x *bp = netdev_priv(dev_instance);
1834         u16 status = bnx2x_ack_int(bp);
1835         u16 mask;
1836         int i;
1837
1838         /* Return here if interrupt is shared and it's not for us */
1839         if (unlikely(status == 0)) {
1840                 DP(NETIF_MSG_INTR, "not our interrupt!\n");
1841                 return IRQ_NONE;
1842         }
1843         DP(NETIF_MSG_INTR, "got an interrupt  status 0x%x\n", status);
1844
1845         /* Return here if interrupt is disabled */
1846         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
1847                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
1848                 return IRQ_HANDLED;
1849         }
1850
1851 #ifdef BNX2X_STOP_ON_ERROR
1852         if (unlikely(bp->panic))
1853                 return IRQ_HANDLED;
1854 #endif
1855
1856         for (i = 0; i < BNX2X_NUM_QUEUES(bp); i++) {
1857                 struct bnx2x_fastpath *fp = &bp->fp[i];
1858
1859                 mask = 0x2 << fp->sb_id;
1860                 if (status & mask) {
1861                         /* Handle Rx and Tx according to SB id */
1862                         prefetch(fp->rx_cons_sb);
1863                         prefetch(&fp->status_blk->u_status_block.
1864                                                 status_block_index);
1865                         prefetch(fp->tx_cons_sb);
1866                         prefetch(&fp->status_blk->c_status_block.
1867                                                 status_block_index);
1868                         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1869                         status &= ~mask;
1870                 }
1871         }
1872
1873 #ifdef BCM_CNIC
1874         mask = 0x2 << CNIC_SB_ID(bp);
1875         if (status & (mask | 0x1)) {
1876                 struct cnic_ops *c_ops = NULL;
1877
1878                 rcu_read_lock();
1879                 c_ops = rcu_dereference(bp->cnic_ops);
1880                 if (c_ops)
1881                         c_ops->cnic_handler(bp->cnic_data, NULL);
1882                 rcu_read_unlock();
1883
1884                 status &= ~mask;
1885         }
1886 #endif
1887
1888         if (unlikely(status & 0x1)) {
1889                 queue_delayed_work(bnx2x_wq, &bp->sp_task, 0);
1890
1891                 status &= ~0x1;
1892                 if (!status)
1893                         return IRQ_HANDLED;
1894         }
1895
1896         if (unlikely(status))
1897                 DP(NETIF_MSG_INTR, "got an unknown interrupt! (status 0x%x)\n",
1898                    status);
1899
1900         return IRQ_HANDLED;
1901 }
1902
1903 /* end of fast path */
1904
1905 static void bnx2x_stats_handle(struct bnx2x *bp, enum bnx2x_stats_event event);
1906
1907 /* Link */
1908
1909 /*
1910  * General service functions
1911  */
1912
1913 static int bnx2x_acquire_hw_lock(struct bnx2x *bp, u32 resource)
1914 {
1915         u32 lock_status;
1916         u32 resource_bit = (1 << resource);
1917         int func = BP_FUNC(bp);
1918         u32 hw_lock_control_reg;
1919         int cnt;
1920
1921         /* Validating that the resource is within range */
1922         if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
1923                 DP(NETIF_MSG_HW,
1924                    "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)\n",
1925                    resource, HW_LOCK_MAX_RESOURCE_VALUE);
1926                 return -EINVAL;
1927         }
1928
1929         if (func <= 5) {
1930                 hw_lock_control_reg = (MISC_REG_DRIVER_CONTROL_1 + func*8);
1931         } else {
1932                 hw_lock_control_reg =
1933                                 (MISC_REG_DRIVER_CONTROL_7 + (func - 6)*8);
1934         }
1935
1936         /* Validating that the resource is not already taken */
1937         lock_status = REG_RD(bp, hw_lock_control_reg);
1938         if (lock_status & resource_bit) {
1939                 DP(NETIF_MSG_HW, "lock_status 0x%x  resource_bit 0x%x\n",
1940                    lock_status, resource_bit);
1941                 return -EEXIST;
1942         }
1943
1944         /* Try for 5 second every 5ms */
1945         for (cnt = 0; cnt < 1000; cnt++) {
1946                 /* Try to acquire the lock */
1947                 REG_WR(bp, hw_lock_control_reg + 4, resource_bit);
1948                 lock_status = REG_RD(bp, hw_lock_control_reg);
1949                 if (lock_status & resource_bit)
1950                         return 0;
1951
1952                 msleep(5);
1953         }
1954         DP(NETIF_MSG_HW, "Timeout\n");
1955         return -EAGAIN;
1956 }
1957
1958 static int bnx2x_release_hw_lock(struct bnx2x *bp, u32 resource)
1959 {
1960         u32 lock_status;
1961         u32 resource_bit = (1 << resource);
1962         int func = BP_FUNC(bp);
1963         u32 hw_lock_control_reg;
1964
1965         DP(NETIF_MSG_HW, "Releasing a lock on resource %d\n", resource);
1966
1967         /* Validating that the resource is within range */
1968         if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
1969                 DP(NETIF_MSG_HW,
1970                    "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)\n",
1971                    resource, HW_LOCK_MAX_RESOURCE_VALUE);
1972                 return -EINVAL;
1973         }
1974
1975         if (func <= 5) {
1976                 hw_lock_control_reg = (MISC_REG_DRIVER_CONTROL_1 + func*8);
1977         } else {
1978                 hw_lock_control_reg =
1979                                 (MISC_REG_DRIVER_CONTROL_7 + (func - 6)*8);
1980         }
1981
1982         /* Validating that the resource is currently taken */
1983         lock_status = REG_RD(bp, hw_lock_control_reg);
1984         if (!(lock_status & resource_bit)) {
1985                 DP(NETIF_MSG_HW, "lock_status 0x%x  resource_bit 0x%x\n",
1986                    lock_status, resource_bit);
1987                 return -EFAULT;
1988         }
1989
1990         REG_WR(bp, hw_lock_control_reg, resource_bit);
1991         return 0;
1992 }
1993
1994 /* HW Lock for shared dual port PHYs */
1995 static void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1996 {
1997         mutex_lock(&bp->port.phy_mutex);
1998
1999         if (bp->port.need_hw_lock)
2000                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
2001 }
2002
2003 static void bnx2x_release_phy_lock(struct bnx2x *bp)
2004 {
2005         if (bp->port.need_hw_lock)
2006                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
2007
2008         mutex_unlock(&bp->port.phy_mutex);
2009 }
2010
2011 int bnx2x_get_gpio(struct bnx2x *bp, int gpio_num, u8 port)
2012 {
2013         /* The GPIO should be swapped if swap register is set and active */
2014         int gpio_port = (REG_RD(bp, NIG_REG_PORT_SWAP) &&
2015                          REG_RD(bp, NIG_REG_STRAP_OVERRIDE)) ^ port;
2016         int gpio_shift = gpio_num +
2017                         (gpio_port ? MISC_REGISTERS_GPIO_PORT_SHIFT : 0);
2018         u32 gpio_mask = (1 << gpio_shift);
2019         u32 gpio_reg;
2020         int value;
2021
2022         if (gpio_num > MISC_REGISTERS_GPIO_3) {
2023                 BNX2X_ERR("Invalid GPIO %d\n", gpio_num);
2024                 return -EINVAL;
2025         }
2026
2027         /* read GPIO value */
2028         gpio_reg = REG_RD(bp, MISC_REG_GPIO);
2029
2030         /* get the requested pin value */
2031         if ((gpio_reg & gpio_mask) == gpio_mask)
2032                 value = 1;
2033         else
2034                 value = 0;
2035
2036         DP(NETIF_MSG_LINK, "pin %d  value 0x%x\n", gpio_num, value);
2037
2038         return value;
2039 }
2040
2041 int bnx2x_set_gpio(struct bnx2x *bp, int gpio_num, u32 mode, u8 port)
2042 {
2043         /* The GPIO should be swapped if swap register is set and active */
2044         int gpio_port = (REG_RD(bp, NIG_REG_PORT_SWAP) &&
2045                          REG_RD(bp, NIG_REG_STRAP_OVERRIDE)) ^ port;
2046         int gpio_shift = gpio_num +
2047                         (gpio_port ? MISC_REGISTERS_GPIO_PORT_SHIFT : 0);
2048         u32 gpio_mask = (1 << gpio_shift);
2049         u32 gpio_reg;
2050
2051         if (gpio_num > MISC_REGISTERS_GPIO_3) {
2052                 BNX2X_ERR("Invalid GPIO %d\n", gpio_num);
2053                 return -EINVAL;
2054         }
2055
2056         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);
2057         /* read GPIO and mask except the float bits */
2058         gpio_reg = (REG_RD(bp, MISC_REG_GPIO) & MISC_REGISTERS_GPIO_FLOAT);
2059
2060         switch (mode) {
2061         case MISC_REGISTERS_GPIO_OUTPUT_LOW:
2062                 DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> output low\n",
2063                    gpio_num, gpio_shift);
2064                 /* clear FLOAT and set CLR */
2065                 gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
2066                 gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_CLR_POS);
2067                 break;
2068
2069         case MISC_REGISTERS_GPIO_OUTPUT_HIGH:
2070                 DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> output high\n",
2071                    gpio_num, gpio_shift);
2072                 /* clear FLOAT and set SET */
2073                 gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
2074                 gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_SET_POS);
2075                 break;
2076
2077         case MISC_REGISTERS_GPIO_INPUT_HI_Z:
2078                 DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> input\n",
2079                    gpio_num, gpio_shift);
2080                 /* set FLOAT */
2081                 gpio_reg |= (gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
2082                 break;
2083
2084         default:
2085                 break;
2086         }
2087
2088         REG_WR(bp, MISC_REG_GPIO, gpio_reg);
2089         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);
2090
2091         return 0;
2092 }
2093
2094 int bnx2x_set_gpio_int(struct bnx2x *bp, int gpio_num, u32 mode, u8 port)
2095 {
2096         /* The GPIO should be swapped if swap register is set and active */
2097         int gpio_port = (REG_RD(bp, NIG_REG_PORT_SWAP) &&
2098                          REG_RD(bp, NIG_REG_STRAP_OVERRIDE)) ^ port;
2099         int gpio_shift = gpio_num +
2100                         (gpio_port ? MISC_REGISTERS_GPIO_PORT_SHIFT : 0);
2101         u32 gpio_mask = (1 << gpio_shift);
2102         u32 gpio_reg;
2103
2104         if (gpio_num > MISC_REGISTERS_GPIO_3) {
2105                 BNX2X_ERR("Invalid GPIO %d\n", gpio_num);
2106                 return -EINVAL;
2107         }
2108
2109         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);
2110         /* read GPIO int */
2111         gpio_reg = REG_RD(bp, MISC_REG_GPIO_INT);
2112
2113         switch (mode) {
2114         case MISC_REGISTERS_GPIO_INT_OUTPUT_CLR:
2115                 DP(NETIF_MSG_LINK, "Clear GPIO INT %d (shift %d) -> "
2116                                    "output low\n", gpio_num, gpio_shift);
2117                 /* clear SET and set CLR */
2118                 gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_INT_SET_POS);
2119                 gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_INT_CLR_POS);
2120                 break;
2121
2122         case MISC_REGISTERS_GPIO_INT_OUTPUT_SET:
2123                 DP(NETIF_MSG_LINK, "Set GPIO INT %d (shift %d) -> "
2124                                    "output high\n", gpio_num, gpio_shift);
2125                 /* clear CLR and set SET */
2126                 gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_INT_CLR_POS);
2127                 gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_INT_SET_POS);
2128                 break;
2129
2130         default:
2131                 break;
2132         }
2133
2134         REG_WR(bp, MISC_REG_GPIO_INT, gpio_reg);
2135         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);
2136
2137         return 0;
2138 }
2139
2140 static int bnx2x_set_spio(struct bnx2x *bp, int spio_num, u32 mode)
2141 {
2142         u32 spio_mask = (1 << spio_num);
2143         u32 spio_reg;
2144
2145         if ((spio_num < MISC_REGISTERS_SPIO_4) ||
2146             (spio_num > MISC_REGISTERS_SPIO_7)) {
2147                 BNX2X_ERR("Invalid SPIO %d\n", spio_num);
2148                 return -EINVAL;
2149         }
2150
2151         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_SPIO);
2152         /* read SPIO and mask except the float bits */
2153         spio_reg = (REG_RD(bp, MISC_REG_SPIO) & MISC_REGISTERS_SPIO_FLOAT);
2154
2155         switch (mode) {
2156         case MISC_REGISTERS_SPIO_OUTPUT_LOW:
2157                 DP(NETIF_MSG_LINK, "Set SPIO %d -> output low\n", spio_num);
2158                 /* clear FLOAT and set CLR */
2159                 spio_reg &= ~(spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
2160                 spio_reg |=  (spio_mask << MISC_REGISTERS_SPIO_CLR_POS);
2161                 break;
2162
2163         case MISC_REGISTERS_SPIO_OUTPUT_HIGH:
2164                 DP(NETIF_MSG_LINK, "Set SPIO %d -> output high\n", spio_num);
2165                 /* clear FLOAT and set SET */
2166                 spio_reg &= ~(spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
2167                 spio_reg |=  (spio_mask << MISC_REGISTERS_SPIO_SET_POS);
2168                 break;
2169
2170         case MISC_REGISTERS_SPIO_INPUT_HI_Z:
2171                 DP(NETIF_MSG_LINK, "Set SPIO %d -> input\n", spio_num);
2172                 /* set FLOAT */
2173                 spio_reg |= (spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
2174                 break;
2175
2176         default:
2177                 break;
2178         }
2179
2180         REG_WR(bp, MISC_REG_SPIO, spio_reg);
2181         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_SPIO);
2182
2183         return 0;
2184 }
2185
2186 static void bnx2x_calc_fc_adv(struct bnx2x *bp)
2187 {
2188         switch (bp->link_vars.ieee_fc &
2189                 MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_MASK) {
2190         case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_NONE:
2191                 bp->port.advertising &= ~(ADVERTISED_Asym_Pause |
2192                                           ADVERTISED_Pause);
2193                 break;
2194
2195         case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH:
2196                 bp->port.advertising |= (ADVERTISED_Asym_Pause |
2197                                          ADVERTISED_Pause);
2198                 break;
2199
2200         case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC:
2201                 bp->port.advertising |= ADVERTISED_Asym_Pause;
2202                 break;
2203
2204         default:
2205                 bp->port.advertising &= ~(ADVERTISED_Asym_Pause |
2206                                           ADVERTISED_Pause);
2207                 break;
2208         }
2209 }
2210
2211 static void bnx2x_link_report(struct bnx2x *bp)
2212 {
2213         if (bp->flags & MF_FUNC_DIS) {
2214                 netif_carrier_off(bp->dev);
2215                 netdev_err(bp->dev, "NIC Link is Down\n");
2216                 return;
2217         }
2218
2219         if (bp->link_vars.link_up) {
2220                 u16 line_speed;
2221
2222                 if (bp->state == BNX2X_STATE_OPEN)
2223                         netif_carrier_on(bp->dev);
2224                 netdev_info(bp->dev, "NIC Link is Up, ");
2225
2226                 line_speed = bp->link_vars.line_speed;
2227                 if (IS_E1HMF(bp)) {
2228                         u16 vn_max_rate;
2229
2230                         vn_max_rate =
2231                                 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
2232                                  FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
2233                         if (vn_max_rate < line_speed)
2234                                 line_speed = vn_max_rate;
2235                 }
2236                 pr_cont("%d Mbps ", line_speed);
2237
2238                 if (bp->link_vars.duplex == DUPLEX_FULL)
2239                         pr_cont("full duplex");
2240                 else
2241                         pr_cont("half duplex");
2242
2243                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
2244                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
2245                                 pr_cont(", receive ");
2246                                 if (bp->link_vars.flow_ctrl &
2247                                     BNX2X_FLOW_CTRL_TX)
2248                                         pr_cont("& transmit ");
2249                         } else {
2250                                 pr_cont(", transmit ");
2251                         }
2252                         pr_cont("flow control ON");
2253                 }
2254                 pr_cont("\n");
2255
2256         } else { /* link_down */
2257                 netif_carrier_off(bp->dev);
2258                 netdev_err(bp->dev, "NIC Link is Down\n");
2259         }
2260 }
2261
2262 static u8 bnx2x_initial_phy_init(struct bnx2x *bp, int load_mode)
2263 {
2264         if (!BP_NOMCP(bp)) {
2265                 u8 rc;
2266
2267                 /* Initialize link parameters structure variables */
2268                 /* It is recommended to turn off RX FC for jumbo frames
2269                    for better performance */
2270                 if (bp->dev->mtu > 5000)
2271                         bp->link_params.req_fc_auto_adv = BNX2X_FLOW_CTRL_TX;
2272                 else
2273                         bp->link_params.req_fc_auto_adv = BNX2X_FLOW_CTRL_BOTH;
2274
2275                 bnx2x_acquire_phy_lock(bp);
2276
2277                 if (load_mode == LOAD_DIAG)
2278                         bp->link_params.loopback_mode = LOOPBACK_XGXS_10;
2279
2280                 rc = bnx2x_phy_init(&bp->link_params, &bp->link_vars);
2281
2282                 bnx2x_release_phy_lock(bp);
2283
2284                 bnx2x_calc_fc_adv(bp);
2285
2286                 if (CHIP_REV_IS_SLOW(bp) && bp->link_vars.link_up) {
2287                         bnx2x_stats_handle(bp, STATS_EVENT_LINK_UP);
2288                         bnx2x_link_report(bp);
2289                 }
2290
2291                 return rc;
2292         }
2293         BNX2X_ERR("Bootcode is missing - can not initialize link\n");
2294         return -EINVAL;
2295 }
2296
2297 static void bnx2x_link_set(struct bnx2x *bp)
2298 {
2299         if (!BP_NOMCP(bp)) {
2300                 bnx2x_acquire_phy_lock(bp);
2301                 bnx2x_phy_init(&bp->link_params, &bp->link_vars);
2302                 bnx2x_release_phy_lock(bp);
2303
2304                 bnx2x_calc_fc_adv(bp);
2305         } else
2306                 BNX2X_ERR("Bootcode is missing - can not set link\n");
2307 }
2308
2309 static void bnx2x__link_reset(struct bnx2x *bp)
2310 {
2311         if (!BP_NOMCP(bp)) {
2312                 bnx2x_acquire_phy_lock(bp);
2313                 bnx2x_link_reset(&bp->link_params, &bp->link_vars, 1);
2314                 bnx2x_release_phy_lock(bp);
2315         } else
2316                 BNX2X_ERR("Bootcode is missing - can not reset link\n");
2317 }
2318
2319 static u8 bnx2x_link_test(struct bnx2x *bp)
2320 {
2321         u8 rc = 0;
2322
2323         if (!BP_NOMCP(bp)) {
2324                 bnx2x_acquire_phy_lock(bp);
2325                 rc = bnx2x_test_link(&bp->link_params, &bp->link_vars);
2326                 bnx2x_release_phy_lock(bp);
2327         } else
2328                 BNX2X_ERR("Bootcode is missing - can not test link\n");
2329
2330         return rc;
2331 }
2332
2333 static void bnx2x_init_port_minmax(struct bnx2x *bp)
2334 {
2335         u32 r_param = bp->link_vars.line_speed / 8;
2336         u32 fair_periodic_timeout_usec;
2337         u32 t_fair;
2338
2339         memset(&(bp->cmng.rs_vars), 0,
2340                sizeof(struct rate_shaping_vars_per_port));
2341         memset(&(bp->cmng.fair_vars), 0, sizeof(struct fairness_vars_per_port));
2342
2343         /* 100 usec in SDM ticks = 25 since each tick is 4 usec */
2344         bp->cmng.rs_vars.rs_periodic_timeout = RS_PERIODIC_TIMEOUT_USEC / 4;
2345
2346         /* this is the threshold below which no timer arming will occur
2347            1.25 coefficient is for the threshold to be a little bigger
2348            than the real time, to compensate for timer in-accuracy */
2349         bp->cmng.rs_vars.rs_threshold =
2350                                 (RS_PERIODIC_TIMEOUT_USEC * r_param * 5) / 4;
2351
2352         /* resolution of fairness timer */
2353         fair_periodic_timeout_usec = QM_ARB_BYTES / r_param;
2354         /* for 10G it is 1000usec. for 1G it is 10000usec. */
2355         t_fair = T_FAIR_COEF / bp->link_vars.line_speed;
2356
2357         /* this is the threshold below which we won't arm the timer anymore */
2358         bp->cmng.fair_vars.fair_threshold = QM_ARB_BYTES;
2359
2360         /* we multiply by 1e3/8 to get bytes/msec.
2361            We don't want the credits to pass a credit
2362            of the t_fair*FAIR_MEM (algorithm resolution) */
2363         bp->cmng.fair_vars.upper_bound = r_param * t_fair * FAIR_MEM;
2364         /* since each tick is 4 usec */
2365         bp->cmng.fair_vars.fairness_timeout = fair_periodic_timeout_usec / 4;
2366 }
2367
2368 /* Calculates the sum of vn_min_rates.
2369    It's needed for further normalizing of the min_rates.
2370    Returns:
2371      sum of vn_min_rates.
2372        or
2373      0 - if all the min_rates are 0.
2374      In the later case fainess algorithm should be deactivated.
2375      If not all min_rates are zero then those that are zeroes will be set to 1.
2376  */
2377 static void bnx2x_calc_vn_weight_sum(struct bnx2x *bp)
2378 {
2379         int all_zero = 1;
2380         int port = BP_PORT(bp);
2381         int vn;
2382
2383         bp->vn_weight_sum = 0;
2384         for (vn = VN_0; vn < E1HVN_MAX; vn++) {
2385                 int func = 2*vn + port;
2386                 u32 vn_cfg = SHMEM_RD(bp, mf_cfg.func_mf_config[func].config);
2387                 u32 vn_min_rate = ((vn_cfg & FUNC_MF_CFG_MIN_BW_MASK) >>
2388                                    FUNC_MF_CFG_MIN_BW_SHIFT) * 100;
2389
2390                 /* Skip hidden vns */
2391                 if (vn_cfg & FUNC_MF_CFG_FUNC_HIDE)
2392                         continue;
2393
2394                 /* If min rate is zero - set it to 1 */
2395                 if (!vn_min_rate)
2396                         vn_min_rate = DEF_MIN_RATE;
2397                 else
2398                         all_zero = 0;
2399
2400                 bp->vn_weight_sum += vn_min_rate;
2401         }
2402
2403         /* ... only if all min rates are zeros - disable fairness */
2404         if (all_zero) {
2405                 bp->cmng.flags.cmng_enables &=
2406                                         ~CMNG_FLAGS_PER_PORT_FAIRNESS_VN;
2407                 DP(NETIF_MSG_IFUP, "All MIN values are zeroes"
2408                    "  fairness will be disabled\n");
2409         } else
2410                 bp->cmng.flags.cmng_enables |=
2411                                         CMNG_FLAGS_PER_PORT_FAIRNESS_VN;
2412 }
2413
2414 static void bnx2x_init_vn_minmax(struct bnx2x *bp, int func)
2415 {
2416         struct rate_shaping_vars_per_vn m_rs_vn;
2417         struct fairness_vars_per_vn m_fair_vn;
2418         u32 vn_cfg = SHMEM_RD(bp, mf_cfg.func_mf_config[func].config);
2419         u16 vn_min_rate, vn_max_rate;
2420         int i;
2421
2422         /* If function is hidden - set min and max to zeroes */
2423         if (vn_cfg & FUNC_MF_CFG_FUNC_HIDE) {
2424                 vn_min_rate = 0;
2425                 vn_max_rate = 0;
2426
2427         } else {
2428                 vn_min_rate = ((vn_cfg & FUNC_MF_CFG_MIN_BW_MASK) >>
2429                                 FUNC_MF_CFG_MIN_BW_SHIFT) * 100;
2430                 /* If min rate is zero - set it to 1 */
2431                 if (!vn_min_rate)
2432                         vn_min_rate = DEF_MIN_RATE;
2433                 vn_max_rate = ((vn_cfg & FUNC_MF_CFG_MAX_BW_MASK) >>
2434                                 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
2435         }
2436         DP(NETIF_MSG_IFUP,
2437            "func %d: vn_min_rate %d  vn_max_rate %d  vn_weight_sum %d\n",
2438            func, vn_min_rate, vn_max_rate, bp->vn_weight_sum);
2439
2440         memset(&m_rs_vn, 0, sizeof(struct rate_shaping_vars_per_vn));
2441         memset(&m_fair_vn, 0, sizeof(struct fairness_vars_per_vn));
2442
2443         /* global vn counter - maximal Mbps for this vn */
2444         m_rs_vn.vn_counter.rate = vn_max_rate;
2445
2446         /* quota - number of bytes transmitted in this period */
2447         m_rs_vn.vn_counter.quota =
2448                                 (vn_max_rate * RS_PERIODIC_TIMEOUT_USEC) / 8;
2449
2450         if (bp->vn_weight_sum) {
2451                 /* credit for each period of the fairness algorithm:
2452                    number of bytes in T_FAIR (the vn share the port rate).
2453                    vn_weight_sum should not be larger than 10000, thus
2454                    T_FAIR_COEF / (8 * vn_weight_sum) will always be greater
2455                    than zero */
2456                 m_fair_vn.vn_credit_delta =
2457                         max_t(u32, (vn_min_rate * (T_FAIR_COEF /
2458                                                    (8 * bp->vn_weight_sum))),
2459                               (bp->cmng.fair_vars.fair_threshold * 2));
2460                 DP(NETIF_MSG_IFUP, "m_fair_vn.vn_credit_delta %d\n",
2461                    m_fair_vn.vn_credit_delta);
2462         }
2463
2464         /* Store it to internal memory */
2465         for (i = 0; i < sizeof(struct rate_shaping_vars_per_vn)/4; i++)
2466                 REG_WR(bp, BAR_XSTRORM_INTMEM +
2467                        XSTORM_RATE_SHAPING_PER_VN_VARS_OFFSET(func) + i * 4,
2468                        ((u32 *)(&m_rs_vn))[i]);
2469
2470         for (i = 0; i < sizeof(struct fairness_vars_per_vn)/4; i++)
2471                 REG_WR(bp, BAR_XSTRORM_INTMEM +
2472                        XSTORM_FAIRNESS_PER_VN_VARS_OFFSET(func) + i * 4,
2473                        ((u32 *)(&m_fair_vn))[i]);
2474 }
2475
2476
2477 /* This function is called upon link interrupt */
2478 static void bnx2x_link_attn(struct bnx2x *bp)
2479 {
2480         u32 prev_link_status = bp->link_vars.link_status;
2481         /* Make sure that we are synced with the current statistics */
2482         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2483
2484         bnx2x_link_update(&bp->link_params, &bp->link_vars);
2485
2486         if (bp->link_vars.link_up) {
2487
2488                 /* dropless flow control */
2489                 if (CHIP_IS_E1H(bp) && bp->dropless_fc) {
2490                         int port = BP_PORT(bp);
2491                         u32 pause_enabled = 0;
2492
2493                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
2494                                 pause_enabled = 1;
2495
2496                         REG_WR(bp, BAR_USTRORM_INTMEM +
2497                                USTORM_ETH_PAUSE_ENABLED_OFFSET(port),
2498                                pause_enabled);
2499                 }
2500
2501                 if (bp->link_vars.mac_type == MAC_TYPE_BMAC) {
2502                         struct host_port_stats *pstats;
2503
2504                         pstats = bnx2x_sp(bp, port_stats);
2505                         /* reset old bmac stats */
2506                         memset(&(pstats->mac_stx[0]), 0,
2507                                sizeof(struct mac_stx));
2508                 }
2509                 if (bp->state == BNX2X_STATE_OPEN)
2510                         bnx2x_stats_handle(bp, STATS_EVENT_LINK_UP);
2511         }
2512
2513         /* indicate link status only if link status actually changed */
2514         if (prev_link_status != bp->link_vars.link_status)
2515                 bnx2x_link_report(bp);
2516
2517         if (IS_E1HMF(bp)) {
2518                 int port = BP_PORT(bp);
2519                 int func;
2520                 int vn;
2521
2522                 /* Set the attention towards other drivers on the same port */
2523                 for (vn = VN_0; vn < E1HVN_MAX; vn++) {
2524                         if (vn == BP_E1HVN(bp))
2525                                 continue;
2526
2527                         func = ((vn << 1) | port);
2528                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_0 +
2529                                (LINK_SYNC_ATTENTION_BIT_FUNC_0 + func)*4, 1);
2530                 }
2531
2532                 if (bp->link_vars.link_up) {
2533                         int i;
2534
2535                         /* Init rate shaping and fairness contexts */
2536                         bnx2x_init_port_minmax(bp);
2537
2538                         for (vn = VN_0; vn < E1HVN_MAX; vn++)
2539                                 bnx2x_init_vn_minmax(bp, 2*vn + port);
2540
2541                         /* Store it to internal memory */
2542                         for (i = 0;
2543                              i < sizeof(struct cmng_struct_per_port) / 4; i++)
2544                                 REG_WR(bp, BAR_XSTRORM_INTMEM +
2545                                   XSTORM_CMNG_PER_PORT_VARS_OFFSET(port) + i*4,
2546                                        ((u32 *)(&bp->cmng))[i]);
2547                 }
2548         }
2549 }
2550
2551 static void bnx2x__link_status_update(struct bnx2x *bp)
2552 {
2553         if ((bp->state != BNX2X_STATE_OPEN) || (bp->flags & MF_FUNC_DIS))
2554                 return;
2555
2556         bnx2x_link_status_update(&bp->link_params, &bp->link_vars);
2557
2558         if (bp->link_vars.link_up)
2559                 bnx2x_stats_handle(bp, STATS_EVENT_LINK_UP);
2560         else
2561                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2562
2563         bnx2x_calc_vn_weight_sum(bp);
2564
2565         /* indicate link status */
2566         bnx2x_link_report(bp);
2567 }
2568
2569 static void bnx2x_pmf_update(struct bnx2x *bp)
2570 {
2571         int port = BP_PORT(bp);
2572         u32 val;
2573
2574         bp->port.pmf = 1;
2575         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2576
2577         /* enable nig attention */
2578         val = (0xff0f | (1 << (BP_E1HVN(bp) + 4)));
2579         REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, val);
2580         REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, val);
2581
2582         bnx2x_stats_handle(bp, STATS_EVENT_PMF);
2583 }
2584
2585 /* end of Link */
2586
2587 /* slow path */
2588
2589 /*
2590  * General service functions
2591  */
2592
2593 /* send the MCP a request, block until there is a reply */
2594 u32 bnx2x_fw_command(struct bnx2x *bp, u32 command)
2595 {
2596         int func = BP_FUNC(bp);
2597         u32 seq = ++bp->fw_seq;
2598         u32 rc = 0;
2599         u32 cnt = 1;
2600         u8 delay = CHIP_REV_IS_SLOW(bp) ? 100 : 10;
2601
2602         mutex_lock(&bp->fw_mb_mutex);
2603         SHMEM_WR(bp, func_mb[func].drv_mb_header, (command | seq));
2604         DP(BNX2X_MSG_MCP, "wrote command (%x) to FW MB\n", (command | seq));
2605
2606         do {
2607                 /* let the FW do it's magic ... */
2608                 msleep(delay);
2609
2610                 rc = SHMEM_RD(bp, func_mb[func].fw_mb_header);
2611
2612                 /* Give the FW up to 5 second (500*10ms) */
2613         } while ((seq != (rc & FW_MSG_SEQ_NUMBER_MASK)) && (cnt++ < 500));
2614
2615         DP(BNX2X_MSG_MCP, "[after %d ms] read (%x) seq is (%x) from FW MB\n",
2616            cnt*delay, rc, seq);
2617
2618         /* is this a reply to our command? */
2619         if (seq == (rc & FW_MSG_SEQ_NUMBER_MASK))
2620                 rc &= FW_MSG_CODE_MASK;
2621         else {
2622                 /* FW BUG! */
2623                 BNX2X_ERR("FW failed to respond!\n");
2624                 bnx2x_fw_dump(bp);
2625                 rc = 0;
2626         }
2627         mutex_unlock(&bp->fw_mb_mutex);
2628
2629         return rc;
2630 }
2631
2632 static void bnx2x_set_eth_mac_addr_e1h(struct bnx2x *bp, int set);
2633 static void bnx2x_set_rx_mode(struct net_device *dev);
2634
2635 static void bnx2x_e1h_disable(struct bnx2x *bp)
2636 {
2637         int port = BP_PORT(bp);
2638
2639         netif_tx_disable(bp->dev);
2640
2641         REG_WR(bp, NIG_REG_LLH0_FUNC_EN + port*8, 0);
2642
2643         netif_carrier_off(bp->dev);
2644 }
2645
2646 static void bnx2x_e1h_enable(struct bnx2x *bp)
2647 {
2648         int port = BP_PORT(bp);
2649
2650         REG_WR(bp, NIG_REG_LLH0_FUNC_EN + port*8, 1);
2651
2652         /* Tx queue should be only reenabled */
2653         netif_tx_wake_all_queues(bp->dev);
2654
2655         /*
2656          * Should not call netif_carrier_on since it will be called if the link
2657          * is up when checking for link state
2658          */
2659 }
2660
2661 static void bnx2x_update_min_max(struct bnx2x *bp)
2662 {
2663         int port = BP_PORT(bp);
2664         int vn, i;
2665
2666         /* Init rate shaping and fairness contexts */
2667         bnx2x_init_port_minmax(bp);
2668
2669         bnx2x_calc_vn_weight_sum(bp);
2670
2671         for (vn = VN_0; vn < E1HVN_MAX; vn++)
2672                 bnx2x_init_vn_minmax(bp, 2*vn + port);
2673
2674         if (bp->port.pmf) {
2675                 int func;
2676
2677                 /* Set the attention towards other drivers on the same port */
2678                 for (vn = VN_0; vn < E1HVN_MAX; vn++) {
2679                         if (vn == BP_E1HVN(bp))
2680                                 continue;
2681
2682                         func = ((vn << 1) | port);
2683                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_0 +
2684                                (LINK_SYNC_ATTENTION_BIT_FUNC_0 + func)*4, 1);
2685                 }
2686
2687                 /* Store it to internal memory */
2688                 for (i = 0; i < sizeof(struct cmng_struct_per_port) / 4; i++)
2689                         REG_WR(bp, BAR_XSTRORM_INTMEM +
2690                                XSTORM_CMNG_PER_PORT_VARS_OFFSET(port) + i*4,
2691                                ((u32 *)(&bp->cmng))[i]);
2692         }
2693 }
2694
2695 static void bnx2x_dcc_event(struct bnx2x *bp, u32 dcc_event)
2696 {
2697         DP(BNX2X_MSG_MCP, "dcc_event 0x%x\n", dcc_event);
2698
2699         if (dcc_event & DRV_STATUS_DCC_DISABLE_ENABLE_PF) {
2700
2701                 /*
2702                  * This is the only place besides the function initialization
2703                  * where the bp->flags can change so it is done without any
2704                  * locks
2705                  */
2706                 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
2707                         DP(NETIF_MSG_IFDOWN, "mf_cfg function disabled\n");
2708                         bp->flags |= MF_FUNC_DIS;
2709
2710                         bnx2x_e1h_disable(bp);
2711                 } else {
2712                         DP(NETIF_MSG_IFUP, "mf_cfg function enabled\n");
2713                         bp->flags &= ~MF_FUNC_DIS;
2714
2715                         bnx2x_e1h_enable(bp);
2716                 }
2717                 dcc_event &= ~DRV_STATUS_DCC_DISABLE_ENABLE_PF;
2718         }
2719         if (dcc_event & DRV_STATUS_DCC_BANDWIDTH_ALLOCATION) {
2720
2721                 bnx2x_update_min_max(bp);
2722                 dcc_event &= ~DRV_STATUS_DCC_BANDWIDTH_ALLOCATION;
2723         }
2724
2725         /* Report results to MCP */
2726         if (dcc_event)
2727                 bnx2x_fw_command(bp, DRV_MSG_CODE_DCC_FAILURE);
2728         else
2729                 bnx2x_fw_command(bp, DRV_MSG_CODE_DCC_OK);
2730 }
2731
2732 /* must be called under the spq lock */
2733 static inline struct eth_spe *bnx2x_sp_get_next(struct bnx2x *bp)
2734 {
2735         struct eth_spe *next_spe = bp->spq_prod_bd;
2736
2737         if (bp->spq_prod_bd == bp->spq_last_bd) {
2738                 bp->spq_prod_bd = bp->spq;
2739                 bp->spq_prod_idx = 0;
2740                 DP(NETIF_MSG_TIMER, "end of spq\n");
2741         } else {
2742                 bp->spq_prod_bd++;
2743                 bp->spq_prod_idx++;
2744         }
2745         return next_spe;
2746 }
2747
2748 /* must be called under the spq lock */
2749 static inline void bnx2x_sp_prod_update(struct bnx2x *bp)
2750 {
2751         int func = BP_FUNC(bp);
2752
2753         /* Make sure that BD data is updated before writing the producer */
2754         wmb();
2755
2756         REG_WR(bp, BAR_XSTRORM_INTMEM + XSTORM_SPQ_PROD_OFFSET(func),
2757                bp->spq_prod_idx);
2758         mmiowb();
2759 }
2760
2761 /* the slow path queue is odd since completions arrive on the fastpath ring */
2762 static int bnx2x_sp_post(struct bnx2x *bp, int command, int cid,
2763                          u32 data_hi, u32 data_lo, int common)
2764 {
2765         struct eth_spe *spe;
2766
2767 #ifdef BNX2X_STOP_ON_ERROR
2768         if (unlikely(bp->panic))
2769                 return -EIO;
2770 #endif
2771
2772         spin_lock_bh(&bp->spq_lock);
2773
2774         if (!bp->spq_left) {
2775                 BNX2X_ERR("BUG! SPQ ring full!\n");
2776                 spin_unlock_bh(&bp->spq_lock);
2777                 bnx2x_panic();
2778                 return -EBUSY;
2779         }
2780
2781         spe = bnx2x_sp_get_next(bp);
2782
2783         /* CID needs port number to be encoded int it */
2784         spe->hdr.conn_and_cmd_data =
2785                         cpu_to_le32((command << SPE_HDR_CMD_ID_SHIFT) |
2786                                     HW_CID(bp, cid));
2787         spe->hdr.type = cpu_to_le16(ETH_CONNECTION_TYPE);
2788         if (common)
2789                 spe->hdr.type |=
2790                         cpu_to_le16((1 << SPE_HDR_COMMON_RAMROD_SHIFT));
2791
2792         spe->data.mac_config_addr.hi = cpu_to_le32(data_hi);
2793         spe->data.mac_config_addr.lo = cpu_to_le32(data_lo);
2794
2795         bp->spq_left--;
2796
2797         DP(BNX2X_MSG_SP/*NETIF_MSG_TIMER*/,
2798            "SPQE[%x] (%x:%x)  command %d  hw_cid %x  data (%x:%x)  left %x\n",
2799            bp->spq_prod_idx, (u32)U64_HI(bp->spq_mapping),
2800            (u32)(U64_LO(bp->spq_mapping) +
2801            (void *)bp->spq_prod_bd - (void *)bp->spq), command,
2802            HW_CID(bp, cid), data_hi, data_lo, bp->spq_left);
2803
2804         bnx2x_sp_prod_update(bp);
2805         spin_unlock_bh(&bp->spq_lock);
2806         return 0;
2807 }
2808
2809 /* acquire split MCP access lock register */
2810 static int bnx2x_acquire_alr(struct bnx2x *bp)
2811 {
2812         u32 j, val;
2813         int rc = 0;
2814
2815         might_sleep();
2816         for (j = 0; j < 1000; j++) {
2817                 val = (1UL << 31);
2818                 REG_WR(bp, GRCBASE_MCP + 0x9c, val);
2819                 val = REG_RD(bp, GRCBASE_MCP + 0x9c);
2820                 if (val & (1L << 31))
2821                         break;
2822
2823                 msleep(5);
2824         }
2825         if (!(val & (1L << 31))) {
2826                 BNX2X_ERR("Cannot acquire MCP access lock register\n");
2827                 rc = -EBUSY;
2828         }
2829
2830         return rc;
2831 }
2832
2833 /* release split MCP access lock register */
2834 static void bnx2x_release_alr(struct bnx2x *bp)
2835 {
2836         REG_WR(bp, GRCBASE_MCP + 0x9c, 0);
2837 }
2838
2839 static inline u16 bnx2x_update_dsb_idx(struct bnx2x *bp)
2840 {
2841         struct host_def_status_block *def_sb = bp->def_status_blk;
2842         u16 rc = 0;
2843
2844         barrier(); /* status block is written to by the chip */
2845         if (bp->def_att_idx != def_sb->atten_status_block.attn_bits_index) {
2846                 bp->def_att_idx = def_sb->atten_status_block.attn_bits_index;
2847                 rc |= 1;
2848         }
2849         if (bp->def_c_idx != def_sb->c_def_status_block.status_block_index) {
2850                 bp->def_c_idx = def_sb->c_def_status_block.status_block_index;
2851                 rc |= 2;
2852         }
2853         if (bp->def_u_idx != def_sb->u_def_status_block.status_block_index) {
2854                 bp->def_u_idx = def_sb->u_def_status_block.status_block_index;
2855                 rc |= 4;
2856         }
2857         if (bp->def_x_idx != def_sb->x_def_status_block.status_block_index) {
2858                 bp->def_x_idx = def_sb->x_def_status_block.status_block_index;
2859                 rc |= 8;
2860         }
2861         if (bp->def_t_idx != def_sb->t_def_status_block.status_block_index) {
2862                 bp->def_t_idx = def_sb->t_def_status_block.status_block_index;
2863                 rc |= 16;
2864         }
2865         return rc;
2866 }
2867
2868 /*
2869  * slow path service functions
2870  */
2871
2872 static void bnx2x_attn_int_asserted(struct bnx2x *bp, u32 asserted)
2873 {
2874         int port = BP_PORT(bp);
2875         u32 hc_addr = (HC_REG_COMMAND_REG + port*32 +
2876                        COMMAND_REG_ATTN_BITS_SET);
2877         u32 aeu_addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
2878                               MISC_REG_AEU_MASK_ATTN_FUNC_0;
2879         u32 nig_int_mask_addr = port ? NIG_REG_MASK_INTERRUPT_PORT1 :
2880                                        NIG_REG_MASK_INTERRUPT_PORT0;
2881         u32 aeu_mask;
2882         u32 nig_mask = 0;
2883
2884         if (bp->attn_state & asserted)
2885                 BNX2X_ERR("IGU ERROR\n");
2886
2887         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
2888         aeu_mask = REG_RD(bp, aeu_addr);
2889
2890         DP(NETIF_MSG_HW, "aeu_mask %x  newly asserted %x\n",
2891            aeu_mask, asserted);
2892         aeu_mask &= ~(asserted & 0x3ff);
2893         DP(NETIF_MSG_HW, "new mask %x\n", aeu_mask);
2894
2895         REG_WR(bp, aeu_addr, aeu_mask);
2896         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
2897
2898         DP(NETIF_MSG_HW, "attn_state %x\n", bp->attn_state);
2899         bp->attn_state |= asserted;
2900         DP(NETIF_MSG_HW, "new state %x\n", bp->attn_state);
2901
2902         if (asserted & ATTN_HARD_WIRED_MASK) {
2903                 if (asserted & ATTN_NIG_FOR_FUNC) {
2904
2905                         bnx2x_acquire_phy_lock(bp);
2906
2907                         /* save nig interrupt mask */
2908                         nig_mask = REG_RD(bp, nig_int_mask_addr);
2909                         REG_WR(bp, nig_int_mask_addr, 0);
2910
2911                         bnx2x_link_attn(bp);
2912
2913                         /* handle unicore attn? */
2914                 }
2915                 if (asserted & ATTN_SW_TIMER_4_FUNC)
2916                         DP(NETIF_MSG_HW, "ATTN_SW_TIMER_4_FUNC!\n");
2917
2918                 if (asserted & GPIO_2_FUNC)
2919                         DP(NETIF_MSG_HW, "GPIO_2_FUNC!\n");
2920
2921                 if (asserted & GPIO_3_FUNC)
2922                         DP(NETIF_MSG_HW, "GPIO_3_FUNC!\n");
2923
2924                 if (asserted & GPIO_4_FUNC)
2925                         DP(NETIF_MSG_HW, "GPIO_4_FUNC!\n");
2926
2927                 if (port == 0) {
2928                         if (asserted & ATTN_GENERAL_ATTN_1) {
2929                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_1!\n");
2930                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_1, 0x0);
2931                         }
2932                         if (asserted & ATTN_GENERAL_ATTN_2) {
2933                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_2!\n");
2934                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_2, 0x0);
2935                         }
2936                         if (asserted & ATTN_GENERAL_ATTN_3) {
2937                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_3!\n");
2938                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_3, 0x0);
2939                         }
2940                 } else {
2941                         if (asserted & ATTN_GENERAL_ATTN_4) {
2942                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_4!\n");
2943                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_4, 0x0);
2944                         }
2945                         if (asserted & ATTN_GENERAL_ATTN_5) {
2946                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_5!\n");
2947                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_5, 0x0);
2948                         }
2949                         if (asserted & ATTN_GENERAL_ATTN_6) {
2950                                 DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_6!\n");
2951                                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_6, 0x0);
2952                         }
2953                 }
2954
2955         } /* if hardwired */
2956
2957         DP(NETIF_MSG_HW, "about to mask 0x%08x at HC addr 0x%x\n",
2958            asserted, hc_addr);
2959         REG_WR(bp, hc_addr, asserted);
2960
2961         /* now set back the mask */
2962         if (asserted & ATTN_NIG_FOR_FUNC) {
2963                 REG_WR(bp, nig_int_mask_addr, nig_mask);
2964                 bnx2x_release_phy_lock(bp);
2965         }
2966 }
2967
2968 static inline void bnx2x_fan_failure(struct bnx2x *bp)
2969 {
2970         int port = BP_PORT(bp);
2971
2972         /* mark the failure */
2973         bp->link_params.ext_phy_config &= ~PORT_HW_CFG_XGXS_EXT_PHY_TYPE_MASK;
2974         bp->link_params.ext_phy_config |= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE;
2975         SHMEM_WR(bp, dev_info.port_hw_config[port].external_phy_config,
2976                  bp->link_params.ext_phy_config);
2977
2978         /* log the failure */
2979         netdev_err(bp->dev, "Fan Failure on Network Controller has caused"
2980                " the driver to shutdown the card to prevent permanent"
2981                " damage.  Please contact OEM Support for assistance\n");
2982 }
2983
2984 static inline void bnx2x_attn_int_deasserted0(struct bnx2x *bp, u32 attn)
2985 {
2986         int port = BP_PORT(bp);
2987         int reg_offset;
2988         u32 val, swap_val, swap_override;
2989
2990         reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_0 :
2991                              MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0);
2992
2993         if (attn & AEU_INPUTS_ATTN_BITS_SPIO5) {
2994
2995                 val = REG_RD(bp, reg_offset);
2996                 val &= ~AEU_INPUTS_ATTN_BITS_SPIO5;
2997                 REG_WR(bp, reg_offset, val);
2998
2999                 BNX2X_ERR("SPIO5 hw attention\n");
3000
3001                 /* Fan failure attention */
3002                 switch (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config)) {
3003                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101:
3004                         /* Low power mode is controlled by GPIO 2 */
3005                         bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_2,
3006                                        MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
3007                         /* The PHY reset is controlled by GPIO 1 */
3008                         bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_1,
3009                                        MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
3010                         break;
3011
3012                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727:
3013                         /* The PHY reset is controlled by GPIO 1 */
3014                         /* fake the port number to cancel the swap done in
3015                            set_gpio() */
3016                         swap_val = REG_RD(bp, NIG_REG_PORT_SWAP);
3017                         swap_override = REG_RD(bp, NIG_REG_STRAP_OVERRIDE);
3018                         port = (swap_val && swap_override) ^ 1;
3019                         bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_1,
3020                                        MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
3021                         break;
3022
3023                 default:
3024                         break;
3025                 }
3026                 bnx2x_fan_failure(bp);
3027         }
3028
3029         if (attn & (AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_0 |
3030                     AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_1)) {
3031                 bnx2x_acquire_phy_lock(bp);
3032                 bnx2x_handle_module_detect_int(&bp->link_params);
3033                 bnx2x_release_phy_lock(bp);
3034         }
3035
3036         if (attn & HW_INTERRUT_ASSERT_SET_0) {
3037
3038                 val = REG_RD(bp, reg_offset);
3039                 val &= ~(attn & HW_INTERRUT_ASSERT_SET_0);
3040                 REG_WR(bp, reg_offset, val);
3041
3042                 BNX2X_ERR("FATAL HW block attention set0 0x%x\n",
3043                           (u32)(attn & HW_INTERRUT_ASSERT_SET_0));
3044                 bnx2x_panic();
3045         }
3046 }
3047
3048 static inline void bnx2x_attn_int_deasserted1(struct bnx2x *bp, u32 attn)
3049 {
3050         u32 val;
3051
3052         if (attn & AEU_INPUTS_ATTN_BITS_DOORBELLQ_HW_INTERRUPT) {
3053
3054                 val = REG_RD(bp, DORQ_REG_DORQ_INT_STS_CLR);
3055                 BNX2X_ERR("DB hw attention 0x%x\n", val);
3056                 /* DORQ discard attention */
3057                 if (val & 0x2)
3058                         BNX2X_ERR("FATAL error from DORQ\n");
3059         }
3060
3061         if (attn & HW_INTERRUT_ASSERT_SET_1) {
3062
3063                 int port = BP_PORT(bp);
3064                 int reg_offset;
3065
3066                 reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_1 :
3067                                      MISC_REG_AEU_ENABLE1_FUNC_0_OUT_1);
3068
3069                 val = REG_RD(bp, reg_offset);
3070                 val &= ~(attn & HW_INTERRUT_ASSERT_SET_1);
3071                 REG_WR(bp, reg_offset, val);
3072
3073                 BNX2X_ERR("FATAL HW block attention set1 0x%x\n",
3074                           (u32)(attn & HW_INTERRUT_ASSERT_SET_1));
3075                 bnx2x_panic();
3076         }
3077 }
3078
3079 static inline void bnx2x_attn_int_deasserted2(struct bnx2x *bp, u32 attn)
3080 {
3081         u32 val;
3082
3083         if (attn & AEU_INPUTS_ATTN_BITS_CFC_HW_INTERRUPT) {
3084
3085                 val = REG_RD(bp, CFC_REG_CFC_INT_STS_CLR);
3086                 BNX2X_ERR("CFC hw attention 0x%x\n", val);
3087                 /* CFC error attention */
3088                 if (val & 0x2)
3089                         BNX2X_ERR("FATAL error from CFC\n");
3090         }
3091
3092         if (attn & AEU_INPUTS_ATTN_BITS_PXP_HW_INTERRUPT) {
3093
3094                 val = REG_RD(bp, PXP_REG_PXP_INT_STS_CLR_0);
3095                 BNX2X_ERR("PXP hw attention 0x%x\n", val);
3096                 /* RQ_USDMDP_FIFO_OVERFLOW */
3097                 if (val & 0x18000)
3098                         BNX2X_ERR("FATAL error from PXP\n");
3099         }
3100
3101         if (attn & HW_INTERRUT_ASSERT_SET_2) {
3102
3103                 int port = BP_PORT(bp);
3104                 int reg_offset;
3105
3106                 reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_2 :
3107                                      MISC_REG_AEU_ENABLE1_FUNC_0_OUT_2);
3108
3109                 val = REG_RD(bp, reg_offset);
3110                 val &= ~(attn & HW_INTERRUT_ASSERT_SET_2);
3111                 REG_WR(bp, reg_offset, val);
3112
3113                 BNX2X_ERR("FATAL HW block attention set2 0x%x\n",
3114                           (u32)(attn & HW_INTERRUT_ASSERT_SET_2));
3115                 bnx2x_panic();
3116         }
3117 }
3118
3119 static inline void bnx2x_attn_int_deasserted3(struct bnx2x *bp, u32 attn)
3120 {
3121         u32 val;
3122
3123         if (attn & EVEREST_GEN_ATTN_IN_USE_MASK) {
3124
3125                 if (attn & BNX2X_PMF_LINK_ASSERT) {
3126                         int func = BP_FUNC(bp);
3127
3128                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_12 + func*4, 0);
3129                         bp->mf_config = SHMEM_RD(bp,
3130                                            mf_cfg.func_mf_config[func].config);
3131                         val = SHMEM_RD(bp, func_mb[func].drv_status);
3132                         if (val & DRV_STATUS_DCC_EVENT_MASK)
3133                                 bnx2x_dcc_event(bp,
3134                                             (val & DRV_STATUS_DCC_EVENT_MASK));
3135                         bnx2x__link_status_update(bp);
3136                         if ((bp->port.pmf == 0) && (val & DRV_STATUS_PMF))
3137                                 bnx2x_pmf_update(bp);
3138
3139                 } else if (attn & BNX2X_MC_ASSERT_BITS) {
3140
3141                         BNX2X_ERR("MC assert!\n");
3142                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_10, 0);
3143                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_9, 0);
3144                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_8, 0);
3145                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_7, 0);
3146                         bnx2x_panic();
3147
3148                 } else if (attn & BNX2X_MCP_ASSERT) {
3149
3150                         BNX2X_ERR("MCP assert!\n");
3151                         REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_11, 0);
3152                         bnx2x_fw_dump(bp);
3153
3154                 } else
3155                         BNX2X_ERR("Unknown HW assert! (attn 0x%x)\n", attn);
3156         }
3157
3158         if (attn & EVEREST_LATCHED_ATTN_IN_USE_MASK) {
3159                 BNX2X_ERR("LATCHED attention 0x%08x (masked)\n", attn);
3160                 if (attn & BNX2X_GRC_TIMEOUT) {
3161                         val = CHIP_IS_E1H(bp) ?
3162                                 REG_RD(bp, MISC_REG_GRC_TIMEOUT_ATTN) : 0;
3163                         BNX2X_ERR("GRC time-out 0x%08x\n", val);
3164                 }
3165                 if (attn & BNX2X_GRC_RSV) {
3166                         val = CHIP_IS_E1H(bp) ?
3167                                 REG_RD(bp, MISC_REG_GRC_RSV_ATTN) : 0;
3168                         BNX2X_ERR("GRC reserved 0x%08x\n", val);
3169                 }
3170                 REG_WR(bp, MISC_REG_AEU_CLR_LATCH_SIGNAL, 0x7ff);
3171         }
3172 }
3173
3174 static int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode);
3175 static int bnx2x_nic_load(struct bnx2x *bp, int load_mode);
3176
3177
3178 #define BNX2X_MISC_GEN_REG      MISC_REG_GENERIC_POR_1
3179 #define LOAD_COUNTER_BITS       16 /* Number of bits for load counter */
3180 #define LOAD_COUNTER_MASK       (((u32)0x1 << LOAD_COUNTER_BITS) - 1)
3181 #define RESET_DONE_FLAG_MASK    (~LOAD_COUNTER_MASK)
3182 #define RESET_DONE_FLAG_SHIFT   LOAD_COUNTER_BITS
3183 #define CHIP_PARITY_SUPPORTED(bp)   (CHIP_IS_E1(bp) || CHIP_IS_E1H(bp))
3184 /*
3185  * should be run under rtnl lock
3186  */
3187 static inline void bnx2x_set_reset_done(struct bnx2x *bp)
3188 {
3189         u32 val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3190         val &= ~(1 << RESET_DONE_FLAG_SHIFT);
3191         REG_WR(bp, BNX2X_MISC_GEN_REG, val);
3192         barrier();
3193         mmiowb();
3194 }
3195
3196 /*
3197  * should be run under rtnl lock
3198  */
3199 static inline void bnx2x_set_reset_in_progress(struct bnx2x *bp)
3200 {
3201         u32 val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3202         val |= (1 << 16);
3203         REG_WR(bp, BNX2X_MISC_GEN_REG, val);
3204         barrier();
3205         mmiowb();
3206 }
3207
3208 /*
3209  * should be run under rtnl lock
3210  */
3211 static inline bool bnx2x_reset_is_done(struct bnx2x *bp)
3212 {
3213         u32 val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3214         DP(NETIF_MSG_HW, "GEN_REG_VAL=0x%08x\n", val);
3215         return (val & RESET_DONE_FLAG_MASK) ? false : true;
3216 }
3217
3218 /*
3219  * should be run under rtnl lock
3220  */
3221 static inline void bnx2x_inc_load_cnt(struct bnx2x *bp)
3222 {
3223         u32 val1, val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3224
3225         DP(NETIF_MSG_HW, "Old GEN_REG_VAL=0x%08x\n", val);
3226
3227         val1 = ((val & LOAD_COUNTER_MASK) + 1) & LOAD_COUNTER_MASK;
3228         REG_WR(bp, BNX2X_MISC_GEN_REG, (val & RESET_DONE_FLAG_MASK) | val1);
3229         barrier();
3230         mmiowb();
3231 }
3232
3233 /*
3234  * should be run under rtnl lock
3235  */
3236 static inline u32 bnx2x_dec_load_cnt(struct bnx2x *bp)
3237 {
3238         u32 val1, val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3239
3240         DP(NETIF_MSG_HW, "Old GEN_REG_VAL=0x%08x\n", val);
3241
3242         val1 = ((val & LOAD_COUNTER_MASK) - 1) & LOAD_COUNTER_MASK;
3243         REG_WR(bp, BNX2X_MISC_GEN_REG, (val & RESET_DONE_FLAG_MASK) | val1);
3244         barrier();
3245         mmiowb();
3246
3247         return val1;
3248 }
3249
3250 /*
3251  * should be run under rtnl lock
3252  */
3253 static inline u32 bnx2x_get_load_cnt(struct bnx2x *bp)
3254 {
3255         return REG_RD(bp, BNX2X_MISC_GEN_REG) & LOAD_COUNTER_MASK;
3256 }
3257
3258 static inline void bnx2x_clear_load_cnt(struct bnx2x *bp)
3259 {
3260         u32 val = REG_RD(bp, BNX2X_MISC_GEN_REG);
3261         REG_WR(bp, BNX2X_MISC_GEN_REG, val & (~LOAD_COUNTER_MASK));
3262 }
3263
3264 static inline void _print_next_block(int idx, const char *blk)
3265 {
3266         if (idx)
3267                 pr_cont(", ");
3268         pr_cont("%s", blk);
3269 }
3270
3271 static inline int bnx2x_print_blocks_with_parity0(u32 sig, int par_num)
3272 {
3273         int i = 0;
3274         u32 cur_bit = 0;
3275         for (i = 0; sig; i++) {
3276                 cur_bit = ((u32)0x1 << i);
3277                 if (sig & cur_bit) {
3278                         switch (cur_bit) {
3279                         case AEU_INPUTS_ATTN_BITS_BRB_PARITY_ERROR:
3280                                 _print_next_block(par_num++, "BRB");
3281                                 break;
3282                         case AEU_INPUTS_ATTN_BITS_PARSER_PARITY_ERROR:
3283                                 _print_next_block(par_num++, "PARSER");
3284                                 break;
3285                         case AEU_INPUTS_ATTN_BITS_TSDM_PARITY_ERROR:
3286                                 _print_next_block(par_num++, "TSDM");
3287                                 break;
3288                         case AEU_INPUTS_ATTN_BITS_SEARCHER_PARITY_ERROR:
3289                                 _print_next_block(par_num++, "SEARCHER");
3290                                 break;
3291                         case AEU_INPUTS_ATTN_BITS_TSEMI_PARITY_ERROR:
3292                                 _print_next_block(par_num++, "TSEMI");
3293                                 break;
3294                         }
3295
3296                         /* Clear the bit */
3297                         sig &= ~cur_bit;
3298                 }
3299         }
3300
3301         return par_num;
3302 }
3303
3304 static inline int bnx2x_print_blocks_with_parity1(u32 sig, int par_num)
3305 {
3306         int i = 0;
3307         u32 cur_bit = 0;
3308         for (i = 0; sig; i++) {
3309                 cur_bit = ((u32)0x1 << i);
3310                 if (sig & cur_bit) {
3311                         switch (cur_bit) {
3312                         case AEU_INPUTS_ATTN_BITS_PBCLIENT_PARITY_ERROR:
3313                                 _print_next_block(par_num++, "PBCLIENT");
3314                                 break;
3315                         case AEU_INPUTS_ATTN_BITS_QM_PARITY_ERROR:
3316                                 _print_next_block(par_num++, "QM");
3317                                 break;
3318                         case AEU_INPUTS_ATTN_BITS_XSDM_PARITY_ERROR:
3319                                 _print_next_block(par_num++, "XSDM");
3320                                 break;
3321                         case AEU_INPUTS_ATTN_BITS_XSEMI_PARITY_ERROR:
3322                                 _print_next_block(par_num++, "XSEMI");
3323                                 break;
3324                         case AEU_INPUTS_ATTN_BITS_DOORBELLQ_PARITY_ERROR:
3325                                 _print_next_block(par_num++, "DOORBELLQ");
3326                                 break;
3327                         case AEU_INPUTS_ATTN_BITS_VAUX_PCI_CORE_PARITY_ERROR:
3328                                 _print_next_block(par_num++, "VAUX PCI CORE");
3329                                 break;
3330                         case AEU_INPUTS_ATTN_BITS_DEBUG_PARITY_ERROR:
3331                                 _print_next_block(par_num++, "DEBUG");
3332                                 break;
3333                         case AEU_INPUTS_ATTN_BITS_USDM_PARITY_ERROR:
3334                                 _print_next_block(par_num++, "USDM");
3335                                 break;
3336                         case AEU_INPUTS_ATTN_BITS_USEMI_PARITY_ERROR:
3337                                 _print_next_block(par_num++, "USEMI");
3338                                 break;
3339                         case AEU_INPUTS_ATTN_BITS_UPB_PARITY_ERROR:
3340                                 _print_next_block(par_num++, "UPB");
3341                                 break;
3342                         case AEU_INPUTS_ATTN_BITS_CSDM_PARITY_ERROR:
3343                                 _print_next_block(par_num++, "CSDM");
3344                                 break;
3345                         }
3346
3347                         /* Clear the bit */
3348                         sig &= ~cur_bit;
3349                 }
3350         }
3351
3352         return par_num;
3353 }
3354
3355 static inline int bnx2x_print_blocks_with_parity2(u32 sig, int par_num)
3356 {
3357         int i = 0;
3358         u32 cur_bit = 0;
3359         for (i = 0; sig; i++) {
3360                 cur_bit = ((u32)0x1 << i);
3361                 if (sig & cur_bit) {
3362                         switch (cur_bit) {
3363                         case AEU_INPUTS_ATTN_BITS_CSEMI_PARITY_ERROR:
3364                                 _print_next_block(par_num++, "CSEMI");
3365                                 break;
3366                         case AEU_INPUTS_ATTN_BITS_PXP_PARITY_ERROR:
3367                                 _print_next_block(par_num++, "PXP");
3368                                 break;
3369                         case AEU_IN_ATTN_BITS_PXPPCICLOCKCLIENT_PARITY_ERROR:
3370                                 _print_next_block(par_num++,
3371                                         "PXPPCICLOCKCLIENT");
3372                                 break;
3373                         case AEU_INPUTS_ATTN_BITS_CFC_PARITY_ERROR:
3374                                 _print_next_block(par_num++, "CFC");
3375                                 break;
3376                         case AEU_INPUTS_ATTN_BITS_CDU_PARITY_ERROR:
3377                                 _print_next_block(par_num++, "CDU");
3378                                 break;
3379                         case AEU_INPUTS_ATTN_BITS_IGU_PARITY_ERROR:
3380                                 _print_next_block(par_num++, "IGU");
3381                                 break;
3382                         case AEU_INPUTS_ATTN_BITS_MISC_PARITY_ERROR:
3383                                 _print_next_block(par_num++, "MISC");
3384                                 break;
3385                         }
3386
3387                         /* Clear the bit */
3388                         sig &= ~cur_bit;
3389                 }
3390         }
3391
3392         return par_num;
3393 }
3394
3395 static inline int bnx2x_print_blocks_with_parity3(u32 sig, int par_num)
3396 {
3397         int i = 0;
3398         u32 cur_bit = 0;
3399         for (i = 0; sig; i++) {
3400                 cur_bit = ((u32)0x1 << i);
3401                 if (sig & cur_bit) {
3402                         switch (cur_bit) {
3403                         case AEU_INPUTS_ATTN_BITS_MCP_LATCHED_ROM_PARITY:
3404                                 _print_next_block(par_num++, "MCP ROM");
3405                                 break;
3406                         case AEU_INPUTS_ATTN_BITS_MCP_LATCHED_UMP_RX_PARITY:
3407                                 _print_next_block(par_num++, "MCP UMP RX");
3408                                 break;
3409                         case AEU_INPUTS_ATTN_BITS_MCP_LATCHED_UMP_TX_PARITY:
3410                                 _print_next_block(par_num++, "MCP UMP TX");
3411                                 break;
3412                         case AEU_INPUTS_ATTN_BITS_MCP_LATCHED_SCPAD_PARITY:
3413                                 _print_next_block(par_num++, "MCP SCPAD");
3414                                 break;
3415                         }
3416
3417                         /* Clear the bit */
3418                         sig &= ~cur_bit;
3419                 }
3420         }
3421
3422         return par_num;
3423 }
3424
3425 static inline bool bnx2x_parity_attn(struct bnx2x *bp, u32 sig0, u32 sig1,
3426                                      u32 sig2, u32 sig3)
3427 {
3428         if ((sig0 & HW_PRTY_ASSERT_SET_0) || (sig1 & HW_PRTY_ASSERT_SET_1) ||
3429             (sig2 & HW_PRTY_ASSERT_SET_2) || (sig3 & HW_PRTY_ASSERT_SET_3)) {
3430                 int par_num = 0;
3431                 DP(NETIF_MSG_HW, "Was parity error: HW block parity attention: "
3432                         "[0]:0x%08x [1]:0x%08x "
3433                         "[2]:0x%08x [3]:0x%08x\n",
3434                           sig0 & HW_PRTY_ASSERT_SET_0,
3435                           sig1 & HW_PRTY_ASSERT_SET_1,
3436                           sig2 & HW_PRTY_ASSERT_SET_2,
3437                           sig3 & HW_PRTY_ASSERT_SET_3);
3438                 printk(KERN_ERR"%s: Parity errors detected in blocks: ",
3439                        bp->dev->name);
3440                 par_num = bnx2x_print_blocks_with_parity0(
3441                         sig0 & HW_PRTY_ASSERT_SET_0, par_num);
3442                 par_num = bnx2x_print_blocks_with_parity1(
3443                         sig1 & HW_PRTY_ASSERT_SET_1, par_num);
3444                 par_num = bnx2x_print_blocks_with_parity2(
3445                         sig2 & HW_PRTY_ASSERT_SET_2, par_num);
3446                 par_num = bnx2x_print_blocks_with_parity3(
3447                         sig3 & HW_PRTY_ASSERT_SET_3, par_num);
3448                 printk("\n");
3449                 return true;
3450         } else
3451                 return false;
3452 }
3453
3454 static bool bnx2x_chk_parity_attn(struct bnx2x *bp)
3455 {
3456         struct attn_route attn;
3457         int port = BP_PORT(bp);
3458
3459         attn.sig[0] = REG_RD(bp,
3460                 MISC_REG_AEU_AFTER_INVERT_1_FUNC_0 +
3461                              port*4);
3462         attn.sig[1] = REG_RD(bp,
3463                 MISC_REG_AEU_AFTER_INVERT_2_FUNC_0 +
3464                              port*4);
3465         attn.sig[2] = REG_RD(bp,
3466                 MISC_REG_AEU_AFTER_INVERT_3_FUNC_0 +
3467                              port*4);
3468         attn.sig[3] = REG_RD(bp,
3469                 MISC_REG_AEU_AFTER_INVERT_4_FUNC_0 +
3470                              port*4);
3471
3472         return bnx2x_parity_attn(bp, attn.sig[0], attn.sig[1], attn.sig[2],
3473                                         attn.sig[3]);
3474 }
3475
3476 static void bnx2x_attn_int_deasserted(struct bnx2x *bp, u32 deasserted)
3477 {
3478         struct attn_route attn, *group_mask;
3479         int port = BP_PORT(bp);
3480         int index;
3481         u32 reg_addr;
3482         u32 val;
3483         u32 aeu_mask;
3484
3485         /* need to take HW lock because MCP or other port might also
3486            try to handle this event */
3487         bnx2x_acquire_alr(bp);
3488
3489         if (bnx2x_chk_parity_attn(bp)) {
3490                 bp->recovery_state = BNX2X_RECOVERY_INIT;
3491                 bnx2x_set_reset_in_progress(bp);
3492                 schedule_delayed_work(&bp->reset_task, 0);
3493                 /* Disable HW interrupts */
3494                 bnx2x_int_disable(bp);
3495                 bnx2x_release_alr(bp);
3496                 /* In case of parity errors don't handle attentions so that
3497                  * other function would "see" parity errors.
3498                  */
3499                 return;
3500         }
3501
3502         attn.sig[0] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_1_FUNC_0 + port*4);
3503         attn.sig[1] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_2_FUNC_0 + port*4);
3504         attn.sig[2] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_3_FUNC_0 + port*4);
3505         attn.sig[3] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_4_FUNC_0 + port*4);
3506         DP(NETIF_MSG_HW, "attn: %08x %08x %08x %08x\n",
3507            attn.sig[0], attn.sig[1], attn.sig[2], attn.sig[3]);
3508
3509         for (index = 0; index < MAX_DYNAMIC_ATTN_GRPS; index++) {
3510                 if (deasserted & (1 << index)) {
3511                         group_mask = &bp->attn_group[index];
3512
3513                         DP(NETIF_MSG_HW, "group[%d]: %08x %08x %08x %08x\n",
3514                            index, group_mask->sig[0], group_mask->sig[1],
3515                            group_mask->sig[2], group_mask->sig[3]);
3516
3517                         bnx2x_attn_int_deasserted3(bp,
3518                                         attn.sig[3] & group_mask->sig[3]);
3519                         bnx2x_attn_int_deasserted1(bp,
3520                                         attn.sig[1] & group_mask->sig[1]);
3521                         bnx2x_attn_int_deasserted2(bp,
3522                                         attn.sig[2] & group_mask->sig[2]);
3523                         bnx2x_attn_int_deasserted0(bp,
3524                                         attn.sig[0] & group_mask->sig[0]);
3525                 }
3526         }
3527
3528         bnx2x_release_alr(bp);
3529
3530         reg_addr = (HC_REG_COMMAND_REG + port*32 + COMMAND_REG_ATTN_BITS_CLR);
3531
3532         val = ~deasserted;
3533         DP(NETIF_MSG_HW, "about to mask 0x%08x at HC addr 0x%x\n",
3534            val, reg_addr);
3535         REG_WR(bp, reg_addr, val);
3536
3537         if (~bp->attn_state & deasserted)
3538                 BNX2X_ERR("IGU ERROR\n");
3539
3540         reg_addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
3541                           MISC_REG_AEU_MASK_ATTN_FUNC_0;
3542
3543         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
3544         aeu_mask = REG_RD(bp, reg_addr);
3545
3546         DP(NETIF_MSG_HW, "aeu_mask %x  newly deasserted %x\n",
3547            aeu_mask, deasserted);
3548         aeu_mask |= (deasserted & 0x3ff);
3549         DP(NETIF_MSG_HW, "new mask %x\n", aeu_mask);
3550
3551         REG_WR(bp, reg_addr, aeu_mask);
3552         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
3553
3554         DP(NETIF_MSG_HW, "attn_state %x\n", bp->attn_state);
3555         bp->attn_state &= ~deasserted;
3556         DP(NETIF_MSG_HW, "new state %x\n", bp->attn_state);
3557 }
3558
3559 static void bnx2x_attn_int(struct bnx2x *bp)
3560 {
3561         /* read local copy of bits */
3562         u32 attn_bits = le32_to_cpu(bp->def_status_blk->atten_status_block.
3563                                                                 attn_bits);
3564         u32 attn_ack = le32_to_cpu(bp->def_status_blk->atten_status_block.
3565                                                                 attn_bits_ack);
3566         u32 attn_state = bp->attn_state;
3567
3568         /* look for changed bits */
3569         u32 asserted   =  attn_bits & ~attn_ack & ~attn_state;
3570         u32 deasserted = ~attn_bits &  attn_ack &  attn_state;
3571
3572         DP(NETIF_MSG_HW,
3573            "attn_bits %x  attn_ack %x  asserted %x  deasserted %x\n",
3574            attn_bits, attn_ack, asserted, deasserted);
3575
3576         if (~(attn_bits ^ attn_ack) & (attn_bits ^ attn_state))
3577                 BNX2X_ERR("BAD attention state\n");
3578
3579         /* handle bits that were raised */
3580         if (asserted)
3581                 bnx2x_attn_int_asserted(bp, asserted);
3582
3583         if (deasserted)
3584                 bnx2x_attn_int_deasserted(bp, deasserted);
3585 }
3586
3587 static void bnx2x_sp_task(struct work_struct *work)
3588 {
3589         struct bnx2x *bp = container_of(work, struct bnx2x, sp_task.work);
3590         u16 status;
3591
3592         /* Return here if interrupt is disabled */
3593         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
3594                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
3595                 return;
3596         }
3597
3598         status = bnx2x_update_dsb_idx(bp);
3599 /*      if (status == 0)                                     */
3600 /*              BNX2X_ERR("spurious slowpath interrupt!\n"); */
3601
3602         DP(NETIF_MSG_INTR, "got a slowpath interrupt (status 0x%x)\n", status);
3603
3604         /* HW attentions */
3605         if (status & 0x1) {
3606                 bnx2x_attn_int(bp);
3607                 status &= ~0x1;
3608         }
3609
3610         /* CStorm events: STAT_QUERY */
3611         if (status & 0x2) {
3612                 DP(BNX2X_MSG_SP, "CStorm events: STAT_QUERY\n");
3613                 status &= ~0x2;
3614         }
3615
3616         if (unlikely(status))
3617                 DP(NETIF_MSG_INTR, "got an unknown interrupt! (status 0x%x)\n",
3618                    status);
3619
3620         bnx2x_ack_sb(bp, DEF_SB_ID, ATTENTION_ID, le16_to_cpu(bp->def_att_idx),
3621                      IGU_INT_NOP, 1);
3622         bnx2x_ack_sb(bp, DEF_SB_ID, USTORM_ID, le16_to_cpu(bp->def_u_idx),
3623                      IGU_INT_NOP, 1);
3624         bnx2x_ack_sb(bp, DEF_SB_ID, CSTORM_ID, le16_to_cpu(bp->def_c_idx),
3625                      IGU_INT_NOP, 1);
3626         bnx2x_ack_sb(bp, DEF_SB_ID, XSTORM_ID, le16_to_cpu(bp->def_x_idx),
3627                      IGU_INT_NOP, 1);
3628         bnx2x_ack_sb(bp, DEF_SB_ID, TSTORM_ID, le16_to_cpu(bp->def_t_idx),
3629                      IGU_INT_ENABLE, 1);
3630 }
3631
3632 static irqreturn_t bnx2x_msix_sp_int(int irq, void *dev_instance)
3633 {
3634         struct net_device *dev = dev_instance;
3635         struct bnx2x *bp = netdev_priv(dev);
3636
3637         /* Return here if interrupt is disabled */
3638         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
3639                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
3640                 return IRQ_HANDLED;
3641         }
3642
3643         bnx2x_ack_sb(bp, DEF_SB_ID, TSTORM_ID, 0, IGU_INT_DISABLE, 0);
3644
3645 #ifdef BNX2X_STOP_ON_ERROR
3646         if (unlikely(bp->panic))
3647                 return IRQ_HANDLED;
3648 #endif
3649
3650 #ifdef BCM_CNIC
3651         {
3652                 struct cnic_ops *c_ops;
3653
3654                 rcu_read_lock();
3655                 c_ops = rcu_dereference(bp->cnic_ops);
3656                 if (c_ops)
3657                         c_ops->cnic_handler(bp->cnic_data, NULL);
3658                 rcu_read_unlock();
3659         }
3660 #endif
3661         queue_delayed_work(bnx2x_wq, &bp->sp_task, 0);
3662
3663         return IRQ_HANDLED;
3664 }
3665
3666 /* end of slow path */
3667
3668 /* Statistics */
3669
3670 /****************************************************************************
3671 * Macros
3672 ****************************************************************************/
3673
3674 /* sum[hi:lo] += add[hi:lo] */
3675 #define ADD_64(s_hi, a_hi, s_lo, a_lo) \
3676         do { \
3677                 s_lo += a_lo; \
3678                 s_hi += a_hi + ((s_lo < a_lo) ? 1 : 0); \
3679         } while (0)
3680
3681 /* difference = minuend - subtrahend */
3682 #define DIFF_64(d_hi, m_hi, s_hi, d_lo, m_lo, s_lo) \
3683         do { \
3684                 if (m_lo < s_lo) { \
3685                         /* underflow */ \
3686                         d_hi = m_hi - s_hi; \
3687                         if (d_hi > 0) { \
3688                                 /* we can 'loan' 1 */ \
3689                                 d_hi--; \
3690                                 d_lo = m_lo + (UINT_MAX - s_lo) + 1; \
3691                         } else { \
3692                                 /* m_hi <= s_hi */ \
3693                                 d_hi = 0; \
3694                                 d_lo = 0; \
3695                         } \
3696                 } else { \
3697                         /* m_lo >= s_lo */ \
3698                         if (m_hi < s_hi) { \
3699                                 d_hi = 0; \
3700                                 d_lo = 0; \
3701                         } else { \
3702                                 /* m_hi >= s_hi */ \
3703                                 d_hi = m_hi - s_hi; \
3704                                 d_lo = m_lo - s_lo; \
3705                         } \
3706                 } \
3707         } while (0)
3708
3709 #define UPDATE_STAT64(s, t) \
3710         do { \
3711                 DIFF_64(diff.hi, new->s##_hi, pstats->mac_stx[0].t##_hi, \
3712                         diff.lo, new->s##_lo, pstats->mac_stx[0].t##_lo); \
3713                 pstats->mac_stx[0].t##_hi = new->s##_hi; \
3714                 pstats->mac_stx[0].t##_lo = new->s##_lo; \
3715                 ADD_64(pstats->mac_stx[1].t##_hi, diff.hi, \
3716                        pstats->mac_stx[1].t##_lo, diff.lo); \
3717         } while (0)
3718
3719 #define UPDATE_STAT64_NIG(s, t) \
3720         do { \
3721                 DIFF_64(diff.hi, new->s##_hi, old->s##_hi, \
3722                         diff.lo, new->s##_lo, old->s##_lo); \
3723                 ADD_64(estats->t##_hi, diff.hi, \
3724                        estats->t##_lo, diff.lo); \
3725         } while (0)
3726
3727 /* sum[hi:lo] += add */
3728 #define ADD_EXTEND_64(s_hi, s_lo, a) \
3729         do { \
3730                 s_lo += a; \
3731                 s_hi += (s_lo < a) ? 1 : 0; \
3732         } while (0)
3733
3734 #define UPDATE_EXTEND_STAT(s) \
3735         do { \
3736                 ADD_EXTEND_64(pstats->mac_stx[1].s##_hi, \
3737                               pstats->mac_stx[1].s##_lo, \
3738                               new->s); \
3739         } while (0)
3740
3741 #define UPDATE_EXTEND_TSTAT(s, t) \
3742         do { \
3743                 diff = le32_to_cpu(tclient->s) - le32_to_cpu(old_tclient->s); \
3744                 old_tclient->s = tclient->s; \
3745                 ADD_EXTEND_64(qstats->t##_hi, qstats->t##_lo, diff); \
3746         } while (0)
3747
3748 #define UPDATE_EXTEND_USTAT(s, t) \
3749         do { \
3750                 diff = le32_to_cpu(uclient->s) - le32_to_cpu(old_uclient->s); \
3751                 old_uclient->s = uclient->s; \
3752                 ADD_EXTEND_64(qstats->t##_hi, qstats->t##_lo, diff); \
3753         } while (0)
3754
3755 #define UPDATE_EXTEND_XSTAT(s, t) \
3756         do { \
3757                 diff = le32_to_cpu(xclient->s) - le32_to_cpu(old_xclient->s); \
3758                 old_xclient->s = xclient->s; \
3759                 ADD_EXTEND_64(qstats->t##_hi, qstats->t##_lo, diff); \
3760         } while (0)
3761
3762 /* minuend -= subtrahend */
3763 #define SUB_64(m_hi, s_hi, m_lo, s_lo) \
3764         do { \
3765                 DIFF_64(m_hi, m_hi, s_hi, m_lo, m_lo, s_lo); \
3766         } while (0)
3767
3768 /* minuend[hi:lo] -= subtrahend */
3769 #define SUB_EXTEND_64(m_hi, m_lo, s) \
3770         do { \
3771                 SUB_64(m_hi, 0, m_lo, s); \
3772         } while (0)
3773
3774 #define SUB_EXTEND_USTAT(s, t) \
3775         do { \
3776                 diff = le32_to_cpu(uclient->s) - le32_to_cpu(old_uclient->s); \
3777                 SUB_EXTEND_64(qstats->t##_hi, qstats->t##_lo, diff); \
3778         } while (0)
3779
3780 /*
3781  * General service functions
3782  */
3783
3784 static inline long bnx2x_hilo(u32 *hiref)
3785 {
3786         u32 lo = *(hiref + 1);
3787 #if (BITS_PER_LONG == 64)
3788         u32 hi = *hiref;
3789
3790         return HILO_U64(hi, lo);
3791 #else
3792         return lo;
3793 #endif
3794 }
3795
3796 /*
3797  * Init service functions
3798  */
3799
3800 static void bnx2x_storm_stats_post(struct bnx2x *bp)
3801 {
3802         if (!bp->stats_pending) {
3803                 struct eth_query_ramrod_data ramrod_data = {0};
3804                 int i, rc;
3805
3806                 ramrod_data.drv_counter = bp->stats_counter++;
3807                 ramrod_data.collect_port = bp->port.pmf ? 1 : 0;
3808                 for_each_queue(bp, i)
3809                         ramrod_data.ctr_id_vector |= (1 << bp->fp[i].cl_id);
3810
3811                 rc = bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_STAT_QUERY, 0,
3812                                    ((u32 *)&ramrod_data)[1],
3813                                    ((u32 *)&ramrod_data)[0], 0);
3814                 if (rc == 0) {
3815                         /* stats ramrod has it's own slot on the spq */
3816                         bp->spq_left++;
3817                         bp->stats_pending = 1;
3818                 }
3819         }
3820 }
3821
3822 static void bnx2x_hw_stats_post(struct bnx2x *bp)
3823 {
3824         struct dmae_command *dmae = &bp->stats_dmae;
3825         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
3826
3827         *stats_comp = DMAE_COMP_VAL;
3828         if (CHIP_REV_IS_SLOW(bp))
3829                 return;
3830
3831         /* loader */
3832         if (bp->executer_idx) {
3833                 int loader_idx = PMF_DMAE_C(bp);
3834
3835                 memset(dmae, 0, sizeof(struct dmae_command));
3836
3837                 dmae->opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
3838                                 DMAE_CMD_C_DST_GRC | DMAE_CMD_C_ENABLE |
3839                                 DMAE_CMD_DST_RESET |
3840 #ifdef __BIG_ENDIAN
3841                                 DMAE_CMD_ENDIANITY_B_DW_SWAP |
3842 #else
3843                                 DMAE_CMD_ENDIANITY_DW_SWAP |
3844 #endif
3845                                 (BP_PORT(bp) ? DMAE_CMD_PORT_1 :
3846                                                DMAE_CMD_PORT_0) |
3847                                 (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
3848                 dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, dmae[0]));
3849                 dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, dmae[0]));
3850                 dmae->dst_addr_lo = (DMAE_REG_CMD_MEM +
3851                                      sizeof(struct dmae_command) *
3852                                      (loader_idx + 1)) >> 2;
3853                 dmae->dst_addr_hi = 0;
3854                 dmae->len = sizeof(struct dmae_command) >> 2;
3855                 if (CHIP_IS_E1(bp))
3856                         dmae->len--;
3857                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx + 1] >> 2;
3858                 dmae->comp_addr_hi = 0;
3859                 dmae->comp_val = 1;
3860
3861                 *stats_comp = 0;
3862                 bnx2x_post_dmae(bp, dmae, loader_idx);
3863
3864         } else if (bp->func_stx) {
3865                 *stats_comp = 0;
3866                 bnx2x_post_dmae(bp, dmae, INIT_DMAE_C(bp));
3867         }
3868 }
3869
3870 static int bnx2x_stats_comp(struct bnx2x *bp)
3871 {
3872         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
3873         int cnt = 10;
3874
3875         might_sleep();
3876         while (*stats_comp != DMAE_COMP_VAL) {
3877                 if (!cnt) {
3878                         BNX2X_ERR("timeout waiting for stats finished\n");
3879                         break;
3880                 }
3881                 cnt--;
3882                 msleep(1);
3883         }
3884         return 1;
3885 }
3886
3887 /*
3888  * Statistics service functions
3889  */
3890
3891 static void bnx2x_stats_pmf_update(struct bnx2x *bp)
3892 {
3893         struct dmae_command *dmae;
3894         u32 opcode;
3895         int loader_idx = PMF_DMAE_C(bp);
3896         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
3897
3898         /* sanity */
3899         if (!IS_E1HMF(bp) || !bp->port.pmf || !bp->port.port_stx) {
3900                 BNX2X_ERR("BUG!\n");
3901                 return;
3902         }
3903
3904         bp->executer_idx = 0;
3905
3906         opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
3907                   DMAE_CMD_C_ENABLE |
3908                   DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
3909 #ifdef __BIG_ENDIAN
3910                   DMAE_CMD_ENDIANITY_B_DW_SWAP |
3911 #else
3912                   DMAE_CMD_ENDIANITY_DW_SWAP |
3913 #endif
3914                   (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
3915                   (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
3916
3917         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
3918         dmae->opcode = (opcode | DMAE_CMD_C_DST_GRC);
3919         dmae->src_addr_lo = bp->port.port_stx >> 2;
3920         dmae->src_addr_hi = 0;
3921         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, port_stats));
3922         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, port_stats));
3923         dmae->len = DMAE_LEN32_RD_MAX;
3924         dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
3925         dmae->comp_addr_hi = 0;
3926         dmae->comp_val = 1;
3927
3928         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
3929         dmae->opcode = (opcode | DMAE_CMD_C_DST_PCI);
3930         dmae->src_addr_lo = (bp->port.port_stx >> 2) + DMAE_LEN32_RD_MAX;
3931         dmae->src_addr_hi = 0;
3932         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, port_stats) +
3933                                    DMAE_LEN32_RD_MAX * 4);
3934         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, port_stats) +
3935                                    DMAE_LEN32_RD_MAX * 4);
3936         dmae->len = (sizeof(struct host_port_stats) >> 2) - DMAE_LEN32_RD_MAX;
3937         dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
3938         dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
3939         dmae->comp_val = DMAE_COMP_VAL;
3940
3941         *stats_comp = 0;
3942         bnx2x_hw_stats_post(bp);
3943         bnx2x_stats_comp(bp);
3944 }
3945
3946 static void bnx2x_port_stats_init(struct bnx2x *bp)
3947 {
3948         struct dmae_command *dmae;
3949         int port = BP_PORT(bp);
3950         int vn = BP_E1HVN(bp);
3951         u32 opcode;
3952         int loader_idx = PMF_DMAE_C(bp);
3953         u32 mac_addr;
3954         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
3955
3956         /* sanity */
3957         if (!bp->link_vars.link_up || !bp->port.pmf) {
3958                 BNX2X_ERR("BUG!\n");
3959                 return;
3960         }
3961
3962         bp->executer_idx = 0;
3963
3964         /* MCP */
3965         opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
3966                   DMAE_CMD_C_DST_GRC | DMAE_CMD_C_ENABLE |
3967                   DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
3968 #ifdef __BIG_ENDIAN
3969                   DMAE_CMD_ENDIANITY_B_DW_SWAP |
3970 #else
3971                   DMAE_CMD_ENDIANITY_DW_SWAP |
3972 #endif
3973                   (port ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
3974                   (vn << DMAE_CMD_E1HVN_SHIFT));
3975
3976         if (bp->port.port_stx) {
3977
3978                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
3979                 dmae->opcode = opcode;
3980                 dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, port_stats));
3981                 dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, port_stats));
3982                 dmae->dst_addr_lo = bp->port.port_stx >> 2;
3983                 dmae->dst_addr_hi = 0;
3984                 dmae->len = sizeof(struct host_port_stats) >> 2;
3985                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
3986                 dmae->comp_addr_hi = 0;
3987                 dmae->comp_val = 1;
3988         }
3989
3990         if (bp->func_stx) {
3991
3992                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
3993                 dmae->opcode = opcode;
3994                 dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, func_stats));
3995                 dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, func_stats));
3996                 dmae->dst_addr_lo = bp->func_stx >> 2;
3997                 dmae->dst_addr_hi = 0;
3998                 dmae->len = sizeof(struct host_func_stats) >> 2;
3999                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4000                 dmae->comp_addr_hi = 0;
4001                 dmae->comp_val = 1;
4002         }
4003
4004         /* MAC */
4005         opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
4006                   DMAE_CMD_C_DST_GRC | DMAE_CMD_C_ENABLE |
4007                   DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4008 #ifdef __BIG_ENDIAN
4009                   DMAE_CMD_ENDIANITY_B_DW_SWAP |
4010 #else
4011                   DMAE_CMD_ENDIANITY_DW_SWAP |
4012 #endif
4013                   (port ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4014                   (vn << DMAE_CMD_E1HVN_SHIFT));
4015
4016         if (bp->link_vars.mac_type == MAC_TYPE_BMAC) {
4017
4018                 mac_addr = (port ? NIG_REG_INGRESS_BMAC1_MEM :
4019                                    NIG_REG_INGRESS_BMAC0_MEM);
4020
4021                 /* BIGMAC_REGISTER_TX_STAT_GTPKT ..
4022                    BIGMAC_REGISTER_TX_STAT_GTBYT */
4023                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4024                 dmae->opcode = opcode;
4025                 dmae->src_addr_lo = (mac_addr +
4026                                      BIGMAC_REGISTER_TX_STAT_GTPKT) >> 2;
4027                 dmae->src_addr_hi = 0;
4028                 dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, mac_stats));
4029                 dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, mac_stats));
4030                 dmae->len = (8 + BIGMAC_REGISTER_TX_STAT_GTBYT -
4031                              BIGMAC_REGISTER_TX_STAT_GTPKT) >> 2;
4032                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4033                 dmae->comp_addr_hi = 0;
4034                 dmae->comp_val = 1;
4035
4036                 /* BIGMAC_REGISTER_RX_STAT_GR64 ..
4037                    BIGMAC_REGISTER_RX_STAT_GRIPJ */
4038                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4039                 dmae->opcode = opcode;
4040                 dmae->src_addr_lo = (mac_addr +
4041                                      BIGMAC_REGISTER_RX_STAT_GR64) >> 2;
4042                 dmae->src_addr_hi = 0;
4043                 dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, mac_stats) +
4044                                 offsetof(struct bmac_stats, rx_stat_gr64_lo));
4045                 dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, mac_stats) +
4046                                 offsetof(struct bmac_stats, rx_stat_gr64_lo));
4047                 dmae->len = (8 + BIGMAC_REGISTER_RX_STAT_GRIPJ -
4048                              BIGMAC_REGISTER_RX_STAT_GR64) >> 2;
4049                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4050                 dmae->comp_addr_hi = 0;
4051                 dmae->comp_val = 1;
4052
4053         } else if (bp->link_vars.mac_type == MAC_TYPE_EMAC) {
4054
4055                 mac_addr = (port ? GRCBASE_EMAC1 : GRCBASE_EMAC0);
4056
4057                 /* EMAC_REG_EMAC_RX_STAT_AC (EMAC_REG_EMAC_RX_STAT_AC_COUNT)*/
4058                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4059                 dmae->opcode = opcode;
4060                 dmae->src_addr_lo = (mac_addr +
4061                                      EMAC_REG_EMAC_RX_STAT_AC) >> 2;
4062                 dmae->src_addr_hi = 0;
4063                 dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, mac_stats));
4064                 dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, mac_stats));
4065                 dmae->len = EMAC_REG_EMAC_RX_STAT_AC_COUNT;
4066                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4067                 dmae->comp_addr_hi = 0;
4068                 dmae->comp_val = 1;
4069
4070                 /* EMAC_REG_EMAC_RX_STAT_AC_28 */
4071                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4072                 dmae->opcode = opcode;
4073                 dmae->src_addr_lo = (mac_addr +
4074                                      EMAC_REG_EMAC_RX_STAT_AC_28) >> 2;
4075                 dmae->src_addr_hi = 0;
4076                 dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, mac_stats) +
4077                      offsetof(struct emac_stats, rx_stat_falsecarriererrors));
4078                 dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, mac_stats) +
4079                      offsetof(struct emac_stats, rx_stat_falsecarriererrors));
4080                 dmae->len = 1;
4081                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4082                 dmae->comp_addr_hi = 0;
4083                 dmae->comp_val = 1;
4084
4085                 /* EMAC_REG_EMAC_TX_STAT_AC (EMAC_REG_EMAC_TX_STAT_AC_COUNT)*/
4086                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4087                 dmae->opcode = opcode;
4088                 dmae->src_addr_lo = (mac_addr +
4089                                      EMAC_REG_EMAC_TX_STAT_AC) >> 2;
4090                 dmae->src_addr_hi = 0;
4091                 dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, mac_stats) +
4092                         offsetof(struct emac_stats, tx_stat_ifhcoutoctets));
4093                 dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, mac_stats) +
4094                         offsetof(struct emac_stats, tx_stat_ifhcoutoctets));
4095                 dmae->len = EMAC_REG_EMAC_TX_STAT_AC_COUNT;
4096                 dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4097                 dmae->comp_addr_hi = 0;
4098                 dmae->comp_val = 1;
4099         }
4100
4101         /* NIG */
4102         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4103         dmae->opcode = opcode;
4104         dmae->src_addr_lo = (port ? NIG_REG_STAT1_BRB_DISCARD :
4105                                     NIG_REG_STAT0_BRB_DISCARD) >> 2;
4106         dmae->src_addr_hi = 0;
4107         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, nig_stats));
4108         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, nig_stats));
4109         dmae->len = (sizeof(struct nig_stats) - 4*sizeof(u32)) >> 2;
4110         dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4111         dmae->comp_addr_hi = 0;
4112         dmae->comp_val = 1;
4113
4114         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4115         dmae->opcode = opcode;
4116         dmae->src_addr_lo = (port ? NIG_REG_STAT1_EGRESS_MAC_PKT0 :
4117                                     NIG_REG_STAT0_EGRESS_MAC_PKT0) >> 2;
4118         dmae->src_addr_hi = 0;
4119         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, nig_stats) +
4120                         offsetof(struct nig_stats, egress_mac_pkt0_lo));
4121         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, nig_stats) +
4122                         offsetof(struct nig_stats, egress_mac_pkt0_lo));
4123         dmae->len = (2*sizeof(u32)) >> 2;
4124         dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4125         dmae->comp_addr_hi = 0;
4126         dmae->comp_val = 1;
4127
4128         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4129         dmae->opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
4130                         DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
4131                         DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4132 #ifdef __BIG_ENDIAN
4133                         DMAE_CMD_ENDIANITY_B_DW_SWAP |
4134 #else
4135                         DMAE_CMD_ENDIANITY_DW_SWAP |
4136 #endif
4137                         (port ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4138                         (vn << DMAE_CMD_E1HVN_SHIFT));
4139         dmae->src_addr_lo = (port ? NIG_REG_STAT1_EGRESS_MAC_PKT1 :
4140                                     NIG_REG_STAT0_EGRESS_MAC_PKT1) >> 2;
4141         dmae->src_addr_hi = 0;
4142         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, nig_stats) +
4143                         offsetof(struct nig_stats, egress_mac_pkt1_lo));
4144         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, nig_stats) +
4145                         offsetof(struct nig_stats, egress_mac_pkt1_lo));
4146         dmae->len = (2*sizeof(u32)) >> 2;
4147         dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4148         dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4149         dmae->comp_val = DMAE_COMP_VAL;
4150
4151         *stats_comp = 0;
4152 }
4153
4154 static void bnx2x_func_stats_init(struct bnx2x *bp)
4155 {
4156         struct dmae_command *dmae = &bp->stats_dmae;
4157         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
4158
4159         /* sanity */
4160         if (!bp->func_stx) {
4161                 BNX2X_ERR("BUG!\n");
4162                 return;
4163         }
4164
4165         bp->executer_idx = 0;
4166         memset(dmae, 0, sizeof(struct dmae_command));
4167
4168         dmae->opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
4169                         DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
4170                         DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4171 #ifdef __BIG_ENDIAN
4172                         DMAE_CMD_ENDIANITY_B_DW_SWAP |
4173 #else
4174                         DMAE_CMD_ENDIANITY_DW_SWAP |
4175 #endif
4176                         (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4177                         (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
4178         dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, func_stats));
4179         dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, func_stats));
4180         dmae->dst_addr_lo = bp->func_stx >> 2;
4181         dmae->dst_addr_hi = 0;
4182         dmae->len = sizeof(struct host_func_stats) >> 2;
4183         dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4184         dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4185         dmae->comp_val = DMAE_COMP_VAL;
4186
4187         *stats_comp = 0;
4188 }
4189
4190 static void bnx2x_stats_start(struct bnx2x *bp)
4191 {
4192         if (bp->port.pmf)
4193                 bnx2x_port_stats_init(bp);
4194
4195         else if (bp->func_stx)
4196                 bnx2x_func_stats_init(bp);
4197
4198         bnx2x_hw_stats_post(bp);
4199         bnx2x_storm_stats_post(bp);
4200 }
4201
4202 static void bnx2x_stats_pmf_start(struct bnx2x *bp)
4203 {
4204         bnx2x_stats_comp(bp);
4205         bnx2x_stats_pmf_update(bp);
4206         bnx2x_stats_start(bp);
4207 }
4208
4209 static void bnx2x_stats_restart(struct bnx2x *bp)
4210 {
4211         bnx2x_stats_comp(bp);
4212         bnx2x_stats_start(bp);
4213 }
4214
4215 static void bnx2x_bmac_stats_update(struct bnx2x *bp)
4216 {
4217         struct bmac_stats *new = bnx2x_sp(bp, mac_stats.bmac_stats);
4218         struct host_port_stats *pstats = bnx2x_sp(bp, port_stats);
4219         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4220         struct {
4221                 u32 lo;
4222                 u32 hi;
4223         } diff;
4224
4225         UPDATE_STAT64(rx_stat_grerb, rx_stat_ifhcinbadoctets);
4226         UPDATE_STAT64(rx_stat_grfcs, rx_stat_dot3statsfcserrors);
4227         UPDATE_STAT64(rx_stat_grund, rx_stat_etherstatsundersizepkts);
4228         UPDATE_STAT64(rx_stat_grovr, rx_stat_dot3statsframestoolong);
4229         UPDATE_STAT64(rx_stat_grfrg, rx_stat_etherstatsfragments);
4230         UPDATE_STAT64(rx_stat_grjbr, rx_stat_etherstatsjabbers);
4231         UPDATE_STAT64(rx_stat_grxcf, rx_stat_maccontrolframesreceived);
4232         UPDATE_STAT64(rx_stat_grxpf, rx_stat_xoffstateentered);
4233         UPDATE_STAT64(rx_stat_grxpf, rx_stat_bmac_xpf);
4234         UPDATE_STAT64(tx_stat_gtxpf, tx_stat_outxoffsent);
4235         UPDATE_STAT64(tx_stat_gtxpf, tx_stat_flowcontroldone);
4236         UPDATE_STAT64(tx_stat_gt64, tx_stat_etherstatspkts64octets);
4237         UPDATE_STAT64(tx_stat_gt127,
4238                                 tx_stat_etherstatspkts65octetsto127octets);
4239         UPDATE_STAT64(tx_stat_gt255,
4240                                 tx_stat_etherstatspkts128octetsto255octets);
4241         UPDATE_STAT64(tx_stat_gt511,
4242                                 tx_stat_etherstatspkts256octetsto511octets);
4243         UPDATE_STAT64(tx_stat_gt1023,
4244                                 tx_stat_etherstatspkts512octetsto1023octets);
4245         UPDATE_STAT64(tx_stat_gt1518,
4246                                 tx_stat_etherstatspkts1024octetsto1522octets);
4247         UPDATE_STAT64(tx_stat_gt2047, tx_stat_bmac_2047);
4248         UPDATE_STAT64(tx_stat_gt4095, tx_stat_bmac_4095);
4249         UPDATE_STAT64(tx_stat_gt9216, tx_stat_bmac_9216);
4250         UPDATE_STAT64(tx_stat_gt16383, tx_stat_bmac_16383);
4251         UPDATE_STAT64(tx_stat_gterr,
4252                                 tx_stat_dot3statsinternalmactransmiterrors);
4253         UPDATE_STAT64(tx_stat_gtufl, tx_stat_bmac_ufl);
4254
4255         estats->pause_frames_received_hi =
4256                                 pstats->mac_stx[1].rx_stat_bmac_xpf_hi;
4257         estats->pause_frames_received_lo =
4258                                 pstats->mac_stx[1].rx_stat_bmac_xpf_lo;
4259
4260         estats->pause_frames_sent_hi =
4261                                 pstats->mac_stx[1].tx_stat_outxoffsent_hi;
4262         estats->pause_frames_sent_lo =
4263                                 pstats->mac_stx[1].tx_stat_outxoffsent_lo;
4264 }
4265
4266 static void bnx2x_emac_stats_update(struct bnx2x *bp)
4267 {
4268         struct emac_stats *new = bnx2x_sp(bp, mac_stats.emac_stats);
4269         struct host_port_stats *pstats = bnx2x_sp(bp, port_stats);
4270         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4271
4272         UPDATE_EXTEND_STAT(rx_stat_ifhcinbadoctets);
4273         UPDATE_EXTEND_STAT(tx_stat_ifhcoutbadoctets);
4274         UPDATE_EXTEND_STAT(rx_stat_dot3statsfcserrors);
4275         UPDATE_EXTEND_STAT(rx_stat_dot3statsalignmenterrors);
4276         UPDATE_EXTEND_STAT(rx_stat_dot3statscarriersenseerrors);
4277         UPDATE_EXTEND_STAT(rx_stat_falsecarriererrors);
4278         UPDATE_EXTEND_STAT(rx_stat_etherstatsundersizepkts);
4279         UPDATE_EXTEND_STAT(rx_stat_dot3statsframestoolong);
4280         UPDATE_EXTEND_STAT(rx_stat_etherstatsfragments);
4281         UPDATE_EXTEND_STAT(rx_stat_etherstatsjabbers);
4282         UPDATE_EXTEND_STAT(rx_stat_maccontrolframesreceived);
4283         UPDATE_EXTEND_STAT(rx_stat_xoffstateentered);
4284         UPDATE_EXTEND_STAT(rx_stat_xonpauseframesreceived);
4285         UPDATE_EXTEND_STAT(rx_stat_xoffpauseframesreceived);
4286         UPDATE_EXTEND_STAT(tx_stat_outxonsent);
4287         UPDATE_EXTEND_STAT(tx_stat_outxoffsent);
4288         UPDATE_EXTEND_STAT(tx_stat_flowcontroldone);
4289         UPDATE_EXTEND_STAT(tx_stat_etherstatscollisions);
4290         UPDATE_EXTEND_STAT(tx_stat_dot3statssinglecollisionframes);
4291         UPDATE_EXTEND_STAT(tx_stat_dot3statsmultiplecollisionframes);
4292         UPDATE_EXTEND_STAT(tx_stat_dot3statsdeferredtransmissions);
4293         UPDATE_EXTEND_STAT(tx_stat_dot3statsexcessivecollisions);
4294         UPDATE_EXTEND_STAT(tx_stat_dot3statslatecollisions);
4295         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts64octets);
4296         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts65octetsto127octets);
4297         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts128octetsto255octets);
4298         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts256octetsto511octets);
4299         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts512octetsto1023octets);
4300         UPDATE_EXTEND_STAT(tx_stat_etherstatspkts1024octetsto1522octets);
4301         UPDATE_EXTEND_STAT(tx_stat_etherstatspktsover1522octets);
4302         UPDATE_EXTEND_STAT(tx_stat_dot3statsinternalmactransmiterrors);
4303
4304         estats->pause_frames_received_hi =
4305                         pstats->mac_stx[1].rx_stat_xonpauseframesreceived_hi;
4306         estats->pause_frames_received_lo =
4307                         pstats->mac_stx[1].rx_stat_xonpauseframesreceived_lo;
4308         ADD_64(estats->pause_frames_received_hi,
4309                pstats->mac_stx[1].rx_stat_xoffpauseframesreceived_hi,
4310                estats->pause_frames_received_lo,
4311                pstats->mac_stx[1].rx_stat_xoffpauseframesreceived_lo);
4312
4313         estats->pause_frames_sent_hi =
4314                         pstats->mac_stx[1].tx_stat_outxonsent_hi;
4315         estats->pause_frames_sent_lo =
4316                         pstats->mac_stx[1].tx_stat_outxonsent_lo;
4317         ADD_64(estats->pause_frames_sent_hi,
4318                pstats->mac_stx[1].tx_stat_outxoffsent_hi,
4319                estats->pause_frames_sent_lo,
4320                pstats->mac_stx[1].tx_stat_outxoffsent_lo);
4321 }
4322
4323 static int bnx2x_hw_stats_update(struct bnx2x *bp)
4324 {
4325         struct nig_stats *new = bnx2x_sp(bp, nig_stats);
4326         struct nig_stats *old = &(bp->port.old_nig_stats);
4327         struct host_port_stats *pstats = bnx2x_sp(bp, port_stats);
4328         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4329         struct {
4330                 u32 lo;
4331                 u32 hi;
4332         } diff;
4333
4334         if (bp->link_vars.mac_type == MAC_TYPE_BMAC)
4335                 bnx2x_bmac_stats_update(bp);
4336
4337         else if (bp->link_vars.mac_type == MAC_TYPE_EMAC)
4338                 bnx2x_emac_stats_update(bp);
4339
4340         else { /* unreached */
4341                 BNX2X_ERR("stats updated by DMAE but no MAC active\n");
4342                 return -1;
4343         }
4344
4345         ADD_EXTEND_64(pstats->brb_drop_hi, pstats->brb_drop_lo,
4346                       new->brb_discard - old->brb_discard);
4347         ADD_EXTEND_64(estats->brb_truncate_hi, estats->brb_truncate_lo,
4348                       new->brb_truncate - old->brb_truncate);
4349
4350         UPDATE_STAT64_NIG(egress_mac_pkt0,
4351                                         etherstatspkts1024octetsto1522octets);
4352         UPDATE_STAT64_NIG(egress_mac_pkt1, etherstatspktsover1522octets);
4353
4354         memcpy(old, new, sizeof(struct nig_stats));
4355
4356         memcpy(&(estats->rx_stat_ifhcinbadoctets_hi), &(pstats->mac_stx[1]),
4357                sizeof(struct mac_stx));
4358         estats->brb_drop_hi = pstats->brb_drop_hi;
4359         estats->brb_drop_lo = pstats->brb_drop_lo;
4360
4361         pstats->host_port_stats_start = ++pstats->host_port_stats_end;
4362
4363         if (!BP_NOMCP(bp)) {
4364                 u32 nig_timer_max =
4365                         SHMEM_RD(bp, port_mb[BP_PORT(bp)].stat_nig_timer);
4366                 if (nig_timer_max != estats->nig_timer_max) {
4367                         estats->nig_timer_max = nig_timer_max;
4368                         BNX2X_ERR("NIG timer max (%u)\n",
4369                                   estats->nig_timer_max);
4370                 }
4371         }
4372
4373         return 0;
4374 }
4375
4376 static int bnx2x_storm_stats_update(struct bnx2x *bp)
4377 {
4378         struct eth_stats_query *stats = bnx2x_sp(bp, fw_stats);
4379         struct tstorm_per_port_stats *tport =
4380                                         &stats->tstorm_common.port_statistics;
4381         struct host_func_stats *fstats = bnx2x_sp(bp, func_stats);
4382         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4383         int i;
4384
4385         memcpy(&(fstats->total_bytes_received_hi),
4386                &(bnx2x_sp(bp, func_stats_base)->total_bytes_received_hi),
4387                sizeof(struct host_func_stats) - 2*sizeof(u32));
4388         estats->error_bytes_received_hi = 0;
4389         estats->error_bytes_received_lo = 0;
4390         estats->etherstatsoverrsizepkts_hi = 0;
4391         estats->etherstatsoverrsizepkts_lo = 0;
4392         estats->no_buff_discard_hi = 0;
4393         estats->no_buff_discard_lo = 0;
4394
4395         for_each_queue(bp, i) {
4396                 struct bnx2x_fastpath *fp = &bp->fp[i];
4397                 int cl_id = fp->cl_id;
4398                 struct tstorm_per_client_stats *tclient =
4399                                 &stats->tstorm_common.client_statistics[cl_id];
4400                 struct tstorm_per_client_stats *old_tclient = &fp->old_tclient;
4401                 struct ustorm_per_client_stats *uclient =
4402                                 &stats->ustorm_common.client_statistics[cl_id];
4403                 struct ustorm_per_client_stats *old_uclient = &fp->old_uclient;
4404                 struct xstorm_per_client_stats *xclient =
4405                                 &stats->xstorm_common.client_statistics[cl_id];
4406                 struct xstorm_per_client_stats *old_xclient = &fp->old_xclient;
4407                 struct bnx2x_eth_q_stats *qstats = &fp->eth_q_stats;
4408                 u32 diff;
4409
4410                 /* are storm stats valid? */
4411                 if ((u16)(le16_to_cpu(xclient->stats_counter) + 1) !=
4412                                                         bp->stats_counter) {
4413                         DP(BNX2X_MSG_STATS, "[%d] stats not updated by xstorm"
4414                            "  xstorm counter (0x%x) != stats_counter (0x%x)\n",
4415                            i, xclient->stats_counter, bp->stats_counter);
4416                         return -1;
4417                 }
4418                 if ((u16)(le16_to_cpu(tclient->stats_counter) + 1) !=
4419                                                         bp->stats_counter) {
4420                         DP(BNX2X_MSG_STATS, "[%d] stats not updated by tstorm"
4421                            "  tstorm counter (0x%x) != stats_counter (0x%x)\n",
4422                            i, tclient->stats_counter, bp->stats_counter);
4423                         return -2;
4424                 }
4425                 if ((u16)(le16_to_cpu(uclient->stats_counter) + 1) !=
4426                                                         bp->stats_counter) {
4427                         DP(BNX2X_MSG_STATS, "[%d] stats not updated by ustorm"
4428                            "  ustorm counter (0x%x) != stats_counter (0x%x)\n",
4429                            i, uclient->stats_counter, bp->stats_counter);
4430                         return -4;
4431                 }
4432
4433                 qstats->total_bytes_received_hi =
4434                         le32_to_cpu(tclient->rcv_broadcast_bytes.hi);
4435                 qstats->total_bytes_received_lo =
4436                         le32_to_cpu(tclient->rcv_broadcast_bytes.lo);
4437
4438                 ADD_64(qstats->total_bytes_received_hi,
4439                        le32_to_cpu(tclient->rcv_multicast_bytes.hi),
4440                        qstats->total_bytes_received_lo,
4441                        le32_to_cpu(tclient->rcv_multicast_bytes.lo));
4442
4443                 ADD_64(qstats->total_bytes_received_hi,
4444                        le32_to_cpu(tclient->rcv_unicast_bytes.hi),
4445                        qstats->total_bytes_received_lo,
4446                        le32_to_cpu(tclient->rcv_unicast_bytes.lo));
4447
4448                 SUB_64(qstats->total_bytes_received_hi,
4449                        le32_to_cpu(uclient->bcast_no_buff_bytes.hi),
4450                        qstats->total_bytes_received_lo,
4451                        le32_to_cpu(uclient->bcast_no_buff_bytes.lo));
4452
4453                 SUB_64(qstats->total_bytes_received_hi,
4454                        le32_to_cpu(uclient->mcast_no_buff_bytes.hi),
4455                        qstats->total_bytes_received_lo,
4456                        le32_to_cpu(uclient->mcast_no_buff_bytes.lo));
4457
4458                 SUB_64(qstats->total_bytes_received_hi,
4459                        le32_to_cpu(uclient->ucast_no_buff_bytes.hi),
4460                        qstats->total_bytes_received_lo,
4461                        le32_to_cpu(uclient->ucast_no_buff_bytes.lo));
4462
4463                 qstats->valid_bytes_received_hi =
4464                                         qstats->total_bytes_received_hi;
4465                 qstats->valid_bytes_received_lo =
4466                                         qstats->total_bytes_received_lo;
4467
4468                 qstats->error_bytes_received_hi =
4469                                 le32_to_cpu(tclient->rcv_error_bytes.hi);
4470                 qstats->error_bytes_received_lo =
4471                                 le32_to_cpu(tclient->rcv_error_bytes.lo);
4472
4473                 ADD_64(qstats->total_bytes_received_hi,
4474                        qstats->error_bytes_received_hi,
4475                        qstats->total_bytes_received_lo,
4476                        qstats->error_bytes_received_lo);
4477
4478                 UPDATE_EXTEND_TSTAT(rcv_unicast_pkts,
4479                                         total_unicast_packets_received);
4480                 UPDATE_EXTEND_TSTAT(rcv_multicast_pkts,
4481                                         total_multicast_packets_received);
4482                 UPDATE_EXTEND_TSTAT(rcv_broadcast_pkts,
4483                                         total_broadcast_packets_received);
4484                 UPDATE_EXTEND_TSTAT(packets_too_big_discard,
4485                                         etherstatsoverrsizepkts);
4486                 UPDATE_EXTEND_TSTAT(no_buff_discard, no_buff_discard);
4487
4488                 SUB_EXTEND_USTAT(ucast_no_buff_pkts,
4489                                         total_unicast_packets_received);
4490                 SUB_EXTEND_USTAT(mcast_no_buff_pkts,
4491                                         total_multicast_packets_received);
4492                 SUB_EXTEND_USTAT(bcast_no_buff_pkts,
4493                                         total_broadcast_packets_received);
4494                 UPDATE_EXTEND_USTAT(ucast_no_buff_pkts, no_buff_discard);
4495                 UPDATE_EXTEND_USTAT(mcast_no_buff_pkts, no_buff_discard);
4496                 UPDATE_EXTEND_USTAT(bcast_no_buff_pkts, no_buff_discard);
4497
4498                 qstats->total_bytes_transmitted_hi =
4499                                 le32_to_cpu(xclient->unicast_bytes_sent.hi);
4500                 qstats->total_bytes_transmitted_lo =
4501                                 le32_to_cpu(xclient->unicast_bytes_sent.lo);
4502
4503                 ADD_64(qstats->total_bytes_transmitted_hi,
4504                        le32_to_cpu(xclient->multicast_bytes_sent.hi),
4505                        qstats->total_bytes_transmitted_lo,
4506                        le32_to_cpu(xclient->multicast_bytes_sent.lo));
4507
4508                 ADD_64(qstats->total_bytes_transmitted_hi,
4509                        le32_to_cpu(xclient->broadcast_bytes_sent.hi),
4510                        qstats->total_bytes_transmitted_lo,
4511                        le32_to_cpu(xclient->broadcast_bytes_sent.lo));
4512
4513                 UPDATE_EXTEND_XSTAT(unicast_pkts_sent,
4514                                         total_unicast_packets_transmitted);
4515                 UPDATE_EXTEND_XSTAT(multicast_pkts_sent,
4516                                         total_multicast_packets_transmitted);
4517                 UPDATE_EXTEND_XSTAT(broadcast_pkts_sent,
4518                                         total_broadcast_packets_transmitted);
4519
4520                 old_tclient->checksum_discard = tclient->checksum_discard;
4521                 old_tclient->ttl0_discard = tclient->ttl0_discard;
4522
4523                 ADD_64(fstats->total_bytes_received_hi,
4524                        qstats->total_bytes_received_hi,
4525                        fstats->total_bytes_received_lo,
4526                        qstats->total_bytes_received_lo);
4527                 ADD_64(fstats->total_bytes_transmitted_hi,
4528                        qstats->total_bytes_transmitted_hi,
4529                        fstats->total_bytes_transmitted_lo,
4530                        qstats->total_bytes_transmitted_lo);
4531                 ADD_64(fstats->total_unicast_packets_received_hi,
4532                        qstats->total_unicast_packets_received_hi,
4533                        fstats->total_unicast_packets_received_lo,
4534                        qstats->total_unicast_packets_received_lo);
4535                 ADD_64(fstats->total_multicast_packets_received_hi,
4536                        qstats->total_multicast_packets_received_hi,
4537                        fstats->total_multicast_packets_received_lo,
4538                        qstats->total_multicast_packets_received_lo);
4539                 ADD_64(fstats->total_broadcast_packets_received_hi,
4540                        qstats->total_broadcast_packets_received_hi,
4541                        fstats->total_broadcast_packets_received_lo,
4542                        qstats->total_broadcast_packets_received_lo);
4543                 ADD_64(fstats->total_unicast_packets_transmitted_hi,
4544                        qstats->total_unicast_packets_transmitted_hi,
4545                        fstats->total_unicast_packets_transmitted_lo,
4546                        qstats->total_unicast_packets_transmitted_lo);
4547                 ADD_64(fstats->total_multicast_packets_transmitted_hi,
4548                        qstats->total_multicast_packets_transmitted_hi,
4549                        fstats->total_multicast_packets_transmitted_lo,
4550                        qstats->total_multicast_packets_transmitted_lo);
4551                 ADD_64(fstats->total_broadcast_packets_transmitted_hi,
4552                        qstats->total_broadcast_packets_transmitted_hi,
4553                        fstats->total_broadcast_packets_transmitted_lo,
4554                        qstats->total_broadcast_packets_transmitted_lo);
4555                 ADD_64(fstats->valid_bytes_received_hi,
4556                        qstats->valid_bytes_received_hi,
4557                        fstats->valid_bytes_received_lo,
4558                        qstats->valid_bytes_received_lo);
4559
4560                 ADD_64(estats->error_bytes_received_hi,
4561                        qstats->error_bytes_received_hi,
4562                        estats->error_bytes_received_lo,
4563                        qstats->error_bytes_received_lo);
4564                 ADD_64(estats->etherstatsoverrsizepkts_hi,
4565                        qstats->etherstatsoverrsizepkts_hi,
4566                        estats->etherstatsoverrsizepkts_lo,
4567                        qstats->etherstatsoverrsizepkts_lo);
4568                 ADD_64(estats->no_buff_discard_hi, qstats->no_buff_discard_hi,
4569                        estats->no_buff_discard_lo, qstats->no_buff_discard_lo);
4570         }
4571
4572         ADD_64(fstats->total_bytes_received_hi,
4573                estats->rx_stat_ifhcinbadoctets_hi,
4574                fstats->total_bytes_received_lo,
4575                estats->rx_stat_ifhcinbadoctets_lo);
4576
4577         memcpy(estats, &(fstats->total_bytes_received_hi),
4578                sizeof(struct host_func_stats) - 2*sizeof(u32));
4579
4580         ADD_64(estats->etherstatsoverrsizepkts_hi,
4581                estats->rx_stat_dot3statsframestoolong_hi,
4582                estats->etherstatsoverrsizepkts_lo,
4583                estats->rx_stat_dot3statsframestoolong_lo);
4584         ADD_64(estats->error_bytes_received_hi,
4585                estats->rx_stat_ifhcinbadoctets_hi,
4586                estats->error_bytes_received_lo,
4587                estats->rx_stat_ifhcinbadoctets_lo);
4588
4589         if (bp->port.pmf) {
4590                 estats->mac_filter_discard =
4591                                 le32_to_cpu(tport->mac_filter_discard);
4592                 estats->xxoverflow_discard =
4593                                 le32_to_cpu(tport->xxoverflow_discard);
4594                 estats->brb_truncate_discard =
4595                                 le32_to_cpu(tport->brb_truncate_discard);
4596                 estats->mac_discard = le32_to_cpu(tport->mac_discard);
4597         }
4598
4599         fstats->host_func_stats_start = ++fstats->host_func_stats_end;
4600
4601         bp->stats_pending = 0;
4602
4603         return 0;
4604 }
4605
4606 static void bnx2x_net_stats_update(struct bnx2x *bp)
4607 {
4608         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4609         struct net_device_stats *nstats = &bp->dev->stats;
4610         int i;
4611
4612         nstats->rx_packets =
4613                 bnx2x_hilo(&estats->total_unicast_packets_received_hi) +
4614                 bnx2x_hilo(&estats->total_multicast_packets_received_hi) +
4615                 bnx2x_hilo(&estats->total_broadcast_packets_received_hi);
4616
4617         nstats->tx_packets =
4618                 bnx2x_hilo(&estats->total_unicast_packets_transmitted_hi) +
4619                 bnx2x_hilo(&estats->total_multicast_packets_transmitted_hi) +
4620                 bnx2x_hilo(&estats->total_broadcast_packets_transmitted_hi);
4621
4622         nstats->rx_bytes = bnx2x_hilo(&estats->total_bytes_received_hi);
4623
4624         nstats->tx_bytes = bnx2x_hilo(&estats->total_bytes_transmitted_hi);
4625
4626         nstats->rx_dropped = estats->mac_discard;
4627         for_each_queue(bp, i)
4628                 nstats->rx_dropped +=
4629                         le32_to_cpu(bp->fp[i].old_tclient.checksum_discard);
4630
4631         nstats->tx_dropped = 0;
4632
4633         nstats->multicast =
4634                 bnx2x_hilo(&estats->total_multicast_packets_received_hi);
4635
4636         nstats->collisions =
4637                 bnx2x_hilo(&estats->tx_stat_etherstatscollisions_hi);
4638
4639         nstats->rx_length_errors =
4640                 bnx2x_hilo(&estats->rx_stat_etherstatsundersizepkts_hi) +
4641                 bnx2x_hilo(&estats->etherstatsoverrsizepkts_hi);
4642         nstats->rx_over_errors = bnx2x_hilo(&estats->brb_drop_hi) +
4643                                  bnx2x_hilo(&estats->brb_truncate_hi);
4644         nstats->rx_crc_errors =
4645                 bnx2x_hilo(&estats->rx_stat_dot3statsfcserrors_hi);
4646         nstats->rx_frame_errors =
4647                 bnx2x_hilo(&estats->rx_stat_dot3statsalignmenterrors_hi);
4648         nstats->rx_fifo_errors = bnx2x_hilo(&estats->no_buff_discard_hi);
4649         nstats->rx_missed_errors = estats->xxoverflow_discard;
4650
4651         nstats->rx_errors = nstats->rx_length_errors +
4652                             nstats->rx_over_errors +
4653                             nstats->rx_crc_errors +
4654                             nstats->rx_frame_errors +
4655                             nstats->rx_fifo_errors +
4656                             nstats->rx_missed_errors;
4657
4658         nstats->tx_aborted_errors =
4659                 bnx2x_hilo(&estats->tx_stat_dot3statslatecollisions_hi) +
4660                 bnx2x_hilo(&estats->tx_stat_dot3statsexcessivecollisions_hi);
4661         nstats->tx_carrier_errors =
4662                 bnx2x_hilo(&estats->rx_stat_dot3statscarriersenseerrors_hi);
4663         nstats->tx_fifo_errors = 0;
4664         nstats->tx_heartbeat_errors = 0;
4665         nstats->tx_window_errors = 0;
4666
4667         nstats->tx_errors = nstats->tx_aborted_errors +
4668                             nstats->tx_carrier_errors +
4669             bnx2x_hilo(&estats->tx_stat_dot3statsinternalmactransmiterrors_hi);
4670 }
4671
4672 static void bnx2x_drv_stats_update(struct bnx2x *bp)
4673 {
4674         struct bnx2x_eth_stats *estats = &bp->eth_stats;
4675         int i;
4676
4677         estats->driver_xoff = 0;
4678         estats->rx_err_discard_pkt = 0;
4679         estats->rx_skb_alloc_failed = 0;
4680         estats->hw_csum_err = 0;
4681         for_each_queue(bp, i) {
4682                 struct bnx2x_eth_q_stats *qstats = &bp->fp[i].eth_q_stats;
4683
4684                 estats->driver_xoff += qstats->driver_xoff;
4685                 estats->rx_err_discard_pkt += qstats->rx_err_discard_pkt;
4686                 estats->rx_skb_alloc_failed += qstats->rx_skb_alloc_failed;
4687                 estats->hw_csum_err += qstats->hw_csum_err;
4688         }
4689 }
4690
4691 static void bnx2x_stats_update(struct bnx2x *bp)
4692 {
4693         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
4694
4695         if (*stats_comp != DMAE_COMP_VAL)
4696                 return;
4697
4698         if (bp->port.pmf)
4699                 bnx2x_hw_stats_update(bp);
4700
4701         if (bnx2x_storm_stats_update(bp) && (bp->stats_pending++ == 3)) {
4702                 BNX2X_ERR("storm stats were not updated for 3 times\n");
4703                 bnx2x_panic();
4704                 return;
4705         }
4706
4707         bnx2x_net_stats_update(bp);
4708         bnx2x_drv_stats_update(bp);
4709
4710         if (netif_msg_timer(bp)) {
4711                 struct bnx2x_eth_stats *estats = &bp->eth_stats;
4712                 int i;
4713
4714                 printk(KERN_DEBUG "%s: brb drops %u  brb truncate %u\n",
4715                        bp->dev->name,
4716                        estats->brb_drop_lo, estats->brb_truncate_lo);
4717
4718                 for_each_queue(bp, i) {
4719                         struct bnx2x_fastpath *fp = &bp->fp[i];
4720                         struct bnx2x_eth_q_stats *qstats = &fp->eth_q_stats;
4721
4722                         printk(KERN_DEBUG "%s: rx usage(%4u)  *rx_cons_sb(%u)"
4723                                           "  rx pkt(%lu)  rx calls(%lu %lu)\n",
4724                                fp->name, (le16_to_cpu(*fp->rx_cons_sb) -
4725                                fp->rx_comp_cons),
4726                                le16_to_cpu(*fp->rx_cons_sb),
4727                                bnx2x_hilo(&qstats->
4728                                           total_unicast_packets_received_hi),
4729                                fp->rx_calls, fp->rx_pkt);
4730                 }
4731
4732                 for_each_queue(bp, i) {
4733                         struct bnx2x_fastpath *fp = &bp->fp[i];
4734                         struct bnx2x_eth_q_stats *qstats = &fp->eth_q_stats;
4735                         struct netdev_queue *txq =
4736                                 netdev_get_tx_queue(bp->dev, i);
4737
4738                         printk(KERN_DEBUG "%s: tx avail(%4u)  *tx_cons_sb(%u)"
4739                                           "  tx pkt(%lu) tx calls (%lu)"
4740                                           "  %s (Xoff events %u)\n",
4741                                fp->name, bnx2x_tx_avail(fp),
4742                                le16_to_cpu(*fp->tx_cons_sb),
4743                                bnx2x_hilo(&qstats->
4744                                           total_unicast_packets_transmitted_hi),
4745                                fp->tx_pkt,
4746                                (netif_tx_queue_stopped(txq) ? "Xoff" : "Xon"),
4747                                qstats->driver_xoff);
4748                 }
4749         }
4750
4751         bnx2x_hw_stats_post(bp);
4752         bnx2x_storm_stats_post(bp);
4753 }
4754
4755 static void bnx2x_port_stats_stop(struct bnx2x *bp)
4756 {
4757         struct dmae_command *dmae;
4758         u32 opcode;
4759         int loader_idx = PMF_DMAE_C(bp);
4760         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
4761
4762         bp->executer_idx = 0;
4763
4764         opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
4765                   DMAE_CMD_C_ENABLE |
4766                   DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4767 #ifdef __BIG_ENDIAN
4768                   DMAE_CMD_ENDIANITY_B_DW_SWAP |
4769 #else
4770                   DMAE_CMD_ENDIANITY_DW_SWAP |
4771 #endif
4772                   (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4773                   (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
4774
4775         if (bp->port.port_stx) {
4776
4777                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4778                 if (bp->func_stx)
4779                         dmae->opcode = (opcode | DMAE_CMD_C_DST_GRC);
4780                 else
4781                         dmae->opcode = (opcode | DMAE_CMD_C_DST_PCI);
4782                 dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, port_stats));
4783                 dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, port_stats));
4784                 dmae->dst_addr_lo = bp->port.port_stx >> 2;
4785                 dmae->dst_addr_hi = 0;
4786                 dmae->len = sizeof(struct host_port_stats) >> 2;
4787                 if (bp->func_stx) {
4788                         dmae->comp_addr_lo = dmae_reg_go_c[loader_idx] >> 2;
4789                         dmae->comp_addr_hi = 0;
4790                         dmae->comp_val = 1;
4791                 } else {
4792                         dmae->comp_addr_lo =
4793                                 U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4794                         dmae->comp_addr_hi =
4795                                 U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4796                         dmae->comp_val = DMAE_COMP_VAL;
4797
4798                         *stats_comp = 0;
4799                 }
4800         }
4801
4802         if (bp->func_stx) {
4803
4804                 dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4805                 dmae->opcode = (opcode | DMAE_CMD_C_DST_PCI);
4806                 dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, func_stats));
4807                 dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, func_stats));
4808                 dmae->dst_addr_lo = bp->func_stx >> 2;
4809                 dmae->dst_addr_hi = 0;
4810                 dmae->len = sizeof(struct host_func_stats) >> 2;
4811                 dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4812                 dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4813                 dmae->comp_val = DMAE_COMP_VAL;
4814
4815                 *stats_comp = 0;
4816         }
4817 }
4818
4819 static void bnx2x_stats_stop(struct bnx2x *bp)
4820 {
4821         int update = 0;
4822
4823         bnx2x_stats_comp(bp);
4824
4825         if (bp->port.pmf)
4826                 update = (bnx2x_hw_stats_update(bp) == 0);
4827
4828         update |= (bnx2x_storm_stats_update(bp) == 0);
4829
4830         if (update) {
4831                 bnx2x_net_stats_update(bp);
4832
4833                 if (bp->port.pmf)
4834                         bnx2x_port_stats_stop(bp);
4835
4836                 bnx2x_hw_stats_post(bp);
4837                 bnx2x_stats_comp(bp);
4838         }
4839 }
4840
4841 static void bnx2x_stats_do_nothing(struct bnx2x *bp)
4842 {
4843 }
4844
4845 static const struct {
4846         void (*action)(struct bnx2x *bp);
4847         enum bnx2x_stats_state next_state;
4848 } bnx2x_stats_stm[STATS_STATE_MAX][STATS_EVENT_MAX] = {
4849 /* state        event   */
4850 {
4851 /* DISABLED     PMF     */ {bnx2x_stats_pmf_update, STATS_STATE_DISABLED},
4852 /*              LINK_UP */ {bnx2x_stats_start,      STATS_STATE_ENABLED},
4853 /*              UPDATE  */ {bnx2x_stats_do_nothing, STATS_STATE_DISABLED},
4854 /*              STOP    */ {bnx2x_stats_do_nothing, STATS_STATE_DISABLED}
4855 },
4856 {
4857 /* ENABLED      PMF     */ {bnx2x_stats_pmf_start,  STATS_STATE_ENABLED},
4858 /*              LINK_UP */ {bnx2x_stats_restart,    STATS_STATE_ENABLED},
4859 /*              UPDATE  */ {bnx2x_stats_update,     STATS_STATE_ENABLED},
4860 /*              STOP    */ {bnx2x_stats_stop,       STATS_STATE_DISABLED}
4861 }
4862 };
4863
4864 static void bnx2x_stats_handle(struct bnx2x *bp, enum bnx2x_stats_event event)
4865 {
4866         enum bnx2x_stats_state state = bp->stats_state;
4867
4868         if (unlikely(bp->panic))
4869                 return;
4870
4871         bnx2x_stats_stm[state][event].action(bp);
4872         bp->stats_state = bnx2x_stats_stm[state][event].next_state;
4873
4874         /* Make sure the state has been "changed" */
4875         smp_wmb();
4876
4877         if ((event != STATS_EVENT_UPDATE) || netif_msg_timer(bp))
4878                 DP(BNX2X_MSG_STATS, "state %d -> event %d -> state %d\n",
4879                    state, event, bp->stats_state);
4880 }
4881
4882 static void bnx2x_port_stats_base_init(struct bnx2x *bp)
4883 {
4884         struct dmae_command *dmae;
4885         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
4886
4887         /* sanity */
4888         if (!bp->port.pmf || !bp->port.port_stx) {
4889                 BNX2X_ERR("BUG!\n");
4890                 return;
4891         }
4892
4893         bp->executer_idx = 0;
4894
4895         dmae = bnx2x_sp(bp, dmae[bp->executer_idx++]);
4896         dmae->opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
4897                         DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
4898                         DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4899 #ifdef __BIG_ENDIAN
4900                         DMAE_CMD_ENDIANITY_B_DW_SWAP |
4901 #else
4902                         DMAE_CMD_ENDIANITY_DW_SWAP |
4903 #endif
4904                         (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4905                         (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
4906         dmae->src_addr_lo = U64_LO(bnx2x_sp_mapping(bp, port_stats));
4907         dmae->src_addr_hi = U64_HI(bnx2x_sp_mapping(bp, port_stats));
4908         dmae->dst_addr_lo = bp->port.port_stx >> 2;
4909         dmae->dst_addr_hi = 0;
4910         dmae->len = sizeof(struct host_port_stats) >> 2;
4911         dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4912         dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4913         dmae->comp_val = DMAE_COMP_VAL;
4914
4915         *stats_comp = 0;
4916         bnx2x_hw_stats_post(bp);
4917         bnx2x_stats_comp(bp);
4918 }
4919
4920 static void bnx2x_func_stats_base_init(struct bnx2x *bp)
4921 {
4922         int vn, vn_max = IS_E1HMF(bp) ? E1HVN_MAX : E1VN_MAX;
4923         int port = BP_PORT(bp);
4924         int func;
4925         u32 func_stx;
4926
4927         /* sanity */
4928         if (!bp->port.pmf || !bp->func_stx) {
4929                 BNX2X_ERR("BUG!\n");
4930                 return;
4931         }
4932
4933         /* save our func_stx */
4934         func_stx = bp->func_stx;
4935
4936         for (vn = VN_0; vn < vn_max; vn++) {
4937                 func = 2*vn + port;
4938
4939                 bp->func_stx = SHMEM_RD(bp, func_mb[func].fw_mb_param);
4940                 bnx2x_func_stats_init(bp);
4941                 bnx2x_hw_stats_post(bp);
4942                 bnx2x_stats_comp(bp);
4943         }
4944
4945         /* restore our func_stx */
4946         bp->func_stx = func_stx;
4947 }
4948
4949 static void bnx2x_func_stats_base_update(struct bnx2x *bp)
4950 {
4951         struct dmae_command *dmae = &bp->stats_dmae;
4952         u32 *stats_comp = bnx2x_sp(bp, stats_comp);
4953
4954         /* sanity */
4955         if (!bp->func_stx) {
4956                 BNX2X_ERR("BUG!\n");
4957                 return;
4958         }
4959
4960         bp->executer_idx = 0;
4961         memset(dmae, 0, sizeof(struct dmae_command));
4962
4963         dmae->opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
4964                         DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
4965                         DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
4966 #ifdef __BIG_ENDIAN
4967                         DMAE_CMD_ENDIANITY_B_DW_SWAP |
4968 #else
4969                         DMAE_CMD_ENDIANITY_DW_SWAP |
4970 #endif
4971                         (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
4972                         (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
4973         dmae->src_addr_lo = bp->func_stx >> 2;
4974         dmae->src_addr_hi = 0;
4975         dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, func_stats_base));
4976         dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, func_stats_base));
4977         dmae->len = sizeof(struct host_func_stats) >> 2;
4978         dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, stats_comp));
4979         dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, stats_comp));
4980         dmae->comp_val = DMAE_COMP_VAL;
4981
4982         *stats_comp = 0;
4983         bnx2x_hw_stats_post(bp);
4984         bnx2x_stats_comp(bp);
4985 }
4986
4987 static void bnx2x_stats_init(struct bnx2x *bp)
4988 {
4989         int port = BP_PORT(bp);
4990         int func = BP_FUNC(bp);
4991         int i;
4992
4993         bp->stats_pending = 0;
4994         bp->executer_idx = 0;
4995         bp->stats_counter = 0;
4996
4997         /* port and func stats for management */
4998         if (!BP_NOMCP(bp)) {
4999                 bp->port.port_stx = SHMEM_RD(bp, port_mb[port].port_stx);
5000                 bp->func_stx = SHMEM_RD(bp, func_mb[func].fw_mb_param);
5001
5002         } else {
5003                 bp->port.port_stx = 0;
5004                 bp->func_stx = 0;
5005         }
5006         DP(BNX2X_MSG_STATS, "port_stx 0x%x  func_stx 0x%x\n",
5007            bp->port.port_stx, bp->func_stx);
5008
5009         /* port stats */
5010         memset(&(bp->port.old_nig_stats), 0, sizeof(struct nig_stats));
5011         bp->port.old_nig_stats.brb_discard =
5012                         REG_RD(bp, NIG_REG_STAT0_BRB_DISCARD + port*0x38);
5013         bp->port.old_nig_stats.brb_truncate =
5014                         REG_RD(bp, NIG_REG_STAT0_BRB_TRUNCATE + port*0x38);
5015         REG_RD_DMAE(bp, NIG_REG_STAT0_EGRESS_MAC_PKT0 + port*0x50,
5016                     &(bp->port.old_nig_stats.egress_mac_pkt0_lo), 2);
5017         REG_RD_DMAE(bp, NIG_REG_STAT0_EGRESS_MAC_PKT1 + port*0x50,
5018                     &(bp->port.old_nig_stats.egress_mac_pkt1_lo), 2);
5019
5020         /* function stats */
5021         for_each_queue(bp, i) {
5022                 struct bnx2x_fastpath *fp = &bp->fp[i];
5023
5024                 memset(&fp->old_tclient, 0,
5025                        sizeof(struct tstorm_per_client_stats));
5026                 memset(&fp->old_uclient, 0,
5027                        sizeof(struct ustorm_per_client_stats));
5028                 memset(&fp->old_xclient, 0,
5029                        sizeof(struct xstorm_per_client_stats));
5030                 memset(&fp->eth_q_stats, 0, sizeof(struct bnx2x_eth_q_stats));
5031         }
5032
5033         memset(&bp->dev->stats, 0, sizeof(struct net_device_stats));
5034         memset(&bp->eth_stats, 0, sizeof(struct bnx2x_eth_stats));
5035
5036         bp->stats_state = STATS_STATE_DISABLED;
5037
5038         if (bp->port.pmf) {
5039                 if (bp->port.port_stx)
5040                         bnx2x_port_stats_base_init(bp);
5041
5042                 if (bp->func_stx)
5043                         bnx2x_func_stats_base_init(bp);
5044
5045         } else if (bp->func_stx)
5046                 bnx2x_func_stats_base_update(bp);
5047 }
5048
5049 static void bnx2x_timer(unsigned long data)
5050 {
5051         struct bnx2x *bp = (struct bnx2x *) data;
5052
5053         if (!netif_running(bp->dev))
5054                 return;
5055
5056         if (atomic_read(&bp->intr_sem) != 0)
5057                 goto timer_restart;
5058
5059         if (poll) {
5060                 struct bnx2x_fastpath *fp = &bp->fp[0];
5061                 int rc;
5062
5063                 bnx2x_tx_int(fp);
5064                 rc = bnx2x_rx_int(fp, 1000);
5065         }
5066
5067         if (!BP_NOMCP(bp)) {
5068                 int func = BP_FUNC(bp);
5069                 u32 drv_pulse;
5070                 u32 mcp_pulse;
5071
5072                 ++bp->fw_drv_pulse_wr_seq;
5073                 bp->fw_drv_pulse_wr_seq &= DRV_PULSE_SEQ_MASK;
5074                 /* TBD - add SYSTEM_TIME */
5075                 drv_pulse = bp->fw_drv_pulse_wr_seq;
5076                 SHMEM_WR(bp, func_mb[func].drv_pulse_mb, drv_pulse);
5077
5078                 mcp_pulse = (SHMEM_RD(bp, func_mb[func].mcp_pulse_mb) &
5079                              MCP_PULSE_SEQ_MASK);
5080                 /* The delta between driver pulse and mcp response
5081                  * should be 1 (before mcp response) or 0 (after mcp response)
5082                  */
5083                 if ((drv_pulse != mcp_pulse) &&
5084                     (drv_pulse != ((mcp_pulse + 1) & MCP_PULSE_SEQ_MASK))) {
5085                         /* someone lost a heartbeat... */
5086                         BNX2X_ERR("drv_pulse (0x%x) != mcp_pulse (0x%x)\n",
5087                                   drv_pulse, mcp_pulse);
5088                 }
5089         }
5090
5091         if (bp->state == BNX2X_STATE_OPEN)
5092                 bnx2x_stats_handle(bp, STATS_EVENT_UPDATE);
5093
5094 timer_restart:
5095         mod_timer(&bp->timer, jiffies + bp->current_interval);
5096 }
5097
5098 /* end of Statistics */
5099
5100 /* nic init */
5101
5102 /*
5103  * nic init service functions
5104  */
5105
5106 static void bnx2x_zero_sb(struct bnx2x *bp, int sb_id)
5107 {
5108         int port = BP_PORT(bp);
5109
5110         /* "CSTORM" */
5111         bnx2x_init_fill(bp, CSEM_REG_FAST_MEMORY +
5112                         CSTORM_SB_HOST_STATUS_BLOCK_U_OFFSET(port, sb_id), 0,
5113                         CSTORM_SB_STATUS_BLOCK_U_SIZE / 4);
5114         bnx2x_init_fill(bp, CSEM_REG_FAST_MEMORY +
5115                         CSTORM_SB_HOST_STATUS_BLOCK_C_OFFSET(port, sb_id), 0,
5116                         CSTORM_SB_STATUS_BLOCK_C_SIZE / 4);
5117 }
5118
5119 static void bnx2x_init_sb(struct bnx2x *bp, struct host_status_block *sb,
5120                           dma_addr_t mapping, int sb_id)
5121 {
5122         int port = BP_PORT(bp);
5123         int func = BP_FUNC(bp);
5124         int index;
5125         u64 section;
5126
5127         /* USTORM */
5128         section = ((u64)mapping) + offsetof(struct host_status_block,
5129                                             u_status_block);
5130         sb->u_status_block.status_block_id = sb_id;
5131
5132         REG_WR(bp, BAR_CSTRORM_INTMEM +
5133                CSTORM_SB_HOST_SB_ADDR_U_OFFSET(port, sb_id), U64_LO(section));
5134         REG_WR(bp, BAR_CSTRORM_INTMEM +
5135                ((CSTORM_SB_HOST_SB_ADDR_U_OFFSET(port, sb_id)) + 4),
5136                U64_HI(section));
5137         REG_WR8(bp, BAR_CSTRORM_INTMEM + FP_USB_FUNC_OFF +
5138                 CSTORM_SB_HOST_STATUS_BLOCK_U_OFFSET(port, sb_id), func);
5139
5140         for (index = 0; index < HC_USTORM_SB_NUM_INDICES; index++)
5141                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5142                          CSTORM_SB_HC_DISABLE_U_OFFSET(port, sb_id, index), 1);
5143
5144         /* CSTORM */
5145         section = ((u64)mapping) + offsetof(struct host_status_block,
5146                                             c_status_block);
5147         sb->c_status_block.status_block_id = sb_id;
5148
5149         REG_WR(bp, BAR_CSTRORM_INTMEM +
5150                CSTORM_SB_HOST_SB_ADDR_C_OFFSET(port, sb_id), U64_LO(section));
5151         REG_WR(bp, BAR_CSTRORM_INTMEM +
5152                ((CSTORM_SB_HOST_SB_ADDR_C_OFFSET(port, sb_id)) + 4),
5153                U64_HI(section));
5154         REG_WR8(bp, BAR_CSTRORM_INTMEM + FP_CSB_FUNC_OFF +
5155                 CSTORM_SB_HOST_STATUS_BLOCK_C_OFFSET(port, sb_id), func);
5156
5157         for (index = 0; index < HC_CSTORM_SB_NUM_INDICES; index++)
5158                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5159                          CSTORM_SB_HC_DISABLE_C_OFFSET(port, sb_id, index), 1);
5160
5161         bnx2x_ack_sb(bp, sb_id, CSTORM_ID, 0, IGU_INT_ENABLE, 0);
5162 }
5163
5164 static void bnx2x_zero_def_sb(struct bnx2x *bp)
5165 {
5166         int func = BP_FUNC(bp);
5167
5168         bnx2x_init_fill(bp, TSEM_REG_FAST_MEMORY +
5169                         TSTORM_DEF_SB_HOST_STATUS_BLOCK_OFFSET(func), 0,
5170                         sizeof(struct tstorm_def_status_block)/4);
5171         bnx2x_init_fill(bp, CSEM_REG_FAST_MEMORY +
5172                         CSTORM_DEF_SB_HOST_STATUS_BLOCK_U_OFFSET(func), 0,
5173                         sizeof(struct cstorm_def_status_block_u)/4);
5174         bnx2x_init_fill(bp, CSEM_REG_FAST_MEMORY +
5175                         CSTORM_DEF_SB_HOST_STATUS_BLOCK_C_OFFSET(func), 0,
5176                         sizeof(struct cstorm_def_status_block_c)/4);
5177         bnx2x_init_fill(bp, XSEM_REG_FAST_MEMORY +
5178                         XSTORM_DEF_SB_HOST_STATUS_BLOCK_OFFSET(func), 0,
5179                         sizeof(struct xstorm_def_status_block)/4);
5180 }
5181
5182 static void bnx2x_init_def_sb(struct bnx2x *bp,
5183                               struct host_def_status_block *def_sb,
5184                               dma_addr_t mapping, int sb_id)
5185 {
5186         int port = BP_PORT(bp);
5187         int func = BP_FUNC(bp);
5188         int index, val, reg_offset;
5189         u64 section;
5190
5191         /* ATTN */
5192         section = ((u64)mapping) + offsetof(struct host_def_status_block,
5193                                             atten_status_block);
5194         def_sb->atten_status_block.status_block_id = sb_id;
5195
5196         bp->attn_state = 0;
5197
5198         reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_0 :
5199                              MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0);
5200
5201         for (index = 0; index < MAX_DYNAMIC_ATTN_GRPS; index++) {
5202                 bp->attn_group[index].sig[0] = REG_RD(bp,
5203                                                      reg_offset + 0x10*index);
5204                 bp->attn_group[index].sig[1] = REG_RD(bp,
5205                                                reg_offset + 0x4 + 0x10*index);
5206                 bp->attn_group[index].sig[2] = REG_RD(bp,
5207                                                reg_offset + 0x8 + 0x10*index);
5208                 bp->attn_group[index].sig[3] = REG_RD(bp,
5209                                                reg_offset + 0xc + 0x10*index);
5210         }
5211
5212         reg_offset = (port ? HC_REG_ATTN_MSG1_ADDR_L :
5213                              HC_REG_ATTN_MSG0_ADDR_L);
5214
5215         REG_WR(bp, reg_offset, U64_LO(section));
5216         REG_WR(bp, reg_offset + 4, U64_HI(section));
5217
5218         reg_offset = (port ? HC_REG_ATTN_NUM_P1 : HC_REG_ATTN_NUM_P0);
5219
5220         val = REG_RD(bp, reg_offset);
5221         val |= sb_id;
5222         REG_WR(bp, reg_offset, val);
5223
5224         /* USTORM */
5225         section = ((u64)mapping) + offsetof(struct host_def_status_block,
5226                                             u_def_status_block);
5227         def_sb->u_def_status_block.status_block_id = sb_id;
5228
5229         REG_WR(bp, BAR_CSTRORM_INTMEM +
5230                CSTORM_DEF_SB_HOST_SB_ADDR_U_OFFSET(func), U64_LO(section));
5231         REG_WR(bp, BAR_CSTRORM_INTMEM +
5232                ((CSTORM_DEF_SB_HOST_SB_ADDR_U_OFFSET(func)) + 4),
5233                U64_HI(section));
5234         REG_WR8(bp, BAR_CSTRORM_INTMEM + DEF_USB_FUNC_OFF +
5235                 CSTORM_DEF_SB_HOST_STATUS_BLOCK_U_OFFSET(func), func);
5236
5237         for (index = 0; index < HC_USTORM_DEF_SB_NUM_INDICES; index++)
5238                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5239                          CSTORM_DEF_SB_HC_DISABLE_U_OFFSET(func, index), 1);
5240
5241         /* CSTORM */
5242         section = ((u64)mapping) + offsetof(struct host_def_status_block,
5243                                             c_def_status_block);
5244         def_sb->c_def_status_block.status_block_id = sb_id;
5245
5246         REG_WR(bp, BAR_CSTRORM_INTMEM +
5247                CSTORM_DEF_SB_HOST_SB_ADDR_C_OFFSET(func), U64_LO(section));
5248         REG_WR(bp, BAR_CSTRORM_INTMEM +
5249                ((CSTORM_DEF_SB_HOST_SB_ADDR_C_OFFSET(func)) + 4),
5250                U64_HI(section));
5251         REG_WR8(bp, BAR_CSTRORM_INTMEM + DEF_CSB_FUNC_OFF +
5252                 CSTORM_DEF_SB_HOST_STATUS_BLOCK_C_OFFSET(func), func);
5253
5254         for (index = 0; index < HC_CSTORM_DEF_SB_NUM_INDICES; index++)
5255                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5256                          CSTORM_DEF_SB_HC_DISABLE_C_OFFSET(func, index), 1);
5257
5258         /* TSTORM */
5259         section = ((u64)mapping) + offsetof(struct host_def_status_block,
5260                                             t_def_status_block);
5261         def_sb->t_def_status_block.status_block_id = sb_id;
5262
5263         REG_WR(bp, BAR_TSTRORM_INTMEM +
5264                TSTORM_DEF_SB_HOST_SB_ADDR_OFFSET(func), U64_LO(section));
5265         REG_WR(bp, BAR_TSTRORM_INTMEM +
5266                ((TSTORM_DEF_SB_HOST_SB_ADDR_OFFSET(func)) + 4),
5267                U64_HI(section));
5268         REG_WR8(bp, BAR_TSTRORM_INTMEM + DEF_TSB_FUNC_OFF +
5269                 TSTORM_DEF_SB_HOST_STATUS_BLOCK_OFFSET(func), func);
5270
5271         for (index = 0; index < HC_TSTORM_DEF_SB_NUM_INDICES; index++)
5272                 REG_WR16(bp, BAR_TSTRORM_INTMEM +
5273                          TSTORM_DEF_SB_HC_DISABLE_OFFSET(func, index), 1);
5274
5275         /* XSTORM */
5276         section = ((u64)mapping) + offsetof(struct host_def_status_block,
5277                                             x_def_status_block);
5278         def_sb->x_def_status_block.status_block_id = sb_id;
5279
5280         REG_WR(bp, BAR_XSTRORM_INTMEM +
5281                XSTORM_DEF_SB_HOST_SB_ADDR_OFFSET(func), U64_LO(section));
5282         REG_WR(bp, BAR_XSTRORM_INTMEM +
5283                ((XSTORM_DEF_SB_HOST_SB_ADDR_OFFSET(func)) + 4),
5284                U64_HI(section));
5285         REG_WR8(bp, BAR_XSTRORM_INTMEM + DEF_XSB_FUNC_OFF +
5286                 XSTORM_DEF_SB_HOST_STATUS_BLOCK_OFFSET(func), func);
5287
5288         for (index = 0; index < HC_XSTORM_DEF_SB_NUM_INDICES; index++)
5289                 REG_WR16(bp, BAR_XSTRORM_INTMEM +
5290                          XSTORM_DEF_SB_HC_DISABLE_OFFSET(func, index), 1);
5291
5292         bp->stats_pending = 0;
5293         bp->set_mac_pending = 0;
5294
5295         bnx2x_ack_sb(bp, sb_id, CSTORM_ID, 0, IGU_INT_ENABLE, 0);
5296 }
5297
5298 static void bnx2x_update_coalesce(struct bnx2x *bp)
5299 {
5300         int port = BP_PORT(bp);
5301         int i;
5302
5303         for_each_queue(bp, i) {
5304                 int sb_id = bp->fp[i].sb_id;
5305
5306                 /* HC_INDEX_U_ETH_RX_CQ_CONS */
5307                 REG_WR8(bp, BAR_CSTRORM_INTMEM +
5308                         CSTORM_SB_HC_TIMEOUT_U_OFFSET(port, sb_id,
5309                                                       U_SB_ETH_RX_CQ_INDEX),
5310                         bp->rx_ticks/(4 * BNX2X_BTR));
5311                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5312                          CSTORM_SB_HC_DISABLE_U_OFFSET(port, sb_id,
5313                                                        U_SB_ETH_RX_CQ_INDEX),
5314                          (bp->rx_ticks/(4 * BNX2X_BTR)) ? 0 : 1);
5315
5316                 /* HC_INDEX_C_ETH_TX_CQ_CONS */
5317                 REG_WR8(bp, BAR_CSTRORM_INTMEM +
5318                         CSTORM_SB_HC_TIMEOUT_C_OFFSET(port, sb_id,
5319                                                       C_SB_ETH_TX_CQ_INDEX),
5320                         bp->tx_ticks/(4 * BNX2X_BTR));
5321                 REG_WR16(bp, BAR_CSTRORM_INTMEM +
5322                          CSTORM_SB_HC_DISABLE_C_OFFSET(port, sb_id,
5323                                                        C_SB_ETH_TX_CQ_INDEX),
5324                          (bp->tx_ticks/(4 * BNX2X_BTR)) ? 0 : 1);
5325         }
5326 }
5327
5328 static inline void bnx2x_free_tpa_pool(struct bnx2x *bp,
5329                                        struct bnx2x_fastpath *fp, int last)
5330 {
5331         int i;
5332
5333         for (i = 0; i < last; i++) {
5334                 struct sw_rx_bd *rx_buf = &(fp->tpa_pool[i]);
5335                 struct sk_buff *skb = rx_buf->skb;
5336
5337                 if (skb == NULL) {
5338                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
5339                         continue;
5340                 }
5341
5342                 if (fp->tpa_state[i] == BNX2X_TPA_START)
5343                         dma_unmap_single(&bp->pdev->dev,
5344                                          dma_unmap_addr(rx_buf, mapping),
5345                                          bp->rx_buf_size, DMA_FROM_DEVICE);
5346
5347                 dev_kfree_skb(skb);
5348                 rx_buf->skb = NULL;
5349         }
5350 }
5351
5352 static void bnx2x_init_rx_rings(struct bnx2x *bp)
5353 {
5354         int func = BP_FUNC(bp);
5355         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
5356                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
5357         u16 ring_prod, cqe_ring_prod;
5358         int i, j;
5359
5360         bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
5361         DP(NETIF_MSG_IFUP,
5362            "mtu %d  rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
5363
5364         if (bp->flags & TPA_ENABLE_FLAG) {
5365
5366                 for_each_queue(bp, j) {
5367                         struct bnx2x_fastpath *fp = &bp->fp[j];
5368
5369                         for (i = 0; i < max_agg_queues; i++) {
5370                                 fp->tpa_pool[i].skb =
5371                                    netdev_alloc_skb(bp->dev, bp->rx_buf_size);
5372                                 if (!fp->tpa_pool[i].skb) {
5373                                         BNX2X_ERR("Failed to allocate TPA "
5374                                                   "skb pool for queue[%d] - "
5375                                                   "disabling TPA on this "
5376                                                   "queue!\n", j);
5377                                         bnx2x_free_tpa_pool(bp, fp, i);
5378                                         fp->disable_tpa = 1;
5379                                         break;
5380                                 }
5381                                 dma_unmap_addr_set((struct sw_rx_bd *)
5382                                                         &bp->fp->tpa_pool[i],
5383                                                    mapping, 0);
5384                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
5385                         }
5386                 }
5387         }
5388
5389         for_each_queue(bp, j) {
5390                 struct bnx2x_fastpath *fp = &bp->fp[j];
5391
5392                 fp->rx_bd_cons = 0;
5393                 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
5394                 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
5395
5396                 /* "next page" elements initialization */
5397                 /* SGE ring */
5398                 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
5399                         struct eth_rx_sge *sge;
5400
5401                         sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
5402                         sge->addr_hi =
5403                                 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
5404                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
5405                         sge->addr_lo =
5406                                 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
5407                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
5408                 }
5409
5410                 bnx2x_init_sge_ring_bit_mask(fp);
5411
5412                 /* RX BD ring */
5413                 for (i = 1; i <= NUM_RX_RINGS; i++) {
5414                         struct eth_rx_bd *rx_bd;
5415
5416                         rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
5417                         rx_bd->addr_hi =
5418                                 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
5419                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
5420                         rx_bd->addr_lo =
5421                                 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
5422                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
5423                 }
5424
5425                 /* CQ ring */
5426                 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
5427                         struct eth_rx_cqe_next_page *nextpg;
5428
5429                         nextpg = (struct eth_rx_cqe_next_page *)
5430                                 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
5431                         nextpg->addr_hi =
5432                                 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
5433                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
5434                         nextpg->addr_lo =
5435                                 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
5436                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
5437                 }
5438
5439                 /* Allocate SGEs and initialize the ring elements */
5440                 for (i = 0, ring_prod = 0;
5441                      i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
5442
5443                         if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
5444                                 BNX2X_ERR("was only able to allocate "
5445                                           "%d rx sges\n", i);
5446                                 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
5447                                 /* Cleanup already allocated elements */
5448                                 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
5449                                 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
5450                                 fp->disable_tpa = 1;
5451                                 ring_prod = 0;
5452                                 break;
5453                         }
5454                         ring_prod = NEXT_SGE_IDX(ring_prod);
5455                 }
5456                 fp->rx_sge_prod = ring_prod;
5457
5458                 /* Allocate BDs and initialize BD ring */
5459                 fp->rx_comp_cons = 0;
5460                 cqe_ring_prod = ring_prod = 0;
5461                 for (i = 0; i < bp->rx_ring_size; i++) {
5462                         if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
5463                                 BNX2X_ERR("was only able to allocate "
5464                                           "%d rx skbs on queue[%d]\n", i, j);
5465                                 fp->eth_q_stats.rx_skb_alloc_failed++;
5466                                 break;
5467                         }
5468                         ring_prod = NEXT_RX_IDX(ring_prod);
5469                         cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
5470                         WARN_ON(ring_prod <= i);
5471                 }
5472
5473                 fp->rx_bd_prod = ring_prod;
5474                 /* must not have more available CQEs than BDs */
5475                 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
5476                                          cqe_ring_prod);
5477                 fp->rx_pkt = fp->rx_calls = 0;
5478
5479                 /* Warning!
5480                  * this will generate an interrupt (to the TSTORM)
5481                  * must only be done after chip is initialized
5482                  */
5483                 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
5484                                      fp->rx_sge_prod);
5485                 if (j != 0)
5486                         continue;
5487
5488                 REG_WR(bp, BAR_USTRORM_INTMEM +
5489                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
5490                        U64_LO(fp->rx_comp_mapping));
5491                 REG_WR(bp, BAR_USTRORM_INTMEM +
5492                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
5493                        U64_HI(fp->rx_comp_mapping));
5494         }
5495 }
5496
5497 static void bnx2x_init_tx_ring(struct bnx2x *bp)
5498 {
5499         int i, j;
5500
5501         for_each_queue(bp, j) {
5502                 struct bnx2x_fastpath *fp = &bp->fp[j];
5503
5504                 for (i = 1; i <= NUM_TX_RINGS; i++) {
5505                         struct eth_tx_next_bd *tx_next_bd =
5506                                 &fp->tx_desc_ring[TX_DESC_CNT * i - 1].next_bd;
5507
5508                         tx_next_bd->addr_hi =
5509                                 cpu_to_le32(U64_HI(fp->tx_desc_mapping +
5510                                             BCM_PAGE_SIZE*(i % NUM_TX_RINGS)));
5511                         tx_next_bd->addr_lo =
5512                                 cpu_to_le32(U64_LO(fp->tx_desc_mapping +
5513                                             BCM_PAGE_SIZE*(i % NUM_TX_RINGS)));
5514                 }
5515
5516                 fp->tx_db.data.header.header = DOORBELL_HDR_DB_TYPE;
5517                 fp->tx_db.data.zero_fill1 = 0;
5518                 fp->tx_db.data.prod = 0;
5519
5520                 fp->tx_pkt_prod = 0;
5521                 fp->tx_pkt_cons = 0;
5522                 fp->tx_bd_prod = 0;
5523                 fp->tx_bd_cons = 0;
5524                 fp->tx_cons_sb = BNX2X_TX_SB_INDEX;
5525                 fp->tx_pkt = 0;
5526         }
5527 }
5528
5529 static void bnx2x_init_sp_ring(struct bnx2x *bp)
5530 {
5531         int func = BP_FUNC(bp);
5532
5533         spin_lock_init(&bp->spq_lock);
5534
5535         bp->spq_left = MAX_SPQ_PENDING;
5536         bp->spq_prod_idx = 0;
5537         bp->dsb_sp_prod = BNX2X_SP_DSB_INDEX;
5538         bp->spq_prod_bd = bp->spq;
5539         bp->spq_last_bd = bp->spq_prod_bd + MAX_SP_DESC_CNT;
5540
5541         REG_WR(bp, XSEM_REG_FAST_MEMORY + XSTORM_SPQ_PAGE_BASE_OFFSET(func),
5542                U64_LO(bp->spq_mapping));
5543         REG_WR(bp,
5544                XSEM_REG_FAST_MEMORY + XSTORM_SPQ_PAGE_BASE_OFFSET(func) + 4,
5545                U64_HI(bp->spq_mapping));
5546
5547         REG_WR(bp, XSEM_REG_FAST_MEMORY + XSTORM_SPQ_PROD_OFFSET(func),
5548                bp->spq_prod_idx);
5549 }
5550
5551 static void bnx2x_init_context(struct bnx2x *bp)
5552 {
5553         int i;
5554
5555         /* Rx */
5556         for_each_queue(bp, i) {
5557                 struct eth_context *context = bnx2x_sp(bp, context[i].eth);
5558                 struct bnx2x_fastpath *fp = &bp->fp[i];
5559                 u8 cl_id = fp->cl_id;
5560
5561                 context->ustorm_st_context.common.sb_index_numbers =
5562                                                 BNX2X_RX_SB_INDEX_NUM;
5563                 context->ustorm_st_context.common.clientId = cl_id;
5564                 context->ustorm_st_context.common.status_block_id = fp->sb_id;
5565                 context->ustorm_st_context.common.flags =
5566                         (USTORM_ETH_ST_CONTEXT_CONFIG_ENABLE_MC_ALIGNMENT |
5567                          USTORM_ETH_ST_CONTEXT_CONFIG_ENABLE_STATISTICS);
5568                 context->ustorm_st_context.common.statistics_counter_id =
5569                                                 cl_id;
5570                 context->ustorm_st_context.common.mc_alignment_log_size =
5571                                                 BNX2X_RX_ALIGN_SHIFT;
5572                 context->ustorm_st_context.common.bd_buff_size =
5573                                                 bp->rx_buf_size;
5574                 context->ustorm_st_context.common.bd_page_base_hi =
5575                                                 U64_HI(fp->rx_desc_mapping);
5576                 context->ustorm_st_context.common.bd_page_base_lo =
5577                                                 U64_LO(fp->rx_desc_mapping);
5578                 if (!fp->disable_tpa) {
5579                         context->ustorm_st_context.common.flags |=
5580                                 USTORM_ETH_ST_CONTEXT_CONFIG_ENABLE_TPA;
5581                         context->ustorm_st_context.common.sge_buff_size =
5582                                 (u16)min_t(u32, SGE_PAGE_SIZE*PAGES_PER_SGE,
5583                                            0xffff);
5584                         context->ustorm_st_context.common.sge_page_base_hi =
5585                                                 U64_HI(fp->rx_sge_mapping);
5586                         context->ustorm_st_context.common.sge_page_base_lo =
5587                                                 U64_LO(fp->rx_sge_mapping);
5588
5589                         context->ustorm_st_context.common.max_sges_for_packet =
5590                                 SGE_PAGE_ALIGN(bp->dev->mtu) >> SGE_PAGE_SHIFT;
5591                         context->ustorm_st_context.common.max_sges_for_packet =
5592                                 ((context->ustorm_st_context.common.
5593                                   max_sges_for_packet + PAGES_PER_SGE - 1) &
5594                                  (~(PAGES_PER_SGE - 1))) >> PAGES_PER_SGE_SHIFT;
5595                 }
5596
5597                 context->ustorm_ag_context.cdu_usage =
5598                         CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, i),
5599                                                CDU_REGION_NUMBER_UCM_AG,
5600                                                ETH_CONNECTION_TYPE);
5601
5602                 context->xstorm_ag_context.cdu_reserved =
5603                         CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, i),
5604                                                CDU_REGION_NUMBER_XCM_AG,
5605                                                ETH_CONNECTION_TYPE);
5606         }
5607
5608         /* Tx */
5609         for_each_queue(bp, i) {
5610                 struct bnx2x_fastpath *fp = &bp->fp[i];
5611                 struct eth_context *context =
5612                         bnx2x_sp(bp, context[i].eth);
5613
5614                 context->cstorm_st_context.sb_index_number =
5615                                                 C_SB_ETH_TX_CQ_INDEX;
5616                 context->cstorm_st_context.status_block_id = fp->sb_id;
5617
5618                 context->xstorm_st_context.tx_bd_page_base_hi =
5619                                                 U64_HI(fp->tx_desc_mapping);
5620                 context->xstorm_st_context.tx_bd_page_base_lo =
5621                                                 U64_LO(fp->tx_desc_mapping);
5622                 context->xstorm_st_context.statistics_data = (fp->cl_id |
5623                                 XSTORM_ETH_ST_CONTEXT_STATISTICS_ENABLE);
5624         }
5625 }
5626
5627 static void bnx2x_init_ind_table(struct bnx2x *bp)
5628 {
5629         int func = BP_FUNC(bp);
5630         int i;
5631
5632         if (bp->multi_mode == ETH_RSS_MODE_DISABLED)
5633                 return;
5634
5635         DP(NETIF_MSG_IFUP,
5636            "Initializing indirection table  multi_mode %d\n", bp->multi_mode);
5637         for (i = 0; i < TSTORM_INDIRECTION_TABLE_SIZE; i++)
5638                 REG_WR8(bp, BAR_TSTRORM_INTMEM +
5639                         TSTORM_INDIRECTION_TABLE_OFFSET(func) + i,
5640                         bp->fp->cl_id + (i % bp->num_queues));
5641 }
5642
5643 static void bnx2x_set_client_config(struct bnx2x *bp)
5644 {
5645         struct tstorm_eth_client_config tstorm_client = {0};
5646         int port = BP_PORT(bp);
5647         int i;
5648
5649         tstorm_client.mtu = bp->dev->mtu;
5650         tstorm_client.config_flags =
5651                                 (TSTORM_ETH_CLIENT_CONFIG_STATSITICS_ENABLE |
5652                                  TSTORM_ETH_CLIENT_CONFIG_E1HOV_REM_ENABLE);
5653 #ifdef BCM_VLAN
5654         if (bp->rx_mode && bp->vlgrp && (bp->flags & HW_VLAN_RX_FLAG)) {
5655                 tstorm_client.config_flags |=
5656                                 TSTORM_ETH_CLIENT_CONFIG_VLAN_REM_ENABLE;
5657                 DP(NETIF_MSG_IFUP, "vlan removal enabled\n");
5658         }
5659 #endif
5660
5661         for_each_queue(bp, i) {
5662                 tstorm_client.statistics_counter_id = bp->fp[i].cl_id;
5663
5664                 REG_WR(bp, BAR_TSTRORM_INTMEM +
5665                        TSTORM_CLIENT_CONFIG_OFFSET(port, bp->fp[i].cl_id),
5666                        ((u32 *)&tstorm_client)[0]);
5667                 REG_WR(bp, BAR_TSTRORM_INTMEM +
5668                        TSTORM_CLIENT_CONFIG_OFFSET(port, bp->fp[i].cl_id) + 4,
5669                        ((u32 *)&tstorm_client)[1]);
5670         }
5671
5672         DP(BNX2X_MSG_OFF, "tstorm_client: 0x%08x 0x%08x\n",
5673            ((u32 *)&tstorm_client)[0], ((u32 *)&tstorm_client)[1]);
5674 }
5675
5676 static void bnx2x_set_storm_rx_mode(struct bnx2x *bp)
5677 {
5678         struct tstorm_eth_mac_filter_config tstorm_mac_filter = {0};
5679         int mode = bp->rx_mode;
5680         int mask = bp->rx_mode_cl_mask;
5681         int func = BP_FUNC(bp);
5682         int port = BP_PORT(bp);
5683         int i;
5684         /* All but management unicast packets should pass to the host as well */
5685         u32 llh_mask =
5686                 NIG_LLH0_BRB1_DRV_MASK_REG_LLH0_BRB1_DRV_MASK_BRCST |
5687                 NIG_LLH0_BRB1_DRV_MASK_REG_LLH0_BRB1_DRV_MASK_MLCST |
5688                 NIG_LLH0_BRB1_DRV_MASK_REG_LLH0_BRB1_DRV_MASK_VLAN |
5689                 NIG_LLH0_BRB1_DRV_MASK_REG_LLH0_BRB1_DRV_MASK_NO_VLAN;
5690
5691         DP(NETIF_MSG_IFUP, "rx mode %d  mask 0x%x\n", mode, mask);
5692
5693         switch (mode) {
5694         case BNX2X_RX_MODE_NONE: /* no Rx */
5695                 tstorm_mac_filter.ucast_drop_all = mask;
5696                 tstorm_mac_filter.mcast_drop_all = mask;
5697                 tstorm_mac_filter.bcast_drop_all = mask;
5698                 break;
5699
5700         case BNX2X_RX_MODE_NORMAL:
5701                 tstorm_mac_filter.bcast_accept_all = mask;
5702                 break;
5703
5704         case BNX2X_RX_MODE_ALLMULTI:
5705                 tstorm_mac_filter.mcast_accept_all = mask;
5706                 tstorm_mac_filter.bcast_accept_all = mask;
5707                 break;
5708
5709         case BNX2X_RX_MODE_PROMISC:
5710                 tstorm_mac_filter.ucast_accept_all = mask;
5711                 tstorm_mac_filter.mcast_accept_all = mask;
5712                 tstorm_mac_filter.bcast_accept_all = mask;
5713                 /* pass management unicast packets as well */
5714                 llh_mask |= NIG_LLH0_BRB1_DRV_MASK_REG_LLH0_BRB1_DRV_MASK_UNCST;
5715                 break;
5716
5717         default:
5718                 BNX2X_ERR("BAD rx mode (%d)\n", mode);
5719                 break;
5720         }
5721
5722         REG_WR(bp,
5723                (port ? NIG_REG_LLH1_BRB1_DRV_MASK : NIG_REG_LLH0_BRB1_DRV_MASK),
5724                llh_mask);
5725
5726         for (i = 0; i < sizeof(struct tstorm_eth_mac_filter_config)/4; i++) {
5727                 REG_WR(bp, BAR_TSTRORM_INTMEM +
5728                        TSTORM_MAC_FILTER_CONFIG_OFFSET(func) + i * 4,
5729                        ((u32 *)&tstorm_mac_filter)[i]);
5730
5731 /*              DP(NETIF_MSG_IFUP, "tstorm_mac_filter[%d]: 0x%08x\n", i,
5732                    ((u32 *)&tstorm_mac_filter)[i]); */
5733         }
5734
5735         if (mode != BNX2X_RX_MODE_NONE)
5736                 bnx2x_set_client_config(bp);
5737 }
5738
5739 static void bnx2x_init_internal_common(struct bnx2x *bp)
5740 {
5741         int i;
5742
5743         /* Zero this manually as its initialization is
5744            currently missing in the initTool */
5745         for (i = 0; i < (USTORM_AGG_DATA_SIZE >> 2); i++)
5746                 REG_WR(bp, BAR_USTRORM_INTMEM +
5747                        USTORM_AGG_DATA_OFFSET + i * 4, 0);
5748 }
5749
5750 static void bnx2x_init_internal_port(struct bnx2x *bp)
5751 {
5752         int port = BP_PORT(bp);
5753
5754         REG_WR(bp,
5755                BAR_CSTRORM_INTMEM + CSTORM_HC_BTR_U_OFFSET(port), BNX2X_BTR);
5756         REG_WR(bp,
5757                BAR_CSTRORM_INTMEM + CSTORM_HC_BTR_C_OFFSET(port), BNX2X_BTR);
5758         REG_WR(bp, BAR_TSTRORM_INTMEM + TSTORM_HC_BTR_OFFSET(port), BNX2X_BTR);
5759         REG_WR(bp, BAR_XSTRORM_INTMEM + XSTORM_HC_BTR_OFFSET(port), BNX2X_BTR);
5760 }
5761
5762 static void bnx2x_init_internal_func(struct bnx2x *bp)
5763 {
5764         struct tstorm_eth_function_common_config tstorm_config = {0};
5765         struct stats_indication_flags stats_flags = {0};
5766         int port = BP_PORT(bp);
5767         int func = BP_FUNC(bp);
5768         int i, j;
5769         u32 offset;
5770         u16 max_agg_size;
5771
5772         tstorm_config.config_flags = RSS_FLAGS(bp);
5773
5774         if (is_multi(bp))
5775                 tstorm_config.rss_result_mask = MULTI_MASK;
5776
5777         /* Enable TPA if needed */
5778         if (bp->flags & TPA_ENABLE_FLAG)
5779                 tstorm_config.config_flags |=
5780                         TSTORM_ETH_FUNCTION_COMMON_CONFIG_ENABLE_TPA;
5781
5782         if (IS_E1HMF(bp))
5783                 tstorm_config.config_flags |=
5784                                 TSTORM_ETH_FUNCTION_COMMON_CONFIG_E1HOV_IN_CAM;
5785
5786         tstorm_config.leading_client_id = BP_L_ID(bp);
5787
5788         REG_WR(bp, BAR_TSTRORM_INTMEM +
5789                TSTORM_FUNCTION_COMMON_CONFIG_OFFSET(func),
5790                (*(u32 *)&tstorm_config));
5791
5792         bp->rx_mode = BNX2X_RX_MODE_NONE; /* no rx until link is up */
5793         bp->rx_mode_cl_mask = (1 << BP_L_ID(bp));
5794         bnx2x_set_storm_rx_mode(bp);
5795
5796         for_each_queue(bp, i) {
5797                 u8 cl_id = bp->fp[i].cl_id;
5798
5799                 /* reset xstorm per client statistics */
5800                 offset = BAR_XSTRORM_INTMEM +
5801                          XSTORM_PER_COUNTER_ID_STATS_OFFSET(port, cl_id);
5802                 for (j = 0;
5803                      j < sizeof(struct xstorm_per_client_stats) / 4; j++)
5804                         REG_WR(bp, offset + j*4, 0);
5805
5806                 /* reset tstorm per client statistics */
5807                 offset = BAR_TSTRORM_INTMEM +
5808                          TSTORM_PER_COUNTER_ID_STATS_OFFSET(port, cl_id);
5809                 for (j = 0;
5810                      j < sizeof(struct tstorm_per_client_stats) / 4; j++)
5811                         REG_WR(bp, offset + j*4, 0);
5812
5813                 /* reset ustorm per client statistics */
5814                 offset = BAR_USTRORM_INTMEM +
5815                          USTORM_PER_COUNTER_ID_STATS_OFFSET(port, cl_id);
5816                 for (j = 0;
5817                      j < sizeof(struct ustorm_per_client_stats) / 4; j++)
5818                         REG_WR(bp, offset + j*4, 0);
5819         }
5820
5821         /* Init statistics related context */
5822         stats_flags.collect_eth = 1;
5823
5824         REG_WR(bp, BAR_XSTRORM_INTMEM + XSTORM_STATS_FLAGS_OFFSET(func),
5825                ((u32 *)&stats_flags)[0]);
5826         REG_WR(bp, BAR_XSTRORM_INTMEM + XSTORM_STATS_FLAGS_OFFSET(func) + 4,
5827                ((u32 *)&stats_flags)[1]);
5828
5829         REG_WR(bp, BAR_TSTRORM_INTMEM + TSTORM_STATS_FLAGS_OFFSET(func),
5830                ((u32 *)&stats_flags)[0]);
5831         REG_WR(bp, BAR_TSTRORM_INTMEM + TSTORM_STATS_FLAGS_OFFSET(func) + 4,
5832                ((u32 *)&stats_flags)[1]);
5833
5834         REG_WR(bp, BAR_USTRORM_INTMEM + USTORM_STATS_FLAGS_OFFSET(func),
5835                ((u32 *)&stats_flags)[0]);
5836         REG_WR(bp, BAR_USTRORM_INTMEM + USTORM_STATS_FLAGS_OFFSET(func) + 4,
5837                ((u32 *)&stats_flags)[1]);
5838
5839         REG_WR(bp, BAR_CSTRORM_INTMEM + CSTORM_STATS_FLAGS_OFFSET(func),
5840                ((u32 *)&stats_flags)[0]);
5841         REG_WR(bp, BAR_CSTRORM_INTMEM + CSTORM_STATS_FLAGS_OFFSET(func) + 4,
5842                ((u32 *)&stats_flags)[1]);
5843
5844         REG_WR(bp, BAR_XSTRORM_INTMEM +
5845                XSTORM_ETH_STATS_QUERY_ADDR_OFFSET(func),
5846                U64_LO(bnx2x_sp_mapping(bp, fw_stats)));
5847         REG_WR(bp, BAR_XSTRORM_INTMEM +
5848                XSTORM_ETH_STATS_QUERY_ADDR_OFFSET(func) + 4,
5849                U64_HI(bnx2x_sp_mapping(bp, fw_stats)));
5850
5851         REG_WR(bp, BAR_TSTRORM_INTMEM +
5852                TSTORM_ETH_STATS_QUERY_ADDR_OFFSET(func),
5853                U64_LO(bnx2x_sp_mapping(bp, fw_stats)));
5854         REG_WR(bp, BAR_TSTRORM_INTMEM +
5855                TSTORM_ETH_STATS_QUERY_ADDR_OFFSET(func) + 4,
5856                U64_HI(bnx2x_sp_mapping(bp, fw_stats)));
5857
5858         REG_WR(bp, BAR_USTRORM_INTMEM +
5859                USTORM_ETH_STATS_QUERY_ADDR_OFFSET(func),
5860                U64_LO(bnx2x_sp_mapping(bp, fw_stats)));
5861         REG_WR(bp, BAR_USTRORM_INTMEM +
5862                USTORM_ETH_STATS_QUERY_ADDR_OFFSET(func) + 4,
5863                U64_HI(bnx2x_sp_mapping(bp, fw_stats)));
5864
5865         if (CHIP_IS_E1H(bp)) {
5866                 REG_WR8(bp, BAR_XSTRORM_INTMEM + XSTORM_FUNCTION_MODE_OFFSET,
5867                         IS_E1HMF(bp));
5868                 REG_WR8(bp, BAR_TSTRORM_INTMEM + TSTORM_FUNCTION_MODE_OFFSET,
5869                         IS_E1HMF(bp));
5870                 REG_WR8(bp, BAR_CSTRORM_INTMEM + CSTORM_FUNCTION_MODE_OFFSET,
5871                         IS_E1HMF(bp));
5872                 REG_WR8(bp, BAR_USTRORM_INTMEM + USTORM_FUNCTION_MODE_OFFSET,
5873                         IS_E1HMF(bp));
5874
5875                 REG_WR16(bp, BAR_XSTRORM_INTMEM + XSTORM_E1HOV_OFFSET(func),
5876                          bp->e1hov);
5877         }
5878
5879         /* Init CQ ring mapping and aggregation size, the FW limit is 8 frags */
5880         max_agg_size = min_t(u32, (min_t(u32, 8, MAX_SKB_FRAGS) *
5881                                    SGE_PAGE_SIZE * PAGES_PER_SGE), 0xffff);
5882         for_each_queue(bp, i) {
5883                 struct bnx2x_fastpath *fp = &bp->fp[i];
5884
5885                 REG_WR(bp, BAR_USTRORM_INTMEM +
5886                        USTORM_CQE_PAGE_BASE_OFFSET(port, fp->cl_id),
5887                        U64_LO(fp->rx_comp_mapping));
5888                 REG_WR(bp, BAR_USTRORM_INTMEM +
5889                        USTORM_CQE_PAGE_BASE_OFFSET(port, fp->cl_id) + 4,
5890                        U64_HI(fp->rx_comp_mapping));
5891
5892                 /* Next page */
5893                 REG_WR(bp, BAR_USTRORM_INTMEM +
5894                        USTORM_CQE_PAGE_NEXT_OFFSET(port, fp->cl_id),
5895                        U64_LO(fp->rx_comp_mapping + BCM_PAGE_SIZE));
5896                 REG_WR(bp, BAR_USTRORM_INTMEM +
5897                        USTORM_CQE_PAGE_NEXT_OFFSET(port, fp->cl_id) + 4,
5898                        U64_HI(fp->rx_comp_mapping + BCM_PAGE_SIZE));
5899
5900                 REG_WR16(bp, BAR_USTRORM_INTMEM +
5901                          USTORM_MAX_AGG_SIZE_OFFSET(port, fp->cl_id),
5902                          max_agg_size);
5903         }
5904
5905         /* dropless flow control */
5906         if (CHIP_IS_E1H(bp)) {
5907                 struct ustorm_eth_rx_pause_data_e1h rx_pause = {0};
5908
5909                 rx_pause.bd_thr_low = 250;
5910                 rx_pause.cqe_thr_low = 250;
5911                 rx_pause.cos = 1;
5912                 rx_pause.sge_thr_low = 0;
5913                 rx_pause.bd_thr_high = 350;
5914                 rx_pause.cqe_thr_high = 350;
5915                 rx_pause.sge_thr_high = 0;
5916
5917                 for_each_queue(bp, i) {
5918                         struct bnx2x_fastpath *fp = &bp->fp[i];
5919
5920                         if (!fp->disable_tpa) {
5921                                 rx_pause.sge_thr_low = 150;
5922                                 rx_pause.sge_thr_high = 250;
5923                         }
5924
5925
5926                         offset = BAR_USTRORM_INTMEM +
5927                                  USTORM_ETH_RING_PAUSE_DATA_OFFSET(port,
5928                                                                    fp->cl_id);
5929                         for (j = 0;
5930                              j < sizeof(struct ustorm_eth_rx_pause_data_e1h)/4;
5931                              j++)
5932                                 REG_WR(bp, offset + j*4,
5933                                        ((u32 *)&rx_pause)[j]);
5934                 }
5935         }
5936
5937         memset(&(bp->cmng), 0, sizeof(struct cmng_struct_per_port));
5938
5939         /* Init rate shaping and fairness contexts */
5940         if (IS_E1HMF(bp)) {
5941                 int vn;
5942
5943                 /* During init there is no active link
5944                    Until link is up, set link rate to 10Gbps */
5945                 bp->link_vars.line_speed = SPEED_10000;
5946                 bnx2x_init_port_minmax(bp);
5947
5948                 if (!BP_NOMCP(bp))
5949                         bp->mf_config =
5950                               SHMEM_RD(bp, mf_cfg.func_mf_config[func].config);
5951                 bnx2x_calc_vn_weight_sum(bp);
5952
5953                 for (vn = VN_0; vn < E1HVN_MAX; vn++)
5954                         bnx2x_init_vn_minmax(bp, 2*vn + port);
5955
5956                 /* Enable rate shaping and fairness */
5957                 bp->cmng.flags.cmng_enables |=
5958                                         CMNG_FLAGS_PER_PORT_RATE_SHAPING_VN;
5959
5960         } else {
5961                 /* rate shaping and fairness are disabled */
5962                 DP(NETIF_MSG_IFUP,
5963                    "single function mode  minmax will be disabled\n");
5964         }
5965
5966
5967         /* Store cmng structures to internal memory */
5968         if (bp->port.pmf)
5969                 for (i = 0; i < sizeof(struct cmng_struct_per_port) / 4; i++)
5970                         REG_WR(bp, BAR_XSTRORM_INTMEM +
5971                                XSTORM_CMNG_PER_PORT_VARS_OFFSET(port) + i * 4,
5972                                ((u32 *)(&bp->cmng))[i]);
5973 }
5974
5975 static void bnx2x_init_internal(struct bnx2x *bp, u32 load_code)
5976 {
5977         switch (load_code) {
5978         case FW_MSG_CODE_DRV_LOAD_COMMON:
5979                 bnx2x_init_internal_common(bp);
5980                 /* no break */
5981
5982         case FW_MSG_CODE_DRV_LOAD_PORT:
5983                 bnx2x_init_internal_port(bp);
5984                 /* no break */
5985
5986         case FW_MSG_CODE_DRV_LOAD_FUNCTION:
5987                 bnx2x_init_internal_func(bp);
5988                 break;
5989
5990         default:
5991                 BNX2X_ERR("Unknown load_code (0x%x) from MCP\n", load_code);
5992                 break;
5993         }
5994 }
5995
5996 static void bnx2x_nic_init(struct bnx2x *bp, u32 load_code)
5997 {
5998         int i;
5999
6000         for_each_queue(bp, i) {
6001                 struct bnx2x_fastpath *fp = &bp->fp[i];
6002
6003                 fp->bp = bp;
6004                 fp->state = BNX2X_FP_STATE_CLOSED;
6005                 fp->index = i;
6006                 fp->cl_id = BP_L_ID(bp) + i;
6007 #ifdef BCM_CNIC
6008                 fp->sb_id = fp->cl_id + 1;
6009 #else
6010                 fp->sb_id = fp->cl_id;
6011 #endif
6012                 DP(NETIF_MSG_IFUP,
6013                    "queue[%d]:  bnx2x_init_sb(%p,%p)  cl_id %d  sb %d\n",
6014                    i, bp, fp->status_blk, fp->cl_id, fp->sb_id);
6015                 bnx2x_init_sb(bp, fp->status_blk, fp->status_blk_mapping,
6016                               fp->sb_id);
6017                 bnx2x_update_fpsb_idx(fp);
6018         }
6019
6020         /* ensure status block indices were read */
6021         rmb();
6022
6023
6024         bnx2x_init_def_sb(bp, bp->def_status_blk, bp->def_status_blk_mapping,
6025                           DEF_SB_ID);
6026         bnx2x_update_dsb_idx(bp);
6027         bnx2x_update_coalesce(bp);
6028         bnx2x_init_rx_rings(bp);
6029         bnx2x_init_tx_ring(bp);
6030         bnx2x_init_sp_ring(bp);
6031         bnx2x_init_context(bp);
6032         bnx2x_init_internal(bp, load_code);
6033         bnx2x_init_ind_table(bp);
6034         bnx2x_stats_init(bp);
6035
6036         /* At this point, we are ready for interrupts */
6037         atomic_set(&bp->intr_sem, 0);
6038
6039         /* flush all before enabling interrupts */
6040         mb();
6041         mmiowb();
6042
6043         bnx2x_int_enable(bp);
6044
6045         /* Check for SPIO5 */
6046         bnx2x_attn_int_deasserted0(bp,
6047                 REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_1_FUNC_0 + BP_PORT(bp)*4) &
6048                                    AEU_INPUTS_ATTN_BITS_SPIO5);
6049 }
6050
6051 /* end of nic init */
6052
6053 /*
6054  * gzip service functions
6055  */
6056
6057 static int bnx2x_gunzip_init(struct bnx2x *bp)
6058 {
6059         bp->gunzip_buf = dma_alloc_coherent(&bp->pdev->dev, FW_BUF_SIZE,
6060                                             &bp->gunzip_mapping, GFP_KERNEL);
6061         if (bp->gunzip_buf  == NULL)
6062                 goto gunzip_nomem1;
6063
6064         bp->strm = kmalloc(sizeof(*bp->strm), GFP_KERNEL);
6065         if (bp->strm  == NULL)
6066                 goto gunzip_nomem2;
6067
6068         bp->strm->workspace = kmalloc(zlib_inflate_workspacesize(),
6069                                       GFP_KERNEL);
6070         if (bp->strm->workspace == NULL)
6071                 goto gunzip_nomem3;
6072
6073         return 0;
6074
6075 gunzip_nomem3:
6076         kfree(bp->strm);
6077         bp->strm = NULL;
6078
6079 gunzip_nomem2:
6080         dma_free_coherent(&bp->pdev->dev, FW_BUF_SIZE, bp->gunzip_buf,
6081                           bp->gunzip_mapping);
6082         bp->gunzip_buf = NULL;
6083
6084 gunzip_nomem1:
6085         netdev_err(bp->dev, "Cannot allocate firmware buffer for"
6086                " un-compression\n");
6087         return -ENOMEM;
6088 }
6089
6090 static void bnx2x_gunzip_end(struct bnx2x *bp)
6091 {
6092         kfree(bp->strm->workspace);
6093
6094         kfree(bp->strm);
6095         bp->strm = NULL;
6096
6097         if (bp->gunzip_buf) {
6098                 dma_free_coherent(&bp->pdev->dev, FW_BUF_SIZE, bp->gunzip_buf,
6099                                   bp->gunzip_mapping);
6100                 bp->gunzip_buf = NULL;
6101         }
6102 }
6103
6104 static int bnx2x_gunzip(struct bnx2x *bp, const u8 *zbuf, int len)
6105 {
6106         int n, rc;
6107
6108         /* check gzip header */
6109         if ((zbuf[0] != 0x1f) || (zbuf[1] != 0x8b) || (zbuf[2] != Z_DEFLATED)) {
6110                 BNX2X_ERR("Bad gzip header\n");
6111                 return -EINVAL;
6112         }
6113
6114         n = 10;
6115
6116 #define FNAME                           0x8
6117
6118         if (zbuf[3] & FNAME)
6119                 while ((zbuf[n++] != 0) && (n < len));
6120
6121         bp->strm->next_in = (typeof(bp->strm->next_in))zbuf + n;
6122         bp->strm->avail_in = len - n;
6123         bp->strm->next_out = bp->gunzip_buf;
6124         bp->strm->avail_out = FW_BUF_SIZE;
6125
6126         rc = zlib_inflateInit2(bp->strm, -MAX_WBITS);
6127         if (rc != Z_OK)
6128                 return rc;
6129
6130         rc = zlib_inflate(bp->strm, Z_FINISH);
6131         if ((rc != Z_OK) && (rc != Z_STREAM_END))
6132                 netdev_err(bp->dev, "Firmware decompression error: %s\n",
6133                            bp->strm->msg);
6134
6135         bp->gunzip_outlen = (FW_BUF_SIZE - bp->strm->avail_out);
6136         if (bp->gunzip_outlen & 0x3)
6137                 netdev_err(bp->dev, "Firmware decompression error:"
6138                                     " gunzip_outlen (%d) not aligned\n",
6139                                 bp->gunzip_outlen);
6140         bp->gunzip_outlen >>= 2;
6141
6142         zlib_inflateEnd(bp->strm);
6143
6144         if (rc == Z_STREAM_END)
6145                 return 0;
6146
6147         return rc;
6148 }
6149
6150 /* nic load/unload */
6151
6152 /*
6153  * General service functions
6154  */
6155
6156 /* send a NIG loopback debug packet */
6157 static void bnx2x_lb_pckt(struct bnx2x *bp)
6158 {
6159         u32 wb_write[3];
6160
6161         /* Ethernet source and destination addresses */
6162         wb_write[0] = 0x55555555;
6163         wb_write[1] = 0x55555555;
6164         wb_write[2] = 0x20;             /* SOP */
6165         REG_WR_DMAE(bp, NIG_REG_DEBUG_PACKET_LB, wb_write, 3);
6166
6167         /* NON-IP protocol */
6168         wb_write[0] = 0x09000000;
6169         wb_write[1] = 0x55555555;
6170         wb_write[2] = 0x10;             /* EOP, eop_bvalid = 0 */
6171         REG_WR_DMAE(bp, NIG_REG_DEBUG_PACKET_LB, wb_write, 3);
6172 }
6173
6174 /* some of the internal memories
6175  * are not directly readable from the driver
6176  * to test them we send debug packets
6177  */
6178 static int bnx2x_int_mem_test(struct bnx2x *bp)
6179 {
6180         int factor;
6181         int count, i;
6182         u32 val = 0;
6183
6184         if (CHIP_REV_IS_FPGA(bp))
6185                 factor = 120;
6186         else if (CHIP_REV_IS_EMUL(bp))
6187                 factor = 200;
6188         else
6189                 factor = 1;
6190
6191         DP(NETIF_MSG_HW, "start part1\n");
6192
6193         /* Disable inputs of parser neighbor blocks */
6194         REG_WR(bp, TSDM_REG_ENABLE_IN1, 0x0);
6195         REG_WR(bp, TCM_REG_PRS_IFEN, 0x0);
6196         REG_WR(bp, CFC_REG_DEBUG0, 0x1);
6197         REG_WR(bp, NIG_REG_PRS_REQ_IN_EN, 0x0);
6198
6199         /*  Write 0 to parser credits for CFC search request */
6200         REG_WR(bp, PRS_REG_CFC_SEARCH_INITIAL_CREDIT, 0x0);
6201
6202         /* send Ethernet packet */
6203         bnx2x_lb_pckt(bp);
6204
6205         /* TODO do i reset NIG statistic? */
6206         /* Wait until NIG register shows 1 packet of size 0x10 */
6207         count = 1000 * factor;
6208         while (count) {
6209
6210                 bnx2x_read_dmae(bp, NIG_REG_STAT2_BRB_OCTET, 2);
6211                 val = *bnx2x_sp(bp, wb_data[0]);
6212                 if (val == 0x10)
6213                         break;
6214
6215                 msleep(10);
6216                 count--;
6217         }
6218         if (val != 0x10) {
6219                 BNX2X_ERR("NIG timeout  val = 0x%x\n", val);
6220                 return -1;
6221         }
6222
6223         /* Wait until PRS register shows 1 packet */
6224         count = 1000 * factor;
6225         while (count) {
6226                 val = REG_RD(bp, PRS_REG_NUM_OF_PACKETS);
6227                 if (val == 1)
6228                         break;
6229
6230                 msleep(10);
6231                 count--;
6232         }
6233         if (val != 0x1) {
6234                 BNX2X_ERR("PRS timeout val = 0x%x\n", val);
6235                 return -2;
6236         }
6237
6238         /* Reset and init BRB, PRS */
6239         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR, 0x03);
6240         msleep(50);
6241         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET, 0x03);
6242         msleep(50);
6243         bnx2x_init_block(bp, BRB1_BLOCK, COMMON_STAGE);
6244         bnx2x_init_block(bp, PRS_BLOCK, COMMON_STAGE);
6245
6246         DP(NETIF_MSG_HW, "part2\n");
6247
6248         /* Disable inputs of parser neighbor blocks */
6249         REG_WR(bp, TSDM_REG_ENABLE_IN1, 0x0);
6250         REG_WR(bp, TCM_REG_PRS_IFEN, 0x0);
6251         REG_WR(bp, CFC_REG_DEBUG0, 0x1);
6252         REG_WR(bp, NIG_REG_PRS_REQ_IN_EN, 0x0);
6253
6254         /* Write 0 to parser credits for CFC search request */
6255         REG_WR(bp, PRS_REG_CFC_SEARCH_INITIAL_CREDIT, 0x0);
6256
6257         /* send 10 Ethernet packets */
6258         for (i = 0; i < 10; i++)
6259                 bnx2x_lb_pckt(bp);
6260
6261         /* Wait until NIG register shows 10 + 1
6262            packets of size 11*0x10 = 0xb0 */
6263         count = 1000 * factor;
6264         while (count) {
6265
6266                 bnx2x_read_dmae(bp, NIG_REG_STAT2_BRB_OCTET, 2);
6267                 val = *bnx2x_sp(bp, wb_data[0]);
6268                 if (val == 0xb0)
6269                         break;
6270
6271                 msleep(10);
6272                 count--;
6273         }
6274         if (val != 0xb0) {
6275                 BNX2X_ERR("NIG timeout  val = 0x%x\n", val);
6276                 return -3;
6277         }
6278
6279         /* Wait until PRS register shows 2 packets */
6280         val = REG_RD(bp, PRS_REG_NUM_OF_PACKETS);
6281         if (val != 2)
6282                 BNX2X_ERR("PRS timeout  val = 0x%x\n", val);
6283
6284         /* Write 1 to parser credits for CFC search request */
6285         REG_WR(bp, PRS_REG_CFC_SEARCH_INITIAL_CREDIT, 0x1);
6286
6287         /* Wait until PRS register shows 3 packets */
6288         msleep(10 * factor);
6289         /* Wait until NIG register shows 1 packet of size 0x10 */
6290         val = REG_RD(bp, PRS_REG_NUM_OF_PACKETS);
6291         if (val != 3)
6292                 BNX2X_ERR("PRS timeout  val = 0x%x\n", val);
6293
6294         /* clear NIG EOP FIFO */
6295         for (i = 0; i < 11; i++)
6296                 REG_RD(bp, NIG_REG_INGRESS_EOP_LB_FIFO);
6297         val = REG_RD(bp, NIG_REG_INGRESS_EOP_LB_EMPTY);
6298         if (val != 1) {
6299                 BNX2X_ERR("clear of NIG failed\n");
6300                 return -4;
6301         }
6302
6303         /* Reset and init BRB, PRS, NIG */
6304         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR, 0x03);
6305         msleep(50);
6306         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET, 0x03);
6307         msleep(50);
6308         bnx2x_init_block(bp, BRB1_BLOCK, COMMON_STAGE);
6309         bnx2x_init_block(bp, PRS_BLOCK, COMMON_STAGE);
6310 #ifndef BCM_CNIC
6311         /* set NIC mode */
6312         REG_WR(bp, PRS_REG_NIC_MODE, 1);
6313 #endif
6314
6315         /* Enable inputs of parser neighbor blocks */
6316         REG_WR(bp, TSDM_REG_ENABLE_IN1, 0x7fffffff);
6317         REG_WR(bp, TCM_REG_PRS_IFEN, 0x1);
6318         REG_WR(bp, CFC_REG_DEBUG0, 0x0);
6319         REG_WR(bp, NIG_REG_PRS_REQ_IN_EN, 0x1);
6320
6321         DP(NETIF_MSG_HW, "done\n");
6322
6323         return 0; /* OK */
6324 }
6325
6326 static void enable_blocks_attention(struct bnx2x *bp)
6327 {
6328         REG_WR(bp, PXP_REG_PXP_INT_MASK_0, 0);
6329         REG_WR(bp, PXP_REG_PXP_INT_MASK_1, 0);
6330         REG_WR(bp, DORQ_REG_DORQ_INT_MASK, 0);
6331         REG_WR(bp, CFC_REG_CFC_INT_MASK, 0);
6332         REG_WR(bp, QM_REG_QM_INT_MASK, 0);
6333         REG_WR(bp, TM_REG_TM_INT_MASK, 0);
6334         REG_WR(bp, XSDM_REG_XSDM_INT_MASK_0, 0);
6335         REG_WR(bp, XSDM_REG_XSDM_INT_MASK_1, 0);
6336         REG_WR(bp, XCM_REG_XCM_INT_MASK, 0);
6337 /*      REG_WR(bp, XSEM_REG_XSEM_INT_MASK_0, 0); */
6338 /*      REG_WR(bp, XSEM_REG_XSEM_INT_MASK_1, 0); */
6339         REG_WR(bp, USDM_REG_USDM_INT_MASK_0, 0);
6340         REG_WR(bp, USDM_REG_USDM_INT_MASK_1, 0);
6341         REG_WR(bp, UCM_REG_UCM_INT_MASK, 0);
6342 /*      REG_WR(bp, USEM_REG_USEM_INT_MASK_0, 0); */
6343 /*      REG_WR(bp, USEM_REG_USEM_INT_MASK_1, 0); */
6344         REG_WR(bp, GRCBASE_UPB + PB_REG_PB_INT_MASK, 0);
6345         REG_WR(bp, CSDM_REG_CSDM_INT_MASK_0, 0);
6346         REG_WR(bp, CSDM_REG_CSDM_INT_MASK_1, 0);
6347         REG_WR(bp, CCM_REG_CCM_INT_MASK, 0);
6348 /*      REG_WR(bp, CSEM_REG_CSEM_INT_MASK_0, 0); */
6349 /*      REG_WR(bp, CSEM_REG_CSEM_INT_MASK_1, 0); */
6350         if (CHIP_REV_IS_FPGA(bp))
6351                 REG_WR(bp, PXP2_REG_PXP2_INT_MASK_0, 0x580000);
6352         else
6353                 REG_WR(bp, PXP2_REG_PXP2_INT_MASK_0, 0x480000);
6354         REG_WR(bp, TSDM_REG_TSDM_INT_MASK_0, 0);
6355         REG_WR(bp, TSDM_REG_TSDM_INT_MASK_1, 0);
6356         REG_WR(bp, TCM_REG_TCM_INT_MASK, 0);
6357 /*      REG_WR(bp, TSEM_REG_TSEM_INT_MASK_0, 0); */
6358 /*      REG_WR(bp, TSEM_REG_TSEM_INT_MASK_1, 0); */
6359         REG_WR(bp, CDU_REG_CDU_INT_MASK, 0);
6360         REG_WR(bp, DMAE_REG_DMAE_INT_MASK, 0);
6361 /*      REG_WR(bp, MISC_REG_MISC_INT_MASK, 0); */
6362         REG_WR(bp, PBF_REG_PBF_INT_MASK, 0X18);         /* bit 3,4 masked */
6363 }
6364
6365 static const struct {
6366         u32 addr;
6367         u32 mask;
6368 } bnx2x_parity_mask[] = {
6369         {PXP_REG_PXP_PRTY_MASK, 0xffffffff},
6370         {PXP2_REG_PXP2_PRTY_MASK_0, 0xffffffff},
6371         {PXP2_REG_PXP2_PRTY_MASK_1, 0xffffffff},
6372         {HC_REG_HC_PRTY_MASK, 0xffffffff},
6373         {MISC_REG_MISC_PRTY_MASK, 0xffffffff},
6374         {QM_REG_QM_PRTY_MASK, 0x0},
6375         {DORQ_REG_DORQ_PRTY_MASK, 0x0},
6376         {GRCBASE_UPB + PB_REG_PB_PRTY_MASK, 0x0},
6377         {GRCBASE_XPB + PB_REG_PB_PRTY_MASK, 0x0},
6378         {SRC_REG_SRC_PRTY_MASK, 0x4}, /* bit 2 */
6379         {CDU_REG_CDU_PRTY_MASK, 0x0},
6380         {CFC_REG_CFC_PRTY_MASK, 0x0},
6381         {DBG_REG_DBG_PRTY_MASK, 0x0},
6382         {DMAE_REG_DMAE_PRTY_MASK, 0x0},
6383         {BRB1_REG_BRB1_PRTY_MASK, 0x0},
6384         {PRS_REG_PRS_PRTY_MASK, (1<<6)},/* bit 6 */
6385         {TSDM_REG_TSDM_PRTY_MASK, 0x18},/* bit 3,4 */
6386         {CSDM_REG_CSDM_PRTY_MASK, 0x8}, /* bit 3 */
6387         {USDM_REG_USDM_PRTY_MASK, 0x38},/* bit 3,4,5 */
6388         {XSDM_REG_XSDM_PRTY_MASK, 0x8}, /* bit 3 */
6389         {TSEM_REG_TSEM_PRTY_MASK_0, 0x0},
6390         {TSEM_REG_TSEM_PRTY_MASK_1, 0x0},
6391         {USEM_REG_USEM_PRTY_MASK_0, 0x0},
6392         {USEM_REG_USEM_PRTY_MASK_1, 0x0},
6393         {CSEM_REG_CSEM_PRTY_MASK_0, 0x0},
6394         {CSEM_REG_CSEM_PRTY_MASK_1, 0x0},
6395         {XSEM_REG_XSEM_PRTY_MASK_0, 0x0},
6396         {XSEM_REG_XSEM_PRTY_MASK_1, 0x0}
6397 };
6398
6399 static void enable_blocks_parity(struct bnx2x *bp)
6400 {
6401         int i, mask_arr_len =
6402                 sizeof(bnx2x_parity_mask)/(sizeof(bnx2x_parity_mask[0]));
6403
6404         for (i = 0; i < mask_arr_len; i++)
6405                 REG_WR(bp, bnx2x_parity_mask[i].addr,
6406                         bnx2x_parity_mask[i].mask);
6407 }
6408
6409
6410 static void bnx2x_reset_common(struct bnx2x *bp)
6411 {
6412         /* reset_common */
6413         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR,
6414                0xd3ffff7f);
6415         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_CLEAR, 0x1403);
6416 }
6417
6418 static void bnx2x_init_pxp(struct bnx2x *bp)
6419 {
6420         u16 devctl;
6421         int r_order, w_order;
6422
6423         pci_read_config_word(bp->pdev,
6424                              bp->pcie_cap + PCI_EXP_DEVCTL, &devctl);
6425         DP(NETIF_MSG_HW, "read 0x%x from devctl\n", devctl);
6426         w_order = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5);
6427         if (bp->mrrs == -1)
6428                 r_order = ((devctl & PCI_EXP_DEVCTL_READRQ) >> 12);
6429         else {
6430                 DP(NETIF_MSG_HW, "force read order to %d\n", bp->mrrs);
6431                 r_order = bp->mrrs;
6432         }
6433
6434         bnx2x_init_pxp_arb(bp, r_order, w_order);
6435 }
6436
6437 static void bnx2x_setup_fan_failure_detection(struct bnx2x *bp)
6438 {
6439         int is_required;
6440         u32 val;
6441         int port;
6442
6443         if (BP_NOMCP(bp))
6444                 return;
6445
6446         is_required = 0;
6447         val = SHMEM_RD(bp, dev_info.shared_hw_config.config2) &
6448               SHARED_HW_CFG_FAN_FAILURE_MASK;
6449
6450         if (val == SHARED_HW_CFG_FAN_FAILURE_ENABLED)
6451                 is_required = 1;
6452
6453         /*
6454          * The fan failure mechanism is usually related to the PHY type since
6455          * the power consumption of the board is affected by the PHY. Currently,
6456          * fan is required for most designs with SFX7101, BCM8727 and BCM8481.
6457          */
6458         else if (val == SHARED_HW_CFG_FAN_FAILURE_PHY_TYPE)
6459                 for (port = PORT_0; port < PORT_MAX; port++) {
6460                         u32 phy_type =
6461                                 SHMEM_RD(bp, dev_info.port_hw_config[port].
6462                                          external_phy_config) &
6463                                 PORT_HW_CFG_XGXS_EXT_PHY_TYPE_MASK;
6464                         is_required |=
6465                                 ((phy_type ==
6466                                   PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101) ||
6467                                  (phy_type ==
6468                                   PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727) ||
6469                                  (phy_type ==
6470                                   PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8481));
6471                 }
6472
6473         DP(NETIF_MSG_HW, "fan detection setting: %d\n", is_required);
6474
6475         if (is_required == 0)
6476                 return;
6477
6478         /* Fan failure is indicated by SPIO 5 */
6479         bnx2x_set_spio(bp, MISC_REGISTERS_SPIO_5,
6480                        MISC_REGISTERS_SPIO_INPUT_HI_Z);
6481
6482         /* set to active low mode */
6483         val = REG_RD(bp, MISC_REG_SPIO_INT);
6484         val |= ((1 << MISC_REGISTERS_SPIO_5) <<
6485                                         MISC_REGISTERS_SPIO_INT_OLD_SET_POS);
6486         REG_WR(bp, MISC_REG_SPIO_INT, val);
6487
6488         /* enable interrupt to signal the IGU */
6489         val = REG_RD(bp, MISC_REG_SPIO_EVENT_EN);
6490         val |= (1 << MISC_REGISTERS_SPIO_5);
6491         REG_WR(bp, MISC_REG_SPIO_EVENT_EN, val);
6492 }
6493
6494 static int bnx2x_init_common(struct bnx2x *bp)
6495 {
6496         u32 val, i;
6497 #ifdef BCM_CNIC
6498         u32 wb_write[2];
6499 #endif
6500
6501         DP(BNX2X_MSG_MCP, "starting common init  func %d\n", BP_FUNC(bp));
6502
6503         bnx2x_reset_common(bp);
6504         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET, 0xffffffff);
6505         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET, 0xfffc);
6506
6507         bnx2x_init_block(bp, MISC_BLOCK, COMMON_STAGE);
6508         if (CHIP_IS_E1H(bp))
6509                 REG_WR(bp, MISC_REG_E1HMF_MODE, IS_E1HMF(bp));
6510
6511         REG_WR(bp, MISC_REG_LCPLL_CTRL_REG_2, 0x100);
6512         msleep(30);
6513         REG_WR(bp, MISC_REG_LCPLL_CTRL_REG_2, 0x0);
6514
6515         bnx2x_init_block(bp, PXP_BLOCK, COMMON_STAGE);
6516         if (CHIP_IS_E1(bp)) {
6517                 /* enable HW interrupt from PXP on USDM overflow
6518                    bit 16 on INT_MASK_0 */
6519                 REG_WR(bp, PXP_REG_PXP_INT_MASK_0, 0);
6520         }
6521
6522         bnx2x_init_block(bp, PXP2_BLOCK, COMMON_STAGE);
6523         bnx2x_init_pxp(bp);
6524
6525 #ifdef __BIG_ENDIAN
6526         REG_WR(bp, PXP2_REG_RQ_QM_ENDIAN_M, 1);
6527         REG_WR(bp, PXP2_REG_RQ_TM_ENDIAN_M, 1);
6528         REG_WR(bp, PXP2_REG_RQ_SRC_ENDIAN_M, 1);
6529         REG_WR(bp, PXP2_REG_RQ_CDU_ENDIAN_M, 1);
6530         REG_WR(bp, PXP2_REG_RQ_DBG_ENDIAN_M, 1);
6531         /* make sure this value is 0 */
6532         REG_WR(bp, PXP2_REG_RQ_HC_ENDIAN_M, 0);
6533
6534 /*      REG_WR(bp, PXP2_REG_RD_PBF_SWAP_MODE, 1); */
6535         REG_WR(bp, PXP2_REG_RD_QM_SWAP_MODE, 1);
6536         REG_WR(bp, PXP2_REG_RD_TM_SWAP_MODE, 1);
6537         REG_WR(bp, PXP2_REG_RD_SRC_SWAP_MODE, 1);
6538         REG_WR(bp, PXP2_REG_RD_CDURD_SWAP_MODE, 1);
6539 #endif
6540
6541         REG_WR(bp, PXP2_REG_RQ_CDU_P_SIZE, 2);
6542 #ifdef BCM_CNIC
6543         REG_WR(bp, PXP2_REG_RQ_TM_P_SIZE, 5);
6544         REG_WR(bp, PXP2_REG_RQ_QM_P_SIZE, 5);
6545         REG_WR(bp, PXP2_REG_RQ_SRC_P_SIZE, 5);
6546 #endif
6547
6548         if (CHIP_REV_IS_FPGA(bp) && CHIP_IS_E1H(bp))
6549                 REG_WR(bp, PXP2_REG_PGL_TAGS_LIMIT, 0x1);
6550
6551         /* let the HW do it's magic ... */
6552         msleep(100);
6553         /* finish PXP init */
6554         val = REG_RD(bp, PXP2_REG_RQ_CFG_DONE);
6555         if (val != 1) {
6556                 BNX2X_ERR("PXP2 CFG failed\n");
6557                 return -EBUSY;
6558         }
6559         val = REG_RD(bp, PXP2_REG_RD_INIT_DONE);
6560         if (val != 1) {
6561                 BNX2X_ERR("PXP2 RD_INIT failed\n");
6562                 return -EBUSY;
6563         }
6564
6565         REG_WR(bp, PXP2_REG_RQ_DISABLE_INPUTS, 0);
6566         REG_WR(bp, PXP2_REG_RD_DISABLE_INPUTS, 0);
6567
6568         bnx2x_init_block(bp, DMAE_BLOCK, COMMON_STAGE);
6569
6570         /* clean the DMAE memory */
6571         bp->dmae_ready = 1;
6572         bnx2x_init_fill(bp, TSEM_REG_PRAM, 0, 8);
6573
6574         bnx2x_init_block(bp, TCM_BLOCK, COMMON_STAGE);
6575         bnx2x_init_block(bp, UCM_BLOCK, COMMON_STAGE);
6576         bnx2x_init_block(bp, CCM_BLOCK, COMMON_STAGE);
6577         bnx2x_init_block(bp, XCM_BLOCK, COMMON_STAGE);
6578
6579         bnx2x_read_dmae(bp, XSEM_REG_PASSIVE_BUFFER, 3);
6580         bnx2x_read_dmae(bp, CSEM_REG_PASSIVE_BUFFER, 3);
6581         bnx2x_read_dmae(bp, TSEM_REG_PASSIVE_BUFFER, 3);
6582         bnx2x_read_dmae(bp, USEM_REG_PASSIVE_BUFFER, 3);
6583
6584         bnx2x_init_block(bp, QM_BLOCK, COMMON_STAGE);
6585
6586 #ifdef BCM_CNIC
6587         wb_write[0] = 0;
6588         wb_write[1] = 0;
6589         for (i = 0; i < 64; i++) {
6590                 REG_WR(bp, QM_REG_BASEADDR + i*4, 1024 * 4 * (i%16));
6591                 bnx2x_init_ind_wr(bp, QM_REG_PTRTBL + i*8, wb_write, 2);
6592
6593                 if (CHIP_IS_E1H(bp)) {
6594                         REG_WR(bp, QM_REG_BASEADDR_EXT_A + i*4, 1024*4*(i%16));
6595                         bnx2x_init_ind_wr(bp, QM_REG_PTRTBL_EXT_A + i*8,
6596                                           wb_write, 2);
6597                 }
6598         }
6599 #endif
6600         /* soft reset pulse */
6601         REG_WR(bp, QM_REG_SOFT_RESET, 1);
6602         REG_WR(bp, QM_REG_SOFT_RESET, 0);
6603
6604 #ifdef BCM_CNIC
6605         bnx2x_init_block(bp, TIMERS_BLOCK, COMMON_STAGE);
6606 #endif
6607
6608         bnx2x_init_block(bp, DQ_BLOCK, COMMON_STAGE);
6609         REG_WR(bp, DORQ_REG_DPM_CID_OFST, BCM_PAGE_SHIFT);
6610         if (!CHIP_REV_IS_SLOW(bp)) {
6611                 /* enable hw interrupt from doorbell Q */
6612                 REG_WR(bp, DORQ_REG_DORQ_INT_MASK, 0);
6613         }
6614
6615         bnx2x_init_block(bp, BRB1_BLOCK, COMMON_STAGE);
6616         bnx2x_init_block(bp, PRS_BLOCK, COMMON_STAGE);
6617         REG_WR(bp, PRS_REG_A_PRSU_20, 0xf);
6618 #ifndef BCM_CNIC
6619         /* set NIC mode */
6620         REG_WR(bp, PRS_REG_NIC_MODE, 1);
6621 #endif
6622         if (CHIP_IS_E1H(bp))
6623                 REG_WR(bp, PRS_REG_E1HOV_MODE, IS_E1HMF(bp));
6624
6625         bnx2x_init_block(bp, TSDM_BLOCK, COMMON_STAGE);
6626         bnx2x_init_block(bp, CSDM_BLOCK, COMMON_STAGE);
6627         bnx2x_init_block(bp, USDM_BLOCK, COMMON_STAGE);
6628         bnx2x_init_block(bp, XSDM_BLOCK, COMMON_STAGE);
6629
6630         bnx2x_init_fill(bp, TSEM_REG_FAST_MEMORY, 0, STORM_INTMEM_SIZE(bp));
6631         bnx2x_init_fill(bp, USEM_REG_FAST_MEMORY, 0, STORM_INTMEM_SIZE(bp));
6632         bnx2x_init_fill(bp, CSEM_REG_FAST_MEMORY, 0, STORM_INTMEM_SIZE(bp));
6633         bnx2x_init_fill(bp, XSEM_REG_FAST_MEMORY, 0, STORM_INTMEM_SIZE(bp));
6634
6635         bnx2x_init_block(bp, TSEM_BLOCK, COMMON_STAGE);
6636         bnx2x_init_block(bp, USEM_BLOCK, COMMON_STAGE);
6637         bnx2x_init_block(bp, CSEM_BLOCK, COMMON_STAGE);
6638         bnx2x_init_block(bp, XSEM_BLOCK, COMMON_STAGE);
6639
6640         /* sync semi rtc */
6641         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR,
6642                0x80000000);
6643         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET,
6644                0x80000000);
6645
6646         bnx2x_init_block(bp, UPB_BLOCK, COMMON_STAGE);
6647         bnx2x_init_block(bp, XPB_BLOCK, COMMON_STAGE);
6648         bnx2x_init_block(bp, PBF_BLOCK, COMMON_STAGE);
6649
6650         REG_WR(bp, SRC_REG_SOFT_RST, 1);
6651         for (i = SRC_REG_KEYRSS0_0; i <= SRC_REG_KEYRSS1_9; i += 4)
6652                 REG_WR(bp, i, random32());
6653         bnx2x_init_block(bp, SRCH_BLOCK, COMMON_STAGE);
6654 #ifdef BCM_CNIC
6655         REG_WR(bp, SRC_REG_KEYSEARCH_0, 0x63285672);
6656         REG_WR(bp, SRC_REG_KEYSEARCH_1, 0x24b8f2cc);
6657         REG_WR(bp, SRC_REG_KEYSEARCH_2, 0x223aef9b);
6658         REG_WR(bp, SRC_REG_KEYSEARCH_3, 0x26001e3a);
6659         REG_WR(bp, SRC_REG_KEYSEARCH_4, 0x7ae91116);
6660         REG_WR(bp, SRC_REG_KEYSEARCH_5, 0x5ce5230b);
6661         REG_WR(bp, SRC_REG_KEYSEARCH_6, 0x298d8adf);
6662         REG_WR(bp, SRC_REG_KEYSEARCH_7, 0x6eb0ff09);
6663         REG_WR(bp, SRC_REG_KEYSEARCH_8, 0x1830f82f);
6664         REG_WR(bp, SRC_REG_KEYSEARCH_9, 0x01e46be7);
6665 #endif
6666         REG_WR(bp, SRC_REG_SOFT_RST, 0);
6667
6668         if (sizeof(union cdu_context) != 1024)
6669                 /* we currently assume that a context is 1024 bytes */
6670                 dev_alert(&bp->pdev->dev, "please adjust the size "
6671                                           "of cdu_context(%ld)\n",
6672                          (long)sizeof(union cdu_context));
6673
6674         bnx2x_init_block(bp, CDU_BLOCK, COMMON_STAGE);
6675         val = (4 << 24) + (0 << 12) + 1024;
6676         REG_WR(bp, CDU_REG_CDU_GLOBAL_PARAMS, val);
6677
6678         bnx2x_init_block(bp, CFC_BLOCK, COMMON_STAGE);
6679         REG_WR(bp, CFC_REG_INIT_REG, 0x7FF);
6680         /* enable context validation interrupt from CFC */
6681         REG_WR(bp, CFC_REG_CFC_INT_MASK, 0);
6682
6683         /* set the thresholds to prevent CFC/CDU race */
6684         REG_WR(bp, CFC_REG_DEBUG0, 0x20020000);
6685
6686         bnx2x_init_block(bp, HC_BLOCK, COMMON_STAGE);
6687         bnx2x_init_block(bp, MISC_AEU_BLOCK, COMMON_STAGE);
6688
6689         bnx2x_init_block(bp, PXPCS_BLOCK, COMMON_STAGE);
6690         /* Reset PCIE errors for debug */
6691         REG_WR(bp, 0x2814, 0xffffffff);
6692         REG_WR(bp, 0x3820, 0xffffffff);
6693
6694         bnx2x_init_block(bp, EMAC0_BLOCK, COMMON_STAGE);
6695         bnx2x_init_block(bp, EMAC1_BLOCK, COMMON_STAGE);
6696         bnx2x_init_block(bp, DBU_BLOCK, COMMON_STAGE);
6697         bnx2x_init_block(bp, DBG_BLOCK, COMMON_STAGE);
6698
6699         bnx2x_init_block(bp, NIG_BLOCK, COMMON_STAGE);
6700         if (CHIP_IS_E1H(bp)) {
6701                 REG_WR(bp, NIG_REG_LLH_MF_MODE, IS_E1HMF(bp));
6702                 REG_WR(bp, NIG_REG_LLH_E1HOV_MODE, IS_E1HMF(bp));
6703         }
6704
6705         if (CHIP_REV_IS_SLOW(bp))
6706                 msleep(200);
6707
6708         /* finish CFC init */
6709         val = reg_poll(bp, CFC_REG_LL_INIT_DONE, 1, 100, 10);
6710         if (val != 1) {
6711                 BNX2X_ERR("CFC LL_INIT failed\n");
6712                 return -EBUSY;
6713         }
6714         val = reg_poll(bp, CFC_REG_AC_INIT_DONE, 1, 100, 10);
6715         if (val != 1) {
6716                 BNX2X_ERR("CFC AC_INIT failed\n");
6717                 return -EBUSY;
6718         }
6719         val = reg_poll(bp, CFC_REG_CAM_INIT_DONE, 1, 100, 10);
6720         if (val != 1) {
6721                 BNX2X_ERR("CFC CAM_INIT failed\n");
6722                 return -EBUSY;
6723         }
6724         REG_WR(bp, CFC_REG_DEBUG0, 0);
6725
6726         /* read NIG statistic
6727            to see if this is our first up since powerup */
6728         bnx2x_read_dmae(bp, NIG_REG_STAT2_BRB_OCTET, 2);
6729         val = *bnx2x_sp(bp, wb_data[0]);
6730
6731         /* do internal memory self test */
6732         if ((CHIP_IS_E1(bp)) && (val == 0) && bnx2x_int_mem_test(bp)) {
6733                 BNX2X_ERR("internal mem self test failed\n");
6734                 return -EBUSY;
6735         }
6736
6737         switch (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config)) {
6738         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8072:
6739         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8073:
6740         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8726:
6741         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727:
6742                 bp->port.need_hw_lock = 1;
6743                 break;
6744
6745         default:
6746                 break;
6747         }
6748
6749         bnx2x_setup_fan_failure_detection(bp);
6750
6751         /* clear PXP2 attentions */
6752         REG_RD(bp, PXP2_REG_PXP2_INT_STS_CLR_0);
6753
6754         enable_blocks_attention(bp);
6755         if (CHIP_PARITY_SUPPORTED(bp))
6756                 enable_blocks_parity(bp);
6757
6758         if (!BP_NOMCP(bp)) {
6759                 bnx2x_acquire_phy_lock(bp);
6760                 bnx2x_common_init_phy(bp, bp->common.shmem_base);
6761                 bnx2x_release_phy_lock(bp);
6762         } else
6763                 BNX2X_ERR("Bootcode is missing - can not initialize link\n");
6764
6765         return 0;
6766 }
6767
6768 static int bnx2x_init_port(struct bnx2x *bp)
6769 {
6770         int port = BP_PORT(bp);
6771         int init_stage = port ? PORT1_STAGE : PORT0_STAGE;
6772         u32 low, high;
6773         u32 val;
6774
6775         DP(BNX2X_MSG_MCP, "starting port init  port %d\n", port);
6776
6777         REG_WR(bp, NIG_REG_MASK_INTERRUPT_PORT0 + port*4, 0);
6778
6779         bnx2x_init_block(bp, PXP_BLOCK, init_stage);
6780         bnx2x_init_block(bp, PXP2_BLOCK, init_stage);
6781
6782         bnx2x_init_block(bp, TCM_BLOCK, init_stage);
6783         bnx2x_init_block(bp, UCM_BLOCK, init_stage);
6784         bnx2x_init_block(bp, CCM_BLOCK, init_stage);
6785         bnx2x_init_block(bp, XCM_BLOCK, init_stage);
6786
6787 #ifdef BCM_CNIC
6788         REG_WR(bp, QM_REG_CONNNUM_0 + port*4, 1024/16 - 1);
6789
6790         bnx2x_init_block(bp, TIMERS_BLOCK, init_stage);
6791         REG_WR(bp, TM_REG_LIN0_SCAN_TIME + port*4, 20);
6792         REG_WR(bp, TM_REG_LIN0_MAX_ACTIVE_CID + port*4, 31);
6793 #endif
6794
6795         bnx2x_init_block(bp, DQ_BLOCK, init_stage);
6796
6797         bnx2x_init_block(bp, BRB1_BLOCK, init_stage);
6798         if (CHIP_REV_IS_SLOW(bp) && !CHIP_IS_E1H(bp)) {
6799                 /* no pause for emulation and FPGA */
6800                 low = 0;
6801                 high = 513;
6802         } else {
6803                 if (IS_E1HMF(bp))
6804                         low = ((bp->flags & ONE_PORT_FLAG) ? 160 : 246);
6805                 else if (bp->dev->mtu > 4096) {
6806                         if (bp->flags & ONE_PORT_FLAG)
6807                                 low = 160;
6808                         else {
6809                                 val = bp->dev->mtu;
6810                                 /* (24*1024 + val*4)/256 */
6811                                 low = 96 + (val/64) + ((val % 64) ? 1 : 0);
6812                         }
6813                 } else
6814                         low = ((bp->flags & ONE_PORT_FLAG) ? 80 : 160);
6815                 high = low + 56;        /* 14*1024/256 */
6816         }
6817         REG_WR(bp, BRB1_REG_PAUSE_LOW_THRESHOLD_0 + port*4, low);
6818         REG_WR(bp, BRB1_REG_PAUSE_HIGH_THRESHOLD_0 + port*4, high);
6819
6820
6821         bnx2x_init_block(bp, PRS_BLOCK, init_stage);
6822
6823         bnx2x_init_block(bp, TSDM_BLOCK, init_stage);
6824         bnx2x_init_block(bp, CSDM_BLOCK, init_stage);
6825         bnx2x_init_block(bp, USDM_BLOCK, init_stage);
6826         bnx2x_init_block(bp, XSDM_BLOCK, init_stage);
6827
6828         bnx2x_init_block(bp, TSEM_BLOCK, init_stage);
6829         bnx2x_init_block(bp, USEM_BLOCK, init_stage);
6830         bnx2x_init_block(bp, CSEM_BLOCK, init_stage);
6831         bnx2x_init_block(bp, XSEM_BLOCK, init_stage);
6832
6833         bnx2x_init_block(bp, UPB_BLOCK, init_stage);
6834         bnx2x_init_block(bp, XPB_BLOCK, init_stage);
6835
6836         bnx2x_init_block(bp, PBF_BLOCK, init_stage);
6837
6838         /* configure PBF to work without PAUSE mtu 9000 */
6839         REG_WR(bp, PBF_REG_P0_PAUSE_ENABLE + port*4, 0);
6840
6841         /* update threshold */
6842         REG_WR(bp, PBF_REG_P0_ARB_THRSH + port*4, (9040/16));
6843         /* update init credit */
6844         REG_WR(bp, PBF_REG_P0_INIT_CRD + port*4, (9040/16) + 553 - 22);
6845
6846         /* probe changes */
6847         REG_WR(bp, PBF_REG_INIT_P0 + port*4, 1);
6848         msleep(5);
6849         REG_WR(bp, PBF_REG_INIT_P0 + port*4, 0);
6850
6851 #ifdef BCM_CNIC
6852         bnx2x_init_block(bp, SRCH_BLOCK, init_stage);
6853 #endif
6854         bnx2x_init_block(bp, CDU_BLOCK, init_stage);
6855         bnx2x_init_block(bp, CFC_BLOCK, init_stage);
6856
6857         if (CHIP_IS_E1(bp)) {
6858                 REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, 0);
6859                 REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, 0);
6860         }
6861         bnx2x_init_block(bp, HC_BLOCK, init_stage);
6862
6863         bnx2x_init_block(bp, MISC_AEU_BLOCK, init_stage);
6864         /* init aeu_mask_attn_func_0/1:
6865          *  - SF mode: bits 3-7 are masked. only bits 0-2 are in use
6866          *  - MF mode: bit 3 is masked. bits 0-2 are in use as in SF
6867          *             bits 4-7 are used for "per vn group attention" */
6868         REG_WR(bp, MISC_REG_AEU_MASK_ATTN_FUNC_0 + port*4,
6869                (IS_E1HMF(bp) ? 0xF7 : 0x7));
6870
6871         bnx2x_init_block(bp, PXPCS_BLOCK, init_stage);
6872         bnx2x_init_block(bp, EMAC0_BLOCK, init_stage);
6873         bnx2x_init_block(bp, EMAC1_BLOCK, init_stage);
6874         bnx2x_init_block(bp, DBU_BLOCK, init_stage);
6875         bnx2x_init_block(bp, DBG_BLOCK, init_stage);
6876
6877         bnx2x_init_block(bp, NIG_BLOCK, init_stage);
6878
6879         REG_WR(bp, NIG_REG_XGXS_SERDES0_MODE_SEL + port*4, 1);
6880
6881         if (CHIP_IS_E1H(bp)) {
6882                 /* 0x2 disable e1hov, 0x1 enable */
6883                 REG_WR(bp, NIG_REG_LLH0_BRB1_DRV_MASK_MF + port*4,
6884                        (IS_E1HMF(bp) ? 0x1 : 0x2));
6885
6886                 {
6887                         REG_WR(bp, NIG_REG_LLFC_ENABLE_0 + port*4, 0);
6888                         REG_WR(bp, NIG_REG_LLFC_OUT_EN_0 + port*4, 0);
6889                         REG_WR(bp, NIG_REG_PAUSE_ENABLE_0 + port*4, 1);
6890                 }
6891         }
6892
6893         bnx2x_init_block(bp, MCP_BLOCK, init_stage);
6894         bnx2x_init_block(bp, DMAE_BLOCK, init_stage);
6895
6896         switch (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config)) {
6897         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8726:
6898                 {
6899                 u32 swap_val, swap_override, aeu_gpio_mask, offset;
6900
6901                 bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_3,
6902                                MISC_REGISTERS_GPIO_INPUT_HI_Z, port);
6903
6904                 /* The GPIO should be swapped if the swap register is
6905                    set and active */
6906                 swap_val = REG_RD(bp, NIG_REG_PORT_SWAP);
6907                 swap_override = REG_RD(bp, NIG_REG_STRAP_OVERRIDE);
6908
6909                 /* Select function upon port-swap configuration */
6910                 if (port == 0) {
6911                         offset = MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0;
6912                         aeu_gpio_mask = (swap_val && swap_override) ?
6913                                 AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_1 :
6914                                 AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_0;
6915                 } else {
6916                         offset = MISC_REG_AEU_ENABLE1_FUNC_1_OUT_0;
6917                         aeu_gpio_mask = (swap_val && swap_override) ?
6918                                 AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_0 :
6919                                 AEU_INPUTS_ATTN_BITS_GPIO3_FUNCTION_1;
6920                 }
6921                 val = REG_RD(bp, offset);
6922                 /* add GPIO3 to group */
6923                 val |= aeu_gpio_mask;
6924                 REG_WR(bp, offset, val);
6925                 }
6926                 break;
6927
6928         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101:
6929         case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727:
6930                 /* add SPIO 5 to group 0 */
6931                 {
6932                 u32 reg_addr = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_0 :
6933                                        MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0);
6934                 val = REG_RD(bp, reg_addr);
6935                 val |= AEU_INPUTS_ATTN_BITS_SPIO5;
6936                 REG_WR(bp, reg_addr, val);
6937                 }
6938                 break;
6939
6940         default:
6941                 break;
6942         }
6943
6944         bnx2x__link_reset(bp);
6945
6946         return 0;
6947 }
6948
6949 #define ILT_PER_FUNC            (768/2)
6950 #define FUNC_ILT_BASE(func)     (func * ILT_PER_FUNC)
6951 /* the phys address is shifted right 12 bits and has an added
6952    1=valid bit added to the 53rd bit
6953    then since this is a wide register(TM)
6954    we split it into two 32 bit writes
6955  */
6956 #define ONCHIP_ADDR1(x)         ((u32)(((u64)x >> 12) & 0xFFFFFFFF))
6957 #define ONCHIP_ADDR2(x)         ((u32)((1 << 20) | ((u64)x >> 44)))
6958 #define PXP_ONE_ILT(x)          (((x) << 10) | x)
6959 #define PXP_ILT_RANGE(f, l)     (((l) << 10) | f)
6960
6961 #ifdef BCM_CNIC
6962 #define CNIC_ILT_LINES          127
6963 #define CNIC_CTX_PER_ILT        16
6964 #else
6965 #define CNIC_ILT_LINES          0
6966 #endif
6967
6968 static void bnx2x_ilt_wr(struct bnx2x *bp, u32 index, dma_addr_t addr)
6969 {
6970         int reg;
6971
6972         if (CHIP_IS_E1H(bp))
6973                 reg = PXP2_REG_RQ_ONCHIP_AT_B0 + index*8;
6974         else /* E1 */
6975                 reg = PXP2_REG_RQ_ONCHIP_AT + index*8;
6976
6977         bnx2x_wb_wr(bp, reg, ONCHIP_ADDR1(addr), ONCHIP_ADDR2(addr));
6978 }
6979
6980 static int bnx2x_init_func(struct bnx2x *bp)
6981 {
6982         int port = BP_PORT(bp);
6983         int func = BP_FUNC(bp);
6984         u32 addr, val;
6985         int i;
6986
6987         DP(BNX2X_MSG_MCP, "starting func init  func %d\n", func);
6988
6989         /* set MSI reconfigure capability */
6990         addr = (port ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0);
6991         val = REG_RD(bp, addr);
6992         val |= HC_CONFIG_0_REG_MSI_ATTN_EN_0;
6993         REG_WR(bp, addr, val);
6994
6995         i = FUNC_ILT_BASE(func);
6996
6997         bnx2x_ilt_wr(bp, i, bnx2x_sp_mapping(bp, context));
6998         if (CHIP_IS_E1H(bp)) {
6999                 REG_WR(bp, PXP2_REG_RQ_CDU_FIRST_ILT, i);
7000                 REG_WR(bp, PXP2_REG_RQ_CDU_LAST_ILT, i + CNIC_ILT_LINES);
7001         } else /* E1 */
7002                 REG_WR(bp, PXP2_REG_PSWRQ_CDU0_L2P + func*4,
7003                        PXP_ILT_RANGE(i, i + CNIC_ILT_LINES));
7004
7005 #ifdef BCM_CNIC
7006         i += 1 + CNIC_ILT_LINES;
7007         bnx2x_ilt_wr(bp, i, bp->timers_mapping);
7008         if (CHIP_IS_E1(bp))
7009                 REG_WR(bp, PXP2_REG_PSWRQ_TM0_L2P + func*4, PXP_ONE_ILT(i));
7010         else {
7011                 REG_WR(bp, PXP2_REG_RQ_TM_FIRST_ILT, i);
7012                 REG_WR(bp, PXP2_REG_RQ_TM_LAST_ILT, i);
7013         }
7014
7015         i++;
7016         bnx2x_ilt_wr(bp, i, bp->qm_mapping);
7017         if (CHIP_IS_E1(bp))
7018                 REG_WR(bp, PXP2_REG_PSWRQ_QM0_L2P + func*4, PXP_ONE_ILT(i));
7019         else {
7020                 REG_WR(bp, PXP2_REG_RQ_QM_FIRST_ILT, i);
7021                 REG_WR(bp, PXP2_REG_RQ_QM_LAST_ILT, i);
7022         }
7023
7024         i++;
7025         bnx2x_ilt_wr(bp, i, bp->t1_mapping);
7026         if (CHIP_IS_E1(bp))
7027                 REG_WR(bp, PXP2_REG_PSWRQ_SRC0_L2P + func*4, PXP_ONE_ILT(i));
7028         else {
7029                 REG_WR(bp, PXP2_REG_RQ_SRC_FIRST_ILT, i);
7030                 REG_WR(bp, PXP2_REG_RQ_SRC_LAST_ILT, i);
7031         }
7032
7033         /* tell the searcher where the T2 table is */
7034         REG_WR(bp, SRC_REG_COUNTFREE0 + port*4, 16*1024/64);
7035
7036         bnx2x_wb_wr(bp, SRC_REG_FIRSTFREE0 + port*16,
7037                     U64_LO(bp->t2_mapping), U64_HI(bp->t2_mapping));
7038
7039         bnx2x_wb_wr(bp, SRC_REG_LASTFREE0 + port*16,
7040                     U64_LO((u64)bp->t2_mapping + 16*1024 - 64),
7041                     U64_HI((u64)bp->t2_mapping + 16*1024 - 64));
7042
7043         REG_WR(bp, SRC_REG_NUMBER_HASH_BITS0 + port*4, 10);
7044 #endif
7045
7046         if (CHIP_IS_E1H(bp)) {
7047                 bnx2x_init_block(bp, MISC_BLOCK, FUNC0_STAGE + func);
7048                 bnx2x_init_block(bp, TCM_BLOCK, FUNC0_STAGE + func);
7049                 bnx2x_init_block(bp, UCM_BLOCK, FUNC0_STAGE + func);
7050                 bnx2x_init_block(bp, CCM_BLOCK, FUNC0_STAGE + func);
7051                 bnx2x_init_block(bp, XCM_BLOCK, FUNC0_STAGE + func);
7052                 bnx2x_init_block(bp, TSEM_BLOCK, FUNC0_STAGE + func);
7053                 bnx2x_init_block(bp, USEM_BLOCK, FUNC0_STAGE + func);
7054                 bnx2x_init_block(bp, CSEM_BLOCK, FUNC0_STAGE + func);
7055                 bnx2x_init_block(bp, XSEM_BLOCK, FUNC0_STAGE + func);
7056
7057                 REG_WR(bp, NIG_REG_LLH0_FUNC_EN + port*8, 1);
7058                 REG_WR(bp, NIG_REG_LLH0_FUNC_VLAN_ID + port*8, bp->e1hov);
7059         }
7060
7061         /* HC init per function */
7062         if (CHIP_IS_E1H(bp)) {
7063                 REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_12 + func*4, 0);
7064
7065                 REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, 0);
7066                 REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, 0);
7067         }
7068         bnx2x_init_block(bp, HC_BLOCK, FUNC0_STAGE + func);
7069
7070         /* Reset PCIE errors for debug */
7071         REG_WR(bp, 0x2114, 0xffffffff);
7072         REG_WR(bp, 0x2120, 0xffffffff);
7073
7074         return 0;
7075 }
7076
7077 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
7078 {
7079         int i, rc = 0;
7080
7081         DP(BNX2X_MSG_MCP, "function %d  load_code %x\n",
7082            BP_FUNC(bp), load_code);
7083
7084         bp->dmae_ready = 0;
7085         mutex_init(&bp->dmae_mutex);
7086         rc = bnx2x_gunzip_init(bp);
7087         if (rc)
7088                 return rc;
7089
7090         switch (load_code) {
7091         case FW_MSG_CODE_DRV_LOAD_COMMON:
7092                 rc = bnx2x_init_common(bp);
7093                 if (rc)
7094                         goto init_hw_err;
7095                 /* no break */
7096
7097         case FW_MSG_CODE_DRV_LOAD_PORT:
7098                 bp->dmae_ready = 1;
7099                 rc = bnx2x_init_port(bp);
7100                 if (rc)
7101                         goto init_hw_err;
7102                 /* no break */
7103
7104         case FW_MSG_CODE_DRV_LOAD_FUNCTION:
7105                 bp->dmae_ready = 1;
7106                 rc = bnx2x_init_func(bp);
7107                 if (rc)
7108                         goto init_hw_err;
7109                 break;
7110
7111         default:
7112                 BNX2X_ERR("Unknown load_code (0x%x) from MCP\n", load_code);
7113                 break;
7114         }
7115
7116         if (!BP_NOMCP(bp)) {
7117                 int func = BP_FUNC(bp);
7118
7119                 bp->fw_drv_pulse_wr_seq =
7120                                 (SHMEM_RD(bp, func_mb[func].drv_pulse_mb) &
7121                                  DRV_PULSE_SEQ_MASK);
7122                 DP(BNX2X_MSG_MCP, "drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
7123         }
7124
7125         /* this needs to be done before gunzip end */
7126         bnx2x_zero_def_sb(bp);
7127         for_each_queue(bp, i)
7128                 bnx2x_zero_sb(bp, BP_L_ID(bp) + i);
7129 #ifdef BCM_CNIC
7130         bnx2x_zero_sb(bp, BP_L_ID(bp) + i);
7131 #endif
7132
7133 init_hw_err:
7134         bnx2x_gunzip_end(bp);
7135
7136         return rc;
7137 }
7138
7139 static void bnx2x_free_mem(struct bnx2x *bp)
7140 {
7141
7142 #define BNX2X_PCI_FREE(x, y, size) \
7143         do { \
7144                 if (x) { \
7145                         dma_free_coherent(&bp->pdev->dev, size, x, y); \
7146                         x = NULL; \
7147                         y = 0; \
7148                 } \
7149         } while (0)
7150
7151 #define BNX2X_FREE(x) \
7152         do { \
7153                 if (x) { \
7154                         vfree(x); \
7155                         x = NULL; \
7156                 } \
7157         } while (0)
7158
7159         int i;
7160
7161         /* fastpath */
7162         /* Common */
7163         for_each_queue(bp, i) {
7164
7165                 /* status blocks */
7166                 BNX2X_PCI_FREE(bnx2x_fp(bp, i, status_blk),
7167                                bnx2x_fp(bp, i, status_blk_mapping),
7168                                sizeof(struct host_status_block));
7169         }
7170         /* Rx */
7171         for_each_queue(bp, i) {
7172
7173                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
7174                 BNX2X_FREE(bnx2x_fp(bp, i, rx_buf_ring));
7175                 BNX2X_PCI_FREE(bnx2x_fp(bp, i, rx_desc_ring),
7176                                bnx2x_fp(bp, i, rx_desc_mapping),
7177                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
7178
7179                 BNX2X_PCI_FREE(bnx2x_fp(bp, i, rx_comp_ring),
7180                                bnx2x_fp(bp, i, rx_comp_mapping),
7181                                sizeof(struct eth_fast_path_rx_cqe) *
7182                                NUM_RCQ_BD);
7183
7184                 /* SGE ring */
7185                 BNX2X_FREE(bnx2x_fp(bp, i, rx_page_ring));
7186                 BNX2X_PCI_FREE(bnx2x_fp(bp, i, rx_sge_ring),
7187                                bnx2x_fp(bp, i, rx_sge_mapping),
7188                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
7189         }
7190         /* Tx */
7191         for_each_queue(bp, i) {
7192
7193                 /* fastpath tx rings: tx_buf tx_desc */
7194                 BNX2X_FREE(bnx2x_fp(bp, i, tx_buf_ring));
7195                 BNX2X_PCI_FREE(bnx2x_fp(bp, i, tx_desc_ring),
7196                                bnx2x_fp(bp, i, tx_desc_mapping),
7197                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
7198         }
7199         /* end of fastpath */
7200
7201         BNX2X_PCI_FREE(bp->def_status_blk, bp->def_status_blk_mapping,
7202                        sizeof(struct host_def_status_block));
7203
7204         BNX2X_PCI_FREE(bp->slowpath, bp->slowpath_mapping,
7205                        sizeof(struct bnx2x_slowpath));
7206
7207 #ifdef BCM_CNIC
7208         BNX2X_PCI_FREE(bp->t1, bp->t1_mapping, 64*1024);
7209         BNX2X_PCI_FREE(bp->t2, bp->t2_mapping, 16*1024);
7210         BNX2X_PCI_FREE(bp->timers, bp->timers_mapping, 8*1024);
7211         BNX2X_PCI_FREE(bp->qm, bp->qm_mapping, 128*1024);
7212         BNX2X_PCI_FREE(bp->cnic_sb, bp->cnic_sb_mapping,
7213                        sizeof(struct host_status_block));
7214 #endif
7215         BNX2X_PCI_FREE(bp->spq, bp->spq_mapping, BCM_PAGE_SIZE);
7216
7217 #undef BNX2X_PCI_FREE
7218 #undef BNX2X_KFREE
7219 }
7220
7221 static int bnx2x_alloc_mem(struct bnx2x *bp)
7222 {
7223
7224 #define BNX2X_PCI_ALLOC(x, y, size) \
7225         do { \
7226                 x = dma_alloc_coherent(&bp->pdev->dev, size, y, GFP_KERNEL); \
7227                 if (x == NULL) \
7228                         goto alloc_mem_err; \
7229                 memset(x, 0, size); \
7230         } while (0)
7231
7232 #define BNX2X_ALLOC(x, size) \
7233         do { \
7234                 x = vmalloc(size); \
7235                 if (x == NULL) \
7236                         goto alloc_mem_err; \
7237                 memset(x, 0, size); \
7238         } while (0)
7239
7240         int i;
7241
7242         /* fastpath */
7243         /* Common */
7244         for_each_queue(bp, i) {
7245                 bnx2x_fp(bp, i, bp) = bp;
7246
7247                 /* status blocks */
7248                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, i, status_blk),
7249                                 &bnx2x_fp(bp, i, status_blk_mapping),
7250                                 sizeof(struct host_status_block));
7251         }
7252         /* Rx */
7253         for_each_queue(bp, i) {
7254
7255                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
7256                 BNX2X_ALLOC(bnx2x_fp(bp, i, rx_buf_ring),
7257                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
7258                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, i, rx_desc_ring),
7259                                 &bnx2x_fp(bp, i, rx_desc_mapping),
7260                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
7261
7262                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, i, rx_comp_ring),
7263                                 &bnx2x_fp(bp, i, rx_comp_mapping),
7264                                 sizeof(struct eth_fast_path_rx_cqe) *
7265                                 NUM_RCQ_BD);
7266
7267                 /* SGE ring */
7268                 BNX2X_ALLOC(bnx2x_fp(bp, i, rx_page_ring),
7269                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
7270                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, i, rx_sge_ring),
7271                                 &bnx2x_fp(bp, i, rx_sge_mapping),
7272                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
7273         }
7274         /* Tx */
7275         for_each_queue(bp, i) {
7276
7277                 /* fastpath tx rings: tx_buf tx_desc */
7278                 BNX2X_ALLOC(bnx2x_fp(bp, i, tx_buf_ring),
7279                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
7280                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, i, tx_desc_ring),
7281                                 &bnx2x_fp(bp, i, tx_desc_mapping),
7282                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
7283         }
7284         /* end of fastpath */
7285
7286         BNX2X_PCI_ALLOC(bp->def_status_blk, &bp->def_status_blk_mapping,
7287                         sizeof(struct host_def_status_block));
7288
7289         BNX2X_PCI_ALLOC(bp->slowpath, &bp->slowpath_mapping,
7290                         sizeof(struct bnx2x_slowpath));
7291
7292 #ifdef BCM_CNIC
7293         BNX2X_PCI_ALLOC(bp->t1, &bp->t1_mapping, 64*1024);
7294
7295         /* allocate searcher T2 table
7296            we allocate 1/4 of alloc num for T2
7297           (which is not entered into the ILT) */
7298         BNX2X_PCI_ALLOC(bp->t2, &bp->t2_mapping, 16*1024);
7299
7300         /* Initialize T2 (for 1024 connections) */
7301         for (i = 0; i < 16*1024; i += 64)
7302                 *(u64 *)((char *)bp->t2 + i + 56) = bp->t2_mapping + i + 64;
7303
7304         /* Timer block array (8*MAX_CONN) phys uncached for now 1024 conns */
7305         BNX2X_PCI_ALLOC(bp->timers, &bp->timers_mapping, 8*1024);
7306
7307         /* QM queues (128*MAX_CONN) */
7308         BNX2X_PCI_ALLOC(bp->qm, &bp->qm_mapping, 128*1024);
7309
7310         BNX2X_PCI_ALLOC(bp->cnic_sb, &bp->cnic_sb_mapping,
7311                         sizeof(struct host_status_block));
7312 #endif
7313
7314         /* Slow path ring */
7315         BNX2X_PCI_ALLOC(bp->spq, &bp->spq_mapping, BCM_PAGE_SIZE);
7316
7317         return 0;
7318
7319 alloc_mem_err:
7320         bnx2x_free_mem(bp);
7321         return -ENOMEM;
7322
7323 #undef BNX2X_PCI_ALLOC
7324 #undef BNX2X_ALLOC
7325 }
7326
7327 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
7328 {
7329         int i;
7330
7331         for_each_queue(bp, i) {
7332                 struct bnx2x_fastpath *fp = &bp->fp[i];
7333
7334                 u16 bd_cons = fp->tx_bd_cons;
7335                 u16 sw_prod = fp->tx_pkt_prod;
7336                 u16 sw_cons = fp->tx_pkt_cons;
7337
7338                 while (sw_cons != sw_prod) {
7339                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
7340                         sw_cons++;
7341                 }
7342         }
7343 }
7344
7345 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
7346 {
7347         int i, j;
7348
7349         for_each_queue(bp, j) {
7350                 struct bnx2x_fastpath *fp = &bp->fp[j];
7351
7352                 for (i = 0; i < NUM_RX_BD; i++) {
7353                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
7354                         struct sk_buff *skb = rx_buf->skb;
7355
7356                         if (skb == NULL)
7357                                 continue;
7358
7359                         dma_unmap_single(&bp->pdev->dev,
7360                                          dma_unmap_addr(rx_buf, mapping),
7361                                          bp->rx_buf_size, DMA_FROM_DEVICE);
7362
7363                         rx_buf->skb = NULL;
7364                         dev_kfree_skb(skb);
7365                 }
7366                 if (!fp->disable_tpa)
7367                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
7368                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
7369                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
7370         }
7371 }
7372
7373 static void bnx2x_free_skbs(struct bnx2x *bp)
7374 {
7375         bnx2x_free_tx_skbs(bp);
7376         bnx2x_free_rx_skbs(bp);
7377 }
7378
7379 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
7380 {
7381         int i, offset = 1;
7382
7383         free_irq(bp->msix_table[0].vector, bp->dev);
7384         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
7385            bp->msix_table[0].vector);
7386
7387 #ifdef BCM_CNIC
7388         offset++;
7389 #endif
7390         for_each_queue(bp, i) {
7391                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
7392                    "state %x\n", i, bp->msix_table[i + offset].vector,
7393                    bnx2x_fp(bp, i, state));
7394
7395                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
7396         }
7397 }
7398
7399 static void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
7400 {
7401         if (bp->flags & USING_MSIX_FLAG) {
7402                 if (!disable_only)
7403                         bnx2x_free_msix_irqs(bp);
7404                 pci_disable_msix(bp->pdev);
7405                 bp->flags &= ~USING_MSIX_FLAG;
7406
7407         } else if (bp->flags & USING_MSI_FLAG) {
7408                 if (!disable_only)
7409                         free_irq(bp->pdev->irq, bp->dev);
7410                 pci_disable_msi(bp->pdev);
7411                 bp->flags &= ~USING_MSI_FLAG;
7412
7413         } else if (!disable_only)
7414                 free_irq(bp->pdev->irq, bp->dev);
7415 }
7416
7417 static int bnx2x_enable_msix(struct bnx2x *bp)
7418 {
7419         int i, rc, offset = 1;
7420         int igu_vec = 0;
7421
7422         bp->msix_table[0].entry = igu_vec;
7423         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
7424
7425 #ifdef BCM_CNIC
7426         igu_vec = BP_L_ID(bp) + offset;
7427         bp->msix_table[1].entry = igu_vec;
7428         DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
7429         offset++;
7430 #endif
7431         for_each_queue(bp, i) {
7432                 igu_vec = BP_L_ID(bp) + offset + i;
7433                 bp->msix_table[i + offset].entry = igu_vec;
7434                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
7435                    "(fastpath #%u)\n", i + offset, igu_vec, i);
7436         }
7437
7438         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
7439                              BNX2X_NUM_QUEUES(bp) + offset);
7440
7441         /*
7442          * reconfigure number of tx/rx queues according to available
7443          * MSI-X vectors
7444          */
7445         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
7446                 /* vectors available for FP */
7447                 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
7448
7449                 DP(NETIF_MSG_IFUP,
7450                    "Trying to use less MSI-X vectors: %d\n", rc);
7451
7452                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
7453
7454                 if (rc) {
7455                         DP(NETIF_MSG_IFUP,
7456                            "MSI-X is not attainable  rc %d\n", rc);
7457                         return rc;
7458                 }
7459
7460                 bp->num_queues = min(bp->num_queues, fp_vec);
7461
7462                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
7463                                   bp->num_queues);
7464         } else if (rc) {
7465                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
7466                 return rc;
7467         }
7468
7469         bp->flags |= USING_MSIX_FLAG;
7470
7471         return 0;
7472 }
7473
7474 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
7475 {
7476         int i, rc, offset = 1;
7477
7478         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
7479                          bp->dev->name, bp->dev);
7480         if (rc) {
7481                 BNX2X_ERR("request sp irq failed\n");
7482                 return -EBUSY;
7483         }
7484
7485 #ifdef BCM_CNIC
7486         offset++;
7487 #endif
7488         for_each_queue(bp, i) {
7489                 struct bnx2x_fastpath *fp = &bp->fp[i];
7490                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
7491                          bp->dev->name, i);
7492
7493                 rc = request_irq(bp->msix_table[i + offset].vector,
7494                                  bnx2x_msix_fp_int, 0, fp->name, fp);
7495                 if (rc) {
7496                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
7497                         bnx2x_free_msix_irqs(bp);
7498                         return -EBUSY;
7499                 }
7500
7501                 fp->state = BNX2X_FP_STATE_IRQ;
7502         }
7503
7504         i = BNX2X_NUM_QUEUES(bp);
7505         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
7506                " ... fp[%d] %d\n",
7507                bp->msix_table[0].vector,
7508                0, bp->msix_table[offset].vector,
7509                i - 1, bp->msix_table[offset + i - 1].vector);
7510
7511         return 0;
7512 }
7513
7514 static int bnx2x_enable_msi(struct bnx2x *bp)
7515 {
7516         int rc;
7517
7518         rc = pci_enable_msi(bp->pdev);
7519         if (rc) {
7520                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
7521                 return -1;
7522         }
7523         bp->flags |= USING_MSI_FLAG;
7524
7525         return 0;
7526 }
7527
7528 static int bnx2x_req_irq(struct bnx2x *bp)
7529 {
7530         unsigned long flags;
7531         int rc;
7532
7533         if (bp->flags & USING_MSI_FLAG)
7534                 flags = 0;
7535         else
7536                 flags = IRQF_SHARED;
7537
7538         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
7539                          bp->dev->name, bp->dev);
7540         if (!rc)
7541                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
7542
7543         return rc;
7544 }
7545
7546 static void bnx2x_napi_enable(struct bnx2x *bp)
7547 {
7548         int i;
7549
7550         for_each_queue(bp, i)
7551                 napi_enable(&bnx2x_fp(bp, i, napi));
7552 }
7553
7554 static void bnx2x_napi_disable(struct bnx2x *bp)
7555 {
7556         int i;
7557
7558         for_each_queue(bp, i)
7559                 napi_disable(&bnx2x_fp(bp, i, napi));
7560 }
7561
7562 static void bnx2x_netif_start(struct bnx2x *bp)
7563 {
7564         int intr_sem;
7565
7566         intr_sem = atomic_dec_and_test(&bp->intr_sem);
7567         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
7568
7569         if (intr_sem) {
7570                 if (netif_running(bp->dev)) {
7571                         bnx2x_napi_enable(bp);
7572                         bnx2x_int_enable(bp);
7573                         if (bp->state == BNX2X_STATE_OPEN)
7574                                 netif_tx_wake_all_queues(bp->dev);
7575                 }
7576         }
7577 }
7578
7579 static void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
7580 {
7581         bnx2x_int_disable_sync(bp, disable_hw);
7582         bnx2x_napi_disable(bp);
7583         netif_tx_disable(bp->dev);
7584 }
7585
7586 /*
7587  * Init service functions
7588  */
7589
7590 /**
7591  * Sets a MAC in a CAM for a few L2 Clients for E1 chip
7592  *
7593  * @param bp driver descriptor
7594  * @param set set or clear an entry (1 or 0)
7595  * @param mac pointer to a buffer containing a MAC
7596  * @param cl_bit_vec bit vector of clients to register a MAC for
7597  * @param cam_offset offset in a CAM to use
7598  * @param with_bcast set broadcast MAC as well
7599  */
7600 static void bnx2x_set_mac_addr_e1_gen(struct bnx2x *bp, int set, u8 *mac,
7601                                       u32 cl_bit_vec, u8 cam_offset,
7602                                       u8 with_bcast)
7603 {
7604         struct mac_configuration_cmd *config = bnx2x_sp(bp, mac_config);
7605         int port = BP_PORT(bp);
7606
7607         /* CAM allocation
7608          * unicasts 0-31:port0 32-63:port1
7609          * multicast 64-127:port0 128-191:port1
7610          */
7611         config->hdr.length = 1 + (with_bcast ? 1 : 0);
7612         config->hdr.offset = cam_offset;
7613         config->hdr.client_id = 0xff;
7614         config->hdr.reserved1 = 0;
7615
7616         /* primary MAC */
7617         config->config_table[0].cam_entry.msb_mac_addr =
7618                                         swab16(*(u16 *)&mac[0]);
7619         config->config_table[0].cam_entry.middle_mac_addr =
7620                                         swab16(*(u16 *)&mac[2]);
7621         config->config_table[0].cam_entry.lsb_mac_addr =
7622                                         swab16(*(u16 *)&mac[4]);
7623         config->config_table[0].cam_entry.flags = cpu_to_le16(port);
7624         if (set)
7625                 config->config_table[0].target_table_entry.flags = 0;
7626         else
7627                 CAM_INVALIDATE(config->config_table[0]);
7628         config->config_table[0].target_table_entry.clients_bit_vector =
7629                                                 cpu_to_le32(cl_bit_vec);
7630         config->config_table[0].target_table_entry.vlan_id = 0;
7631
7632         DP(NETIF_MSG_IFUP, "%s MAC (%04x:%04x:%04x)\n",
7633            (set ? "setting" : "clearing"),
7634            config->config_table[0].cam_entry.msb_mac_addr,
7635            config->config_table[0].cam_entry.middle_mac_addr,
7636            config->config_table[0].cam_entry.lsb_mac_addr);
7637
7638         /* broadcast */
7639         if (with_bcast) {
7640                 config->config_table[1].cam_entry.msb_mac_addr =
7641                         cpu_to_le16(0xffff);
7642                 config->config_table[1].cam_entry.middle_mac_addr =
7643                         cpu_to_le16(0xffff);
7644                 config->config_table[1].cam_entry.lsb_mac_addr =
7645                         cpu_to_le16(0xffff);
7646                 config->config_table[1].cam_entry.flags = cpu_to_le16(port);
7647                 if (set)
7648                         config->config_table[1].target_table_entry.flags =
7649                                         TSTORM_CAM_TARGET_TABLE_ENTRY_BROADCAST;
7650                 else
7651                         CAM_INVALIDATE(config->config_table[1]);
7652                 config->config_table[1].target_table_entry.clients_bit_vector =
7653                                                         cpu_to_le32(cl_bit_vec);
7654                 config->config_table[1].target_table_entry.vlan_id = 0;
7655         }
7656
7657         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_SET_MAC, 0,
7658                       U64_HI(bnx2x_sp_mapping(bp, mac_config)),
7659                       U64_LO(bnx2x_sp_mapping(bp, mac_config)), 0);
7660 }
7661
7662 /**
7663  * Sets a MAC in a CAM for a few L2 Clients for E1H chip
7664  *
7665  * @param bp driver descriptor
7666  * @param set set or clear an entry (1 or 0)
7667  * @param mac pointer to a buffer containing a MAC
7668  * @param cl_bit_vec bit vector of clients to register a MAC for
7669  * @param cam_offset offset in a CAM to use
7670  */
7671 static void bnx2x_set_mac_addr_e1h_gen(struct bnx2x *bp, int set, u8 *mac,
7672                                        u32 cl_bit_vec, u8 cam_offset)
7673 {
7674         struct mac_configuration_cmd_e1h *config =
7675                 (struct mac_configuration_cmd_e1h *)bnx2x_sp(bp, mac_config);
7676
7677         config->hdr.length = 1;
7678         config->hdr.offset = cam_offset;
7679         config->hdr.client_id = 0xff;
7680         config->hdr.reserved1 = 0;
7681
7682         /* primary MAC */
7683         config->config_table[0].msb_mac_addr =
7684                                         swab16(*(u16 *)&mac[0]);
7685         config->config_table[0].middle_mac_addr =
7686                                         swab16(*(u16 *)&mac[2]);
7687         config->config_table[0].lsb_mac_addr =
7688                                         swab16(*(u16 *)&mac[4]);
7689         config->config_table[0].clients_bit_vector =
7690                                         cpu_to_le32(cl_bit_vec);
7691         config->config_table[0].vlan_id = 0;
7692         config->config_table[0].e1hov_id = cpu_to_le16(bp->e1hov);
7693         if (set)
7694                 config->config_table[0].flags = BP_PORT(bp);
7695         else
7696                 config->config_table[0].flags =
7697                                 MAC_CONFIGURATION_ENTRY_E1H_ACTION_TYPE;
7698
7699         DP(NETIF_MSG_IFUP, "%s MAC (%04x:%04x:%04x)  E1HOV %d  CLID mask %d\n",
7700            (set ? "setting" : "clearing"),
7701            config->config_table[0].msb_mac_addr,
7702            config->config_table[0].middle_mac_addr,
7703            config->config_table[0].lsb_mac_addr, bp->e1hov, cl_bit_vec);
7704
7705         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_SET_MAC, 0,
7706                       U64_HI(bnx2x_sp_mapping(bp, mac_config)),
7707                       U64_LO(bnx2x_sp_mapping(bp, mac_config)), 0);
7708 }
7709
7710 static int bnx2x_wait_ramrod(struct bnx2x *bp, int state, int idx,
7711                              int *state_p, int poll)
7712 {
7713         /* can take a while if any port is running */
7714         int cnt = 5000;
7715
7716         DP(NETIF_MSG_IFUP, "%s for state to become %x on IDX [%d]\n",
7717            poll ? "polling" : "waiting", state, idx);
7718
7719         might_sleep();
7720         while (cnt--) {
7721                 if (poll) {
7722                         bnx2x_rx_int(bp->fp, 10);
7723                         /* if index is different from 0
7724                          * the reply for some commands will
7725                          * be on the non default queue
7726                          */
7727                         if (idx)
7728                                 bnx2x_rx_int(&bp->fp[idx], 10);
7729                 }
7730
7731                 mb(); /* state is changed by bnx2x_sp_event() */
7732                 if (*state_p == state) {
7733 #ifdef BNX2X_STOP_ON_ERROR
7734                         DP(NETIF_MSG_IFUP, "exit  (cnt %d)\n", 5000 - cnt);
7735 #endif
7736                         return 0;
7737                 }
7738
7739                 msleep(1);
7740
7741                 if (bp->panic)
7742                         return -EIO;
7743         }
7744
7745         /* timeout! */
7746         BNX2X_ERR("timeout %s for state %x on IDX [%d]\n",
7747                   poll ? "polling" : "waiting", state, idx);
7748 #ifdef BNX2X_STOP_ON_ERROR
7749         bnx2x_panic();
7750 #endif
7751
7752         return -EBUSY;
7753 }
7754
7755 static void bnx2x_set_eth_mac_addr_e1h(struct bnx2x *bp, int set)
7756 {
7757         bp->set_mac_pending++;
7758         smp_wmb();
7759
7760         bnx2x_set_mac_addr_e1h_gen(bp, set, bp->dev->dev_addr,
7761                                    (1 << bp->fp->cl_id), BP_FUNC(bp));
7762
7763         /* Wait for a completion */
7764         bnx2x_wait_ramrod(bp, 0, 0, &bp->set_mac_pending, set ? 0 : 1);
7765 }
7766
7767 static void bnx2x_set_eth_mac_addr_e1(struct bnx2x *bp, int set)
7768 {
7769         bp->set_mac_pending++;
7770         smp_wmb();
7771
7772         bnx2x_set_mac_addr_e1_gen(bp, set, bp->dev->dev_addr,
7773                                   (1 << bp->fp->cl_id), (BP_PORT(bp) ? 32 : 0),
7774                                   1);
7775
7776         /* Wait for a completion */
7777         bnx2x_wait_ramrod(bp, 0, 0, &bp->set_mac_pending, set ? 0 : 1);
7778 }
7779
7780 #ifdef BCM_CNIC
7781 /**
7782  * Set iSCSI MAC(s) at the next enties in the CAM after the ETH
7783  * MAC(s). This function will wait until the ramdord completion
7784  * returns.
7785  *
7786  * @param bp driver handle
7787  * @param set set or clear the CAM entry
7788  *
7789  * @return 0 if cussess, -ENODEV if ramrod doesn't return.
7790  */
7791 static int bnx2x_set_iscsi_eth_mac_addr(struct bnx2x *bp, int set)
7792 {
7793         u32 cl_bit_vec = (1 << BCM_ISCSI_ETH_CL_ID);
7794
7795         bp->set_mac_pending++;
7796         smp_wmb();
7797
7798         /* Send a SET_MAC ramrod */
7799         if (CHIP_IS_E1(bp))
7800                 bnx2x_set_mac_addr_e1_gen(bp, set, bp->iscsi_mac,
7801                                   cl_bit_vec, (BP_PORT(bp) ? 32 : 0) + 2,
7802                                   1);
7803         else
7804                 /* CAM allocation for E1H
7805                 * unicasts: by func number
7806                 * multicast: 20+FUNC*20, 20 each
7807                 */
7808                 bnx2x_set_mac_addr_e1h_gen(bp, set, bp->iscsi_mac,
7809                                    cl_bit_vec, E1H_FUNC_MAX + BP_FUNC(bp));
7810
7811         /* Wait for a completion when setting */
7812         bnx2x_wait_ramrod(bp, 0, 0, &bp->set_mac_pending, set ? 0 : 1);
7813
7814         return 0;
7815 }
7816 #endif
7817
7818 static int bnx2x_setup_leading(struct bnx2x *bp)
7819 {
7820         int rc;
7821
7822         /* reset IGU state */
7823         bnx2x_ack_sb(bp, bp->fp[0].sb_id, CSTORM_ID, 0, IGU_INT_ENABLE, 0);
7824
7825         /* SETUP ramrod */
7826         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_PORT_SETUP, 0, 0, 0, 0);
7827
7828         /* Wait for completion */
7829         rc = bnx2x_wait_ramrod(bp, BNX2X_STATE_OPEN, 0, &(bp->state), 0);
7830
7831         return rc;
7832 }
7833
7834 static int bnx2x_setup_multi(struct bnx2x *bp, int index)
7835 {
7836         struct bnx2x_fastpath *fp = &bp->fp[index];
7837
7838         /* reset IGU state */
7839         bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID, 0, IGU_INT_ENABLE, 0);
7840
7841         /* SETUP ramrod */
7842         fp->state = BNX2X_FP_STATE_OPENING;
7843         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_CLIENT_SETUP, index, 0,
7844                       fp->cl_id, 0);
7845
7846         /* Wait for completion */
7847         return bnx2x_wait_ramrod(bp, BNX2X_FP_STATE_OPEN, index,
7848                                  &(fp->state), 0);
7849 }
7850
7851 static int bnx2x_poll(struct napi_struct *napi, int budget);
7852
7853 static void bnx2x_set_num_queues_msix(struct bnx2x *bp)
7854 {
7855
7856         switch (bp->multi_mode) {
7857         case ETH_RSS_MODE_DISABLED:
7858                 bp->num_queues = 1;
7859                 break;
7860
7861         case ETH_RSS_MODE_REGULAR:
7862                 if (num_queues)
7863                         bp->num_queues = min_t(u32, num_queues,
7864                                                   BNX2X_MAX_QUEUES(bp));
7865                 else
7866                         bp->num_queues = min_t(u32, num_online_cpus(),
7867                                                   BNX2X_MAX_QUEUES(bp));
7868                 break;
7869
7870
7871         default:
7872                 bp->num_queues = 1;
7873                 break;
7874         }
7875 }
7876
7877 static int bnx2x_set_num_queues(struct bnx2x *bp)
7878 {
7879         int rc = 0;
7880
7881         switch (bp->int_mode) {
7882         case INT_MODE_INTx:
7883         case INT_MODE_MSI:
7884                 bp->num_queues = 1;
7885                 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
7886                 break;
7887         default:
7888                 /* Set number of queues according to bp->multi_mode value */
7889                 bnx2x_set_num_queues_msix(bp);
7890
7891                 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
7892                    bp->num_queues);
7893
7894                 /* if we can't use MSI-X we only need one fp,
7895                  * so try to enable MSI-X with the requested number of fp's
7896                  * and fallback to MSI or legacy INTx with one fp
7897                  */
7898                 rc = bnx2x_enable_msix(bp);
7899                 if (rc)
7900                         /* failed to enable MSI-X */
7901                         bp->num_queues = 1;
7902                 break;
7903         }
7904         bp->dev->real_num_tx_queues = bp->num_queues;
7905         return rc;
7906 }
7907
7908 #ifdef BCM_CNIC
7909 static int bnx2x_cnic_notify(struct bnx2x *bp, int cmd);
7910 static void bnx2x_setup_cnic_irq_info(struct bnx2x *bp);
7911 #endif
7912
7913 /* must be called with rtnl_lock */
7914 static int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
7915 {
7916         u32 load_code;
7917         int i, rc;
7918
7919 #ifdef BNX2X_STOP_ON_ERROR
7920         if (unlikely(bp->panic))
7921                 return -EPERM;
7922 #endif
7923
7924         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
7925
7926         rc = bnx2x_set_num_queues(bp);
7927
7928         if (bnx2x_alloc_mem(bp)) {
7929                 bnx2x_free_irq(bp, true);
7930                 return -ENOMEM;
7931         }
7932
7933         for_each_queue(bp, i)
7934                 bnx2x_fp(bp, i, disable_tpa) =
7935                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
7936
7937         for_each_queue(bp, i)
7938                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
7939                                bnx2x_poll, 128);
7940
7941         bnx2x_napi_enable(bp);
7942
7943         if (bp->flags & USING_MSIX_FLAG) {
7944                 rc = bnx2x_req_msix_irqs(bp);
7945                 if (rc) {
7946                         bnx2x_free_irq(bp, true);
7947                         goto load_error1;
7948                 }
7949         } else {
7950                 /* Fall to INTx if failed to enable MSI-X due to lack of
7951                    memory (in bnx2x_set_num_queues()) */
7952                 if ((rc != -ENOMEM) && (int_mode != INT_MODE_INTx))
7953                         bnx2x_enable_msi(bp);
7954                 bnx2x_ack_int(bp);
7955                 rc = bnx2x_req_irq(bp);
7956                 if (rc) {
7957                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
7958                         bnx2x_free_irq(bp, true);
7959                         goto load_error1;
7960                 }
7961                 if (bp->flags & USING_MSI_FLAG) {
7962                         bp->dev->irq = bp->pdev->irq;
7963                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
7964                                     bp->pdev->irq);
7965                 }
7966         }
7967
7968         /* Send LOAD_REQUEST command to MCP
7969            Returns the type of LOAD command:
7970            if it is the first port to be initialized
7971            common blocks should be initialized, otherwise - not
7972         */
7973         if (!BP_NOMCP(bp)) {
7974                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ);
7975                 if (!load_code) {
7976                         BNX2X_ERR("MCP response failure, aborting\n");
7977                         rc = -EBUSY;
7978                         goto load_error2;
7979                 }
7980                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
7981                         rc = -EBUSY; /* other port in diagnostic mode */
7982                         goto load_error2;
7983                 }
7984
7985         } else {
7986                 int port = BP_PORT(bp);
7987
7988                 DP(NETIF_MSG_IFUP, "NO MCP - load counts      %d, %d, %d\n",
7989                    load_count[0], load_count[1], load_count[2]);
7990                 load_count[0]++;
7991                 load_count[1 + port]++;
7992                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts  %d, %d, %d\n",
7993                    load_count[0], load_count[1], load_count[2]);
7994                 if (load_count[0] == 1)
7995                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
7996                 else if (load_count[1 + port] == 1)
7997                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
7998                 else
7999                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
8000         }
8001
8002         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
8003             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
8004                 bp->port.pmf = 1;
8005         else
8006                 bp->port.pmf = 0;
8007         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
8008
8009         /* Initialize HW */
8010         rc = bnx2x_init_hw(bp, load_code);
8011         if (rc) {
8012                 BNX2X_ERR("HW init failed, aborting\n");
8013                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
8014                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
8015                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
8016                 goto load_error2;
8017         }
8018
8019         /* Setup NIC internals and enable interrupts */
8020         bnx2x_nic_init(bp, load_code);
8021
8022         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
8023             (bp->common.shmem2_base))
8024                 SHMEM2_WR(bp, dcc_support,
8025                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
8026                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
8027
8028         /* Send LOAD_DONE command to MCP */
8029         if (!BP_NOMCP(bp)) {
8030                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
8031                 if (!load_code) {
8032                         BNX2X_ERR("MCP response failure, aborting\n");
8033                         rc = -EBUSY;
8034                         goto load_error3;
8035                 }
8036         }
8037
8038         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
8039
8040         rc = bnx2x_setup_leading(bp);
8041         if (rc) {
8042                 BNX2X_ERR("Setup leading failed!\n");
8043 #ifndef BNX2X_STOP_ON_ERROR
8044                 goto load_error3;
8045 #else
8046                 bp->panic = 1;
8047                 return -EBUSY;
8048 #endif
8049         }
8050
8051         if (CHIP_IS_E1H(bp))
8052                 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
8053                         DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
8054                         bp->flags |= MF_FUNC_DIS;
8055                 }
8056
8057         if (bp->state == BNX2X_STATE_OPEN) {
8058 #ifdef BCM_CNIC
8059                 /* Enable Timer scan */
8060                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
8061 #endif
8062                 for_each_nondefault_queue(bp, i) {
8063                         rc = bnx2x_setup_multi(bp, i);
8064                         if (rc)
8065 #ifdef BCM_CNIC
8066                                 goto load_error4;
8067 #else
8068                                 goto load_error3;
8069 #endif
8070                 }
8071
8072                 if (CHIP_IS_E1(bp))
8073                         bnx2x_set_eth_mac_addr_e1(bp, 1);
8074                 else
8075                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
8076 #ifdef BCM_CNIC
8077                 /* Set iSCSI L2 MAC */
8078                 mutex_lock(&bp->cnic_mutex);
8079                 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
8080                         bnx2x_set_iscsi_eth_mac_addr(bp, 1);
8081                         bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
8082                         bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
8083                                       CNIC_SB_ID(bp));
8084                 }
8085                 mutex_unlock(&bp->cnic_mutex);
8086 #endif
8087         }
8088
8089         if (bp->port.pmf)
8090                 bnx2x_initial_phy_init(bp, load_mode);
8091
8092         /* Start fast path */
8093         switch (load_mode) {
8094         case LOAD_NORMAL:
8095                 if (bp->state == BNX2X_STATE_OPEN) {
8096                         /* Tx queue should be only reenabled */
8097                         netif_tx_wake_all_queues(bp->dev);
8098                 }
8099                 /* Initialize the receive filter. */
8100                 bnx2x_set_rx_mode(bp->dev);
8101                 break;
8102
8103         case LOAD_OPEN:
8104                 netif_tx_start_all_queues(bp->dev);
8105                 if (bp->state != BNX2X_STATE_OPEN)
8106                         netif_tx_disable(bp->dev);
8107                 /* Initialize the receive filter. */
8108                 bnx2x_set_rx_mode(bp->dev);
8109                 break;
8110
8111         case LOAD_DIAG:
8112                 /* Initialize the receive filter. */
8113                 bnx2x_set_rx_mode(bp->dev);
8114                 bp->state = BNX2X_STATE_DIAG;
8115                 break;
8116
8117         default:
8118                 break;
8119         }
8120
8121         if (!bp->port.pmf)
8122                 bnx2x__link_status_update(bp);
8123
8124         /* start the timer */
8125         mod_timer(&bp->timer, jiffies + bp->current_interval);
8126
8127 #ifdef BCM_CNIC
8128         bnx2x_setup_cnic_irq_info(bp);
8129         if (bp->state == BNX2X_STATE_OPEN)
8130                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
8131 #endif
8132         bnx2x_inc_load_cnt(bp);
8133
8134         return 0;
8135
8136 #ifdef BCM_CNIC
8137 load_error4:
8138         /* Disable Timer scan */
8139         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
8140 #endif
8141 load_error3:
8142         bnx2x_int_disable_sync(bp, 1);
8143         if (!BP_NOMCP(bp)) {
8144                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
8145                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
8146         }
8147         bp->port.pmf = 0;
8148         /* Free SKBs, SGEs, TPA pool and driver internals */
8149         bnx2x_free_skbs(bp);
8150         for_each_queue(bp, i)
8151                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
8152 load_error2:
8153         /* Release IRQs */
8154         bnx2x_free_irq(bp, false);
8155 load_error1:
8156         bnx2x_napi_disable(bp);
8157         for_each_queue(bp, i)
8158                 netif_napi_del(&bnx2x_fp(bp, i, napi));
8159         bnx2x_free_mem(bp);
8160
8161         return rc;
8162 }
8163
8164 static int bnx2x_stop_multi(struct bnx2x *bp, int index)
8165 {
8166         struct bnx2x_fastpath *fp = &bp->fp[index];
8167         int rc;
8168
8169         /* halt the connection */
8170         fp->state = BNX2X_FP_STATE_HALTING;
8171         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_HALT, index, 0, fp->cl_id, 0);
8172
8173         /* Wait for completion */
8174         rc = bnx2x_wait_ramrod(bp, BNX2X_FP_STATE_HALTED, index,
8175                                &(fp->state), 1);
8176         if (rc) /* timeout */
8177                 return rc;
8178
8179         /* delete cfc entry */
8180         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_CFC_DEL, index, 0, 0, 1);
8181
8182         /* Wait for completion */
8183         rc = bnx2x_wait_ramrod(bp, BNX2X_FP_STATE_CLOSED, index,
8184                                &(fp->state), 1);
8185         return rc;
8186 }
8187
8188 static int bnx2x_stop_leading(struct bnx2x *bp)
8189 {
8190         __le16 dsb_sp_prod_idx;
8191         /* if the other port is handling traffic,
8192            this can take a lot of time */
8193         int cnt = 500;
8194         int rc;
8195
8196         might_sleep();
8197
8198         /* Send HALT ramrod */
8199         bp->fp[0].state = BNX2X_FP_STATE_HALTING;
8200         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_HALT, 0, 0, bp->fp->cl_id, 0);
8201
8202         /* Wait for completion */
8203         rc = bnx2x_wait_ramrod(bp, BNX2X_FP_STATE_HALTED, 0,
8204                                &(bp->fp[0].state), 1);
8205         if (rc) /* timeout */
8206                 return rc;
8207
8208         dsb_sp_prod_idx = *bp->dsb_sp_prod;
8209
8210         /* Send PORT_DELETE ramrod */
8211         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_PORT_DEL, 0, 0, 0, 1);
8212
8213         /* Wait for completion to arrive on default status block
8214            we are going to reset the chip anyway
8215            so there is not much to do if this times out
8216          */
8217         while (dsb_sp_prod_idx == *bp->dsb_sp_prod) {
8218                 if (!cnt) {
8219                         DP(NETIF_MSG_IFDOWN, "timeout waiting for port del "
8220                            "dsb_sp_prod 0x%x != dsb_sp_prod_idx 0x%x\n",
8221                            *bp->dsb_sp_prod, dsb_sp_prod_idx);
8222 #ifdef BNX2X_STOP_ON_ERROR
8223                         bnx2x_panic();
8224 #endif
8225                         rc = -EBUSY;
8226                         break;
8227                 }
8228                 cnt--;
8229                 msleep(1);
8230                 rmb(); /* Refresh the dsb_sp_prod */
8231         }
8232         bp->state = BNX2X_STATE_CLOSING_WAIT4_UNLOAD;
8233         bp->fp[0].state = BNX2X_FP_STATE_CLOSED;
8234
8235         return rc;
8236 }
8237
8238 static void bnx2x_reset_func(struct bnx2x *bp)
8239 {
8240         int port = BP_PORT(bp);
8241         int func = BP_FUNC(bp);
8242         int base, i;
8243
8244         /* Configure IGU */
8245         REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, 0);
8246         REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, 0);
8247
8248 #ifdef BCM_CNIC
8249         /* Disable Timer scan */
8250         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
8251         /*
8252          * Wait for at least 10ms and up to 2 second for the timers scan to
8253          * complete
8254          */
8255         for (i = 0; i < 200; i++) {
8256                 msleep(10);
8257                 if (!REG_RD(bp, TM_REG_LIN0_SCAN_ON + port*4))
8258                         break;
8259         }
8260 #endif
8261         /* Clear ILT */
8262         base = FUNC_ILT_BASE(func);
8263         for (i = base; i < base + ILT_PER_FUNC; i++)
8264                 bnx2x_ilt_wr(bp, i, 0);
8265 }
8266
8267 static void bnx2x_reset_port(struct bnx2x *bp)
8268 {
8269         int port = BP_PORT(bp);
8270         u32 val;
8271
8272         REG_WR(bp, NIG_REG_MASK_INTERRUPT_PORT0 + port*4, 0);
8273
8274         /* Do not rcv packets to BRB */
8275         REG_WR(bp, NIG_REG_LLH0_BRB1_DRV_MASK + port*4, 0x0);
8276         /* Do not direct rcv packets that are not for MCP to the BRB */
8277         REG_WR(bp, (port ? NIG_REG_LLH1_BRB1_NOT_MCP :
8278                            NIG_REG_LLH0_BRB1_NOT_MCP), 0x0);
8279
8280         /* Configure AEU */
8281         REG_WR(bp, MISC_REG_AEU_MASK_ATTN_FUNC_0 + port*4, 0);
8282
8283         msleep(100);
8284         /* Check for BRB port occupancy */
8285         val = REG_RD(bp, BRB1_REG_PORT_NUM_OCC_BLOCKS_0 + port*4);
8286         if (val)
8287                 DP(NETIF_MSG_IFDOWN,
8288                    "BRB1 is not empty  %d blocks are occupied\n", val);
8289
8290         /* TODO: Close Doorbell port? */
8291 }
8292
8293 static void bnx2x_reset_chip(struct bnx2x *bp, u32 reset_code)
8294 {
8295         DP(BNX2X_MSG_MCP, "function %d  reset_code %x\n",
8296            BP_FUNC(bp), reset_code);
8297
8298         switch (reset_code) {
8299         case FW_MSG_CODE_DRV_UNLOAD_COMMON:
8300                 bnx2x_reset_port(bp);
8301                 bnx2x_reset_func(bp);
8302                 bnx2x_reset_common(bp);
8303                 break;
8304
8305         case FW_MSG_CODE_DRV_UNLOAD_PORT:
8306                 bnx2x_reset_port(bp);
8307                 bnx2x_reset_func(bp);
8308                 break;
8309
8310         case FW_MSG_CODE_DRV_UNLOAD_FUNCTION:
8311                 bnx2x_reset_func(bp);
8312                 break;
8313
8314         default:
8315                 BNX2X_ERR("Unknown reset_code (0x%x) from MCP\n", reset_code);
8316                 break;
8317         }
8318 }
8319
8320 static void bnx2x_chip_cleanup(struct bnx2x *bp, int unload_mode)
8321 {
8322         int port = BP_PORT(bp);
8323         u32 reset_code = 0;
8324         int i, cnt, rc;
8325
8326         /* Wait until tx fastpath tasks complete */
8327         for_each_queue(bp, i) {
8328                 struct bnx2x_fastpath *fp = &bp->fp[i];
8329
8330                 cnt = 1000;
8331                 while (bnx2x_has_tx_work_unload(fp)) {
8332
8333                         bnx2x_tx_int(fp);
8334                         if (!cnt) {
8335                                 BNX2X_ERR("timeout waiting for queue[%d]\n",
8336                                           i);
8337 #ifdef BNX2X_STOP_ON_ERROR
8338                                 bnx2x_panic();
8339                                 return -EBUSY;
8340 #else
8341                                 break;
8342 #endif
8343                         }
8344                         cnt--;
8345                         msleep(1);
8346                 }
8347         }
8348         /* Give HW time to discard old tx messages */
8349         msleep(1);
8350
8351         if (CHIP_IS_E1(bp)) {
8352                 struct mac_configuration_cmd *config =
8353                                                 bnx2x_sp(bp, mcast_config);
8354
8355                 bnx2x_set_eth_mac_addr_e1(bp, 0);
8356
8357                 for (i = 0; i < config->hdr.length; i++)
8358                         CAM_INVALIDATE(config->config_table[i]);
8359
8360                 config->hdr.length = i;
8361                 if (CHIP_REV_IS_SLOW(bp))
8362                         config->hdr.offset = BNX2X_MAX_EMUL_MULTI*(1 + port);
8363                 else
8364                         config->hdr.offset = BNX2X_MAX_MULTICAST*(1 + port);
8365                 config->hdr.client_id = bp->fp->cl_id;
8366                 config->hdr.reserved1 = 0;
8367
8368                 bp->set_mac_pending++;
8369                 smp_wmb();
8370
8371                 bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_SET_MAC, 0,
8372                               U64_HI(bnx2x_sp_mapping(bp, mcast_config)),
8373                               U64_LO(bnx2x_sp_mapping(bp, mcast_config)), 0);
8374
8375         } else { /* E1H */
8376                 REG_WR(bp, NIG_REG_LLH0_FUNC_EN + port*8, 0);
8377
8378                 bnx2x_set_eth_mac_addr_e1h(bp, 0);
8379
8380                 for (i = 0; i < MC_HASH_SIZE; i++)
8381                         REG_WR(bp, MC_HASH_OFFSET(bp, i), 0);
8382
8383                 REG_WR(bp, MISC_REG_E1HMF_MODE, 0);
8384         }
8385 #ifdef BCM_CNIC
8386         /* Clear iSCSI L2 MAC */
8387         mutex_lock(&bp->cnic_mutex);
8388         if (bp->cnic_flags & BNX2X_CNIC_FLAG_MAC_SET) {
8389                 bnx2x_set_iscsi_eth_mac_addr(bp, 0);
8390                 bp->cnic_flags &= ~BNX2X_CNIC_FLAG_MAC_SET;
8391         }
8392         mutex_unlock(&bp->cnic_mutex);
8393 #endif
8394
8395         if (unload_mode == UNLOAD_NORMAL)
8396                 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_DIS;
8397
8398         else if (bp->flags & NO_WOL_FLAG)
8399                 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP;
8400
8401         else if (bp->wol) {
8402                 u32 emac_base = port ? GRCBASE_EMAC1 : GRCBASE_EMAC0;
8403                 u8 *mac_addr = bp->dev->dev_addr;
8404                 u32 val;
8405                 /* The mac address is written to entries 1-4 to
8406                    preserve entry 0 which is used by the PMF */
8407                 u8 entry = (BP_E1HVN(bp) + 1)*8;
8408
8409                 val = (mac_addr[0] << 8) | mac_addr[1];
8410                 EMAC_WR(bp, EMAC_REG_EMAC_MAC_MATCH + entry, val);
8411
8412                 val = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
8413                       (mac_addr[4] << 8) | mac_addr[5];
8414                 EMAC_WR(bp, EMAC_REG_EMAC_MAC_MATCH + entry + 4, val);
8415
8416                 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_EN;
8417
8418         } else
8419                 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_DIS;
8420
8421         /* Close multi and leading connections
8422            Completions for ramrods are collected in a synchronous way */
8423         for_each_nondefault_queue(bp, i)
8424                 if (bnx2x_stop_multi(bp, i))
8425                         goto unload_error;
8426
8427         rc = bnx2x_stop_leading(bp);
8428         if (rc) {
8429                 BNX2X_ERR("Stop leading failed!\n");
8430 #ifdef BNX2X_STOP_ON_ERROR
8431                 return -EBUSY;
8432 #else
8433                 goto unload_error;
8434 #endif
8435         }
8436
8437 unload_error:
8438         if (!BP_NOMCP(bp))
8439                 reset_code = bnx2x_fw_command(bp, reset_code);
8440         else {
8441                 DP(NETIF_MSG_IFDOWN, "NO MCP - load counts      %d, %d, %d\n",
8442                    load_count[0], load_count[1], load_count[2]);
8443                 load_count[0]--;
8444                 load_count[1 + port]--;
8445                 DP(NETIF_MSG_IFDOWN, "NO MCP - new load counts  %d, %d, %d\n",
8446                    load_count[0], load_count[1], load_count[2]);
8447                 if (load_count[0] == 0)
8448                         reset_code = FW_MSG_CODE_DRV_UNLOAD_COMMON;
8449                 else if (load_count[1 + port] == 0)
8450                         reset_code = FW_MSG_CODE_DRV_UNLOAD_PORT;
8451                 else
8452                         reset_code = FW_MSG_CODE_DRV_UNLOAD_FUNCTION;
8453         }
8454
8455         if ((reset_code == FW_MSG_CODE_DRV_UNLOAD_COMMON) ||
8456             (reset_code == FW_MSG_CODE_DRV_UNLOAD_PORT))
8457                 bnx2x__link_reset(bp);
8458
8459         /* Reset the chip */
8460         bnx2x_reset_chip(bp, reset_code);
8461
8462         /* Report UNLOAD_DONE to MCP */
8463         if (!BP_NOMCP(bp))
8464                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
8465
8466 }
8467
8468 static inline void bnx2x_disable_close_the_gate(struct bnx2x *bp)
8469 {
8470         u32 val;
8471
8472         DP(NETIF_MSG_HW, "Disabling \"close the gates\"\n");
8473
8474         if (CHIP_IS_E1(bp)) {
8475                 int port = BP_PORT(bp);
8476                 u32 addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
8477                         MISC_REG_AEU_MASK_ATTN_FUNC_0;
8478
8479                 val = REG_RD(bp, addr);
8480                 val &= ~(0x300);
8481                 REG_WR(bp, addr, val);
8482         } else if (CHIP_IS_E1H(bp)) {
8483                 val = REG_RD(bp, MISC_REG_AEU_GENERAL_MASK);
8484                 val &= ~(MISC_AEU_GENERAL_MASK_REG_AEU_PXP_CLOSE_MASK |
8485                          MISC_AEU_GENERAL_MASK_REG_AEU_NIG_CLOSE_MASK);
8486                 REG_WR(bp, MISC_REG_AEU_GENERAL_MASK, val);
8487         }
8488 }
8489
8490 /* must be called with rtnl_lock */
8491 static int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
8492 {
8493         int i;
8494
8495         if (bp->state == BNX2X_STATE_CLOSED) {
8496                 /* Interface has been removed - nothing to recover */
8497                 bp->recovery_state = BNX2X_RECOVERY_DONE;
8498                 bp->is_leader = 0;
8499                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
8500                 smp_wmb();
8501
8502                 return -EINVAL;
8503         }
8504
8505 #ifdef BCM_CNIC
8506         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
8507 #endif
8508         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
8509
8510         /* Set "drop all" */
8511         bp->rx_mode = BNX2X_RX_MODE_NONE;
8512         bnx2x_set_storm_rx_mode(bp);
8513
8514         /* Disable HW interrupts, NAPI and Tx */
8515         bnx2x_netif_stop(bp, 1);
8516         netif_carrier_off(bp->dev);
8517
8518         del_timer_sync(&bp->timer);
8519         SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
8520                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
8521         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
8522
8523         /* Release IRQs */
8524         bnx2x_free_irq(bp, false);
8525
8526         /* Cleanup the chip if needed */
8527         if (unload_mode != UNLOAD_RECOVERY)
8528                 bnx2x_chip_cleanup(bp, unload_mode);
8529
8530         bp->port.pmf = 0;
8531
8532         /* Free SKBs, SGEs, TPA pool and driver internals */
8533         bnx2x_free_skbs(bp);
8534         for_each_queue(bp, i)
8535                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
8536         for_each_queue(bp, i)
8537                 netif_napi_del(&bnx2x_fp(bp, i, napi));
8538         bnx2x_free_mem(bp);
8539
8540         bp->state = BNX2X_STATE_CLOSED;
8541
8542         /* The last driver must disable a "close the gate" if there is no
8543          * parity attention or "process kill" pending.
8544          */
8545         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
8546             bnx2x_reset_is_done(bp))
8547                 bnx2x_disable_close_the_gate(bp);
8548
8549         /* Reset MCP mail box sequence if there is on going recovery */
8550         if (unload_mode == UNLOAD_RECOVERY)
8551                 bp->fw_seq = 0;
8552
8553         return 0;
8554 }
8555
8556 /* Close gates #2, #3 and #4: */
8557 static void bnx2x_set_234_gates(struct bnx2x *bp, bool close)
8558 {
8559         u32 val, addr;
8560
8561         /* Gates #2 and #4a are closed/opened for "not E1" only */
8562         if (!CHIP_IS_E1(bp)) {
8563                 /* #4 */
8564                 val = REG_RD(bp, PXP_REG_HST_DISCARD_DOORBELLS);
8565                 REG_WR(bp, PXP_REG_HST_DISCARD_DOORBELLS,
8566                        close ? (val | 0x1) : (val & (~(u32)1)));
8567                 /* #2 */
8568                 val = REG_RD(bp, PXP_REG_HST_DISCARD_INTERNAL_WRITES);
8569                 REG_WR(bp, PXP_REG_HST_DISCARD_INTERNAL_WRITES,
8570                        close ? (val | 0x1) : (val & (~(u32)1)));
8571         }
8572
8573         /* #3 */
8574         addr = BP_PORT(bp) ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0;
8575         val = REG_RD(bp, addr);
8576         REG_WR(bp, addr, (!close) ? (val | 0x1) : (val & (~(u32)1)));
8577
8578         DP(NETIF_MSG_HW, "%s gates #2, #3 and #4\n",
8579                 close ? "closing" : "opening");
8580         mmiowb();
8581 }
8582
8583 #define SHARED_MF_CLP_MAGIC  0x80000000 /* `magic' bit */
8584
8585 static void bnx2x_clp_reset_prep(struct bnx2x *bp, u32 *magic_val)
8586 {
8587         /* Do some magic... */
8588         u32 val = MF_CFG_RD(bp, shared_mf_config.clp_mb);
8589         *magic_val = val & SHARED_MF_CLP_MAGIC;
8590         MF_CFG_WR(bp, shared_mf_config.clp_mb, val | SHARED_MF_CLP_MAGIC);
8591 }
8592
8593 /* Restore the value of the `magic' bit.
8594  *
8595  * @param pdev Device handle.
8596  * @param magic_val Old value of the `magic' bit.
8597  */
8598 static void bnx2x_clp_reset_done(struct bnx2x *bp, u32 magic_val)
8599 {
8600         /* Restore the `magic' bit value... */
8601         /* u32 val = SHMEM_RD(bp, mf_cfg.shared_mf_config.clp_mb);
8602         SHMEM_WR(bp, mf_cfg.shared_mf_config.clp_mb,
8603                 (val & (~SHARED_MF_CLP_MAGIC)) | magic_val); */
8604         u32 val = MF_CFG_RD(bp, shared_mf_config.clp_mb);
8605         MF_CFG_WR(bp, shared_mf_config.clp_mb,
8606                 (val & (~SHARED_MF_CLP_MAGIC)) | magic_val);
8607 }
8608
8609 /* Prepares for MCP reset: takes care of CLP configurations.
8610  *
8611  * @param bp
8612  * @param magic_val Old value of 'magic' bit.
8613  */
8614 static void bnx2x_reset_mcp_prep(struct bnx2x *bp, u32 *magic_val)
8615 {
8616         u32 shmem;
8617         u32 validity_offset;
8618
8619         DP(NETIF_MSG_HW, "Starting\n");
8620
8621         /* Set `magic' bit in order to save MF config */
8622         if (!CHIP_IS_E1(bp))
8623                 bnx2x_clp_reset_prep(bp, magic_val);
8624
8625         /* Get shmem offset */
8626         shmem = REG_RD(bp, MISC_REG_SHARED_MEM_ADDR);
8627         validity_offset = offsetof(struct shmem_region, validity_map[0]);
8628
8629         /* Clear validity map flags */
8630         if (shmem > 0)
8631                 REG_WR(bp, shmem + validity_offset, 0);
8632 }
8633
8634 #define MCP_TIMEOUT      5000   /* 5 seconds (in ms) */
8635 #define MCP_ONE_TIMEOUT  100    /* 100 ms */
8636
8637 /* Waits for MCP_ONE_TIMEOUT or MCP_ONE_TIMEOUT*10,
8638  * depending on the HW type.
8639  *
8640  * @param bp
8641  */
8642 static inline void bnx2x_mcp_wait_one(struct bnx2x *bp)
8643 {
8644         /* special handling for emulation and FPGA,
8645            wait 10 times longer */
8646         if (CHIP_REV_IS_SLOW(bp))
8647                 msleep(MCP_ONE_TIMEOUT*10);
8648         else
8649                 msleep(MCP_ONE_TIMEOUT);
8650 }
8651
8652 static int bnx2x_reset_mcp_comp(struct bnx2x *bp, u32 magic_val)
8653 {
8654         u32 shmem, cnt, validity_offset, val;
8655         int rc = 0;
8656
8657         msleep(100);
8658
8659         /* Get shmem offset */
8660         shmem = REG_RD(bp, MISC_REG_SHARED_MEM_ADDR);
8661         if (shmem == 0) {
8662                 BNX2X_ERR("Shmem 0 return failure\n");
8663                 rc = -ENOTTY;
8664                 goto exit_lbl;
8665         }
8666
8667         validity_offset = offsetof(struct shmem_region, validity_map[0]);
8668
8669         /* Wait for MCP to come up */
8670         for (cnt = 0; cnt < (MCP_TIMEOUT / MCP_ONE_TIMEOUT); cnt++) {
8671                 /* TBD: its best to check validity map of last port.
8672                  * currently checks on port 0.
8673                  */
8674                 val = REG_RD(bp, shmem + validity_offset);
8675                 DP(NETIF_MSG_HW, "shmem 0x%x validity map(0x%x)=0x%x\n", shmem,
8676                    shmem + validity_offset, val);
8677
8678                 /* check that shared memory is valid. */
8679                 if ((val & (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
8680                     == (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
8681                         break;
8682
8683                 bnx2x_mcp_wait_one(bp);
8684         }
8685
8686         DP(NETIF_MSG_HW, "Cnt=%d Shmem validity map 0x%x\n", cnt, val);
8687
8688         /* Check that shared memory is valid. This indicates that MCP is up. */
8689         if ((val & (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB)) !=
8690             (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB)) {
8691                 BNX2X_ERR("Shmem signature not present. MCP is not up !!\n");
8692                 rc = -ENOTTY;
8693                 goto exit_lbl;
8694         }
8695
8696 exit_lbl:
8697         /* Restore the `magic' bit value */
8698         if (!CHIP_IS_E1(bp))
8699                 bnx2x_clp_reset_done(bp, magic_val);
8700
8701         return rc;
8702 }
8703
8704 static void bnx2x_pxp_prep(struct bnx2x *bp)
8705 {
8706         if (!CHIP_IS_E1(bp)) {
8707                 REG_WR(bp, PXP2_REG_RD_START_INIT, 0);
8708                 REG_WR(bp, PXP2_REG_RQ_RBC_DONE, 0);
8709                 REG_WR(bp, PXP2_REG_RQ_CFG_DONE, 0);
8710                 mmiowb();
8711         }
8712 }
8713
8714 /*
8715  * Reset the whole chip except for:
8716  *      - PCIE core
8717  *      - PCI Glue, PSWHST, PXP/PXP2 RF (all controlled by
8718  *              one reset bit)
8719  *      - IGU
8720  *      - MISC (including AEU)
8721  *      - GRC
8722  *      - RBCN, RBCP
8723  */
8724 static void bnx2x_process_kill_chip_reset(struct bnx2x *bp)
8725 {
8726         u32 not_reset_mask1, reset_mask1, not_reset_mask2, reset_mask2;
8727
8728         not_reset_mask1 =
8729                 MISC_REGISTERS_RESET_REG_1_RST_HC |
8730                 MISC_REGISTERS_RESET_REG_1_RST_PXPV |
8731                 MISC_REGISTERS_RESET_REG_1_RST_PXP;
8732
8733         not_reset_mask2 =
8734                 MISC_REGISTERS_RESET_REG_2_RST_MDIO |
8735                 MISC_REGISTERS_RESET_REG_2_RST_EMAC0_HARD_CORE |
8736                 MISC_REGISTERS_RESET_REG_2_RST_EMAC1_HARD_CORE |
8737                 MISC_REGISTERS_RESET_REG_2_RST_MISC_CORE |
8738                 MISC_REGISTERS_RESET_REG_2_RST_RBCN |
8739                 MISC_REGISTERS_RESET_REG_2_RST_GRC  |
8740                 MISC_REGISTERS_RESET_REG_2_RST_MCP_N_RESET_REG_HARD_CORE |
8741                 MISC_REGISTERS_RESET_REG_2_RST_MCP_N_HARD_CORE_RST_B;
8742
8743         reset_mask1 = 0xffffffff;
8744
8745         if (CHIP_IS_E1(bp))
8746                 reset_mask2 = 0xffff;
8747         else
8748                 reset_mask2 = 0x1ffff;
8749
8750         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR,
8751                reset_mask1 & (~not_reset_mask1));
8752         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_CLEAR,
8753                reset_mask2 & (~not_reset_mask2));
8754
8755         barrier();
8756         mmiowb();
8757
8758         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET, reset_mask1);
8759         REG_WR(bp, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET, reset_mask2);
8760         mmiowb();
8761 }
8762
8763 static int bnx2x_process_kill(struct bnx2x *bp)
8764 {
8765         int cnt = 1000;
8766         u32 val = 0;
8767         u32 sr_cnt, blk_cnt, port_is_idle_0, port_is_idle_1, pgl_exp_rom2;
8768
8769
8770         /* Empty the Tetris buffer, wait for 1s */
8771         do {
8772                 sr_cnt  = REG_RD(bp, PXP2_REG_RD_SR_CNT);
8773                 blk_cnt = REG_RD(bp, PXP2_REG_RD_BLK_CNT);
8774                 port_is_idle_0 = REG_RD(bp, PXP2_REG_RD_PORT_IS_IDLE_0);
8775                 port_is_idle_1 = REG_RD(bp, PXP2_REG_RD_PORT_IS_IDLE_1);
8776                 pgl_exp_rom2 = REG_RD(bp, PXP2_REG_PGL_EXP_ROM2);
8777                 if ((sr_cnt == 0x7e) && (blk_cnt == 0xa0) &&
8778                     ((port_is_idle_0 & 0x1) == 0x1) &&
8779                     ((port_is_idle_1 & 0x1) == 0x1) &&
8780                     (pgl_exp_rom2 == 0xffffffff))
8781                         break;
8782                 msleep(1);
8783         } while (cnt-- > 0);
8784
8785         if (cnt <= 0) {
8786                 DP(NETIF_MSG_HW, "Tetris buffer didn't get empty or there"
8787                           " are still"
8788                           " outstanding read requests after 1s!\n");
8789                 DP(NETIF_MSG_HW, "sr_cnt=0x%08x, blk_cnt=0x%08x,"
8790                           " port_is_idle_0=0x%08x,"
8791                           " port_is_idle_1=0x%08x, pgl_exp_rom2=0x%08x\n",
8792                           sr_cnt, blk_cnt, port_is_idle_0, port_is_idle_1,
8793                           pgl_exp_rom2);
8794                 return -EAGAIN;
8795         }
8796
8797         barrier();
8798
8799         /* Close gates #2, #3 and #4 */
8800         bnx2x_set_234_gates(bp, true);
8801
8802         /* TBD: Indicate that "process kill" is in progress to MCP */
8803
8804         /* Clear "unprepared" bit */
8805         REG_WR(bp, MISC_REG_UNPREPARED, 0);
8806         barrier();
8807
8808         /* Make sure all is written to the chip before the reset */
8809         mmiowb();
8810
8811         /* Wait for 1ms to empty GLUE and PCI-E core queues,
8812          * PSWHST, GRC and PSWRD Tetris buffer.
8813          */
8814         msleep(1);
8815
8816         /* Prepare to chip reset: */
8817         /* MCP */
8818         bnx2x_reset_mcp_prep(bp, &val);
8819
8820         /* PXP */
8821         bnx2x_pxp_prep(bp);
8822         barrier();
8823
8824         /* reset the chip */
8825         bnx2x_process_kill_chip_reset(bp);
8826         barrier();
8827
8828         /* Recover after reset: */
8829         /* MCP */
8830         if (bnx2x_reset_mcp_comp(bp, val))
8831                 return -EAGAIN;
8832
8833         /* PXP */
8834         bnx2x_pxp_prep(bp);
8835
8836         /* Open the gates #2, #3 and #4 */
8837         bnx2x_set_234_gates(bp, false);
8838
8839         /* TBD: IGU/AEU preparation bring back the AEU/IGU to a
8840          * reset state, re-enable attentions. */
8841
8842         return 0;
8843 }
8844
8845 static int bnx2x_leader_reset(struct bnx2x *bp)
8846 {
8847         int rc = 0;
8848         /* Try to recover after the failure */
8849         if (bnx2x_process_kill(bp)) {
8850                 printk(KERN_ERR "%s: Something bad had happen! Aii!\n",
8851                        bp->dev->name);
8852                 rc = -EAGAIN;
8853                 goto exit_leader_reset;
8854         }
8855
8856         /* Clear "reset is in progress" bit and update the driver state */
8857         bnx2x_set_reset_done(bp);
8858         bp->recovery_state = BNX2X_RECOVERY_DONE;
8859
8860 exit_leader_reset:
8861         bp->is_leader = 0;
8862         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
8863         smp_wmb();
8864         return rc;
8865 }
8866
8867 static int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state);
8868
8869 /* Assumption: runs under rtnl lock. This together with the fact
8870  * that it's called only from bnx2x_reset_task() ensure that it
8871  * will never be called when netif_running(bp->dev) is false.
8872  */
8873 static void bnx2x_parity_recover(struct bnx2x *bp)
8874 {
8875         DP(NETIF_MSG_HW, "Handling parity\n");
8876         while (1) {
8877                 switch (bp->recovery_state) {
8878                 case BNX2X_RECOVERY_INIT:
8879                         DP(NETIF_MSG_HW, "State is BNX2X_RECOVERY_INIT\n");
8880                         /* Try to get a LEADER_LOCK HW lock */
8881                         if (bnx2x_trylock_hw_lock(bp,
8882                                 HW_LOCK_RESOURCE_RESERVED_08))
8883                                 bp->is_leader = 1;
8884
8885                         /* Stop the driver */
8886                         /* If interface has been removed - break */
8887                         if (bnx2x_nic_unload(bp, UNLOAD_RECOVERY))
8888                                 return;
8889
8890                         bp->recovery_state = BNX2X_RECOVERY_WAIT;
8891                         /* Ensure "is_leader" and "recovery_state"
8892                          *  update values are seen on other CPUs
8893                          */
8894                         smp_wmb();
8895                         break;
8896
8897                 case BNX2X_RECOVERY_WAIT:
8898                         DP(NETIF_MSG_HW, "State is BNX2X_RECOVERY_WAIT\n");
8899                         if (bp->is_leader) {
8900                                 u32 load_counter = bnx2x_get_load_cnt(bp);
8901                                 if (load_counter) {
8902                                         /* Wait until all other functions get
8903                                          * down.
8904                                          */
8905                                         schedule_delayed_work(&bp->reset_task,
8906                                                                 HZ/10);
8907                                         return;
8908                                 } else {
8909                                         /* If all other functions got down -
8910                                          * try to bring the chip back to
8911                                          * normal. In any case it's an exit
8912                                          * point for a leader.
8913                                          */
8914                                         if (bnx2x_leader_reset(bp) ||
8915                                         bnx2x_nic_load(bp, LOAD_NORMAL)) {
8916                                                 printk(KERN_ERR"%s: Recovery "
8917                                                 "has failed. Power cycle is "
8918                                                 "needed.\n", bp->dev->name);
8919                                                 /* Disconnect this device */
8920                                                 netif_device_detach(bp->dev);
8921                                                 /* Block ifup for all function
8922                                                  * of this ASIC until
8923                                                  * "process kill" or power
8924                                                  * cycle.
8925                                                  */
8926                                                 bnx2x_set_reset_in_progress(bp);
8927                                                 /* Shut down the power */
8928                                                 bnx2x_set_power_state(bp,
8929                                                                 PCI_D3hot);
8930                                                 return;
8931                                         }
8932
8933                                         return;
8934                                 }
8935                         } else { /* non-leader */
8936                                 if (!bnx2x_reset_is_done(bp)) {
8937                                         /* Try to get a LEADER_LOCK HW lock as
8938                                          * long as a former leader may have
8939                                          * been unloaded by the user or
8940                                          * released a leadership by another
8941                                          * reason.
8942                                          */
8943                                         if (bnx2x_trylock_hw_lock(bp,
8944                                             HW_LOCK_RESOURCE_RESERVED_08)) {
8945                                                 /* I'm a leader now! Restart a
8946                                                  * switch case.
8947                                                  */
8948                                                 bp->is_leader = 1;
8949                                                 break;
8950                                         }
8951
8952                                         schedule_delayed_work(&bp->reset_task,
8953                                                                 HZ/10);
8954                                         return;
8955
8956                                 } else { /* A leader has completed
8957                                           * the "process kill". It's an exit
8958                                           * point for a non-leader.
8959                                           */
8960                                         bnx2x_nic_load(bp, LOAD_NORMAL);
8961                                         bp->recovery_state =
8962                                                 BNX2X_RECOVERY_DONE;
8963                                         smp_wmb();
8964                                         return;
8965                                 }
8966                         }
8967                 default:
8968                         return;
8969                 }
8970         }
8971 }
8972
8973 /* bnx2x_nic_unload() flushes the bnx2x_wq, thus reset task is
8974  * scheduled on a general queue in order to prevent a dead lock.
8975  */
8976 static void bnx2x_reset_task(struct work_struct *work)
8977 {
8978         struct bnx2x *bp = container_of(work, struct bnx2x, reset_task.work);
8979
8980 #ifdef BNX2X_STOP_ON_ERROR
8981         BNX2X_ERR("reset task called but STOP_ON_ERROR defined"
8982                   " so reset not done to allow debug dump,\n"
8983          KERN_ERR " you will need to reboot when done\n");
8984         return;
8985 #endif
8986
8987         rtnl_lock();
8988
8989         if (!netif_running(bp->dev))
8990                 goto reset_task_exit;
8991
8992         if (unlikely(bp->recovery_state != BNX2X_RECOVERY_DONE))
8993                 bnx2x_parity_recover(bp);
8994         else {
8995                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
8996                 bnx2x_nic_load(bp, LOAD_NORMAL);
8997         }
8998
8999 reset_task_exit:
9000         rtnl_unlock();
9001 }
9002
9003 /* end of nic load/unload */
9004
9005 /* ethtool_ops */
9006
9007 /*
9008  * Init service functions
9009  */
9010
9011 static inline u32 bnx2x_get_pretend_reg(struct bnx2x *bp, int func)
9012 {
9013         switch (func) {
9014         case 0: return PXP2_REG_PGL_PRETEND_FUNC_F0;
9015         case 1: return PXP2_REG_PGL_PRETEND_FUNC_F1;
9016         case 2: return PXP2_REG_PGL_PRETEND_FUNC_F2;
9017         case 3: return PXP2_REG_PGL_PRETEND_FUNC_F3;
9018         case 4: return PXP2_REG_PGL_PRETEND_FUNC_F4;
9019         case 5: return PXP2_REG_PGL_PRETEND_FUNC_F5;
9020         case 6: return PXP2_REG_PGL_PRETEND_FUNC_F6;
9021         case 7: return PXP2_REG_PGL_PRETEND_FUNC_F7;
9022         default:
9023                 BNX2X_ERR("Unsupported function index: %d\n", func);
9024                 return (u32)(-1);
9025         }
9026 }
9027
9028 static void bnx2x_undi_int_disable_e1h(struct bnx2x *bp, int orig_func)
9029 {
9030         u32 reg = bnx2x_get_pretend_reg(bp, orig_func), new_val;
9031
9032         /* Flush all outstanding writes */
9033         mmiowb();
9034
9035         /* Pretend to be function 0 */
9036         REG_WR(bp, reg, 0);
9037         /* Flush the GRC transaction (in the chip) */
9038         new_val = REG_RD(bp, reg);
9039         if (new_val != 0) {
9040                 BNX2X_ERR("Hmmm... Pretend register wasn't updated: (0,%d)!\n",
9041                           new_val);
9042                 BUG();
9043         }
9044
9045         /* From now we are in the "like-E1" mode */
9046         bnx2x_int_disable(bp);
9047
9048         /* Flush all outstanding writes */
9049         mmiowb();
9050
9051         /* Restore the original funtion settings */
9052         REG_WR(bp, reg, orig_func);
9053         new_val = REG_RD(bp, reg);
9054         if (new_val != orig_func) {
9055                 BNX2X_ERR("Hmmm... Pretend register wasn't updated: (%d,%d)!\n",
9056                           orig_func, new_val);
9057                 BUG();
9058         }
9059 }
9060
9061 static inline void bnx2x_undi_int_disable(struct bnx2x *bp, int func)
9062 {
9063         if (CHIP_IS_E1H(bp))
9064                 bnx2x_undi_int_disable_e1h(bp, func);
9065         else
9066                 bnx2x_int_disable(bp);
9067 }
9068
9069 static void __devinit bnx2x_undi_unload(struct bnx2x *bp)
9070 {
9071         u32 val;
9072
9073         /* Check if there is any driver already loaded */
9074         val = REG_RD(bp, MISC_REG_UNPREPARED);
9075         if (val == 0x1) {
9076                 /* Check if it is the UNDI driver
9077                  * UNDI driver initializes CID offset for normal bell to 0x7
9078                  */
9079                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_UNDI);
9080                 val = REG_RD(bp, DORQ_REG_NORM_CID_OFST);
9081                 if (val == 0x7) {
9082                         u32 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_DIS;
9083                         /* save our func */
9084                         int func = BP_FUNC(bp);
9085                         u32 swap_en;
9086                         u32 swap_val;
9087
9088                         /* clear the UNDI indication */
9089                         REG_WR(bp, DORQ_REG_NORM_CID_OFST, 0);
9090
9091                         BNX2X_DEV_INFO("UNDI is active! reset device\n");
9092
9093                         /* try unload UNDI on port 0 */
9094                         bp->func = 0;
9095                         bp->fw_seq =
9096                                (SHMEM_RD(bp, func_mb[bp->func].drv_mb_header) &
9097                                 DRV_MSG_SEQ_NUMBER_MASK);
9098                         reset_code = bnx2x_fw_command(bp, reset_code);
9099
9100                         /* if UNDI is loaded on the other port */
9101                         if (reset_code != FW_MSG_CODE_DRV_UNLOAD_COMMON) {
9102
9103                                 /* send "DONE" for previous unload */
9104                                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
9105
9106                                 /* unload UNDI on port 1 */
9107                                 bp->func = 1;
9108                                 bp->fw_seq =
9109                                (SHMEM_RD(bp, func_mb[bp->func].drv_mb_header) &
9110                                         DRV_MSG_SEQ_NUMBER_MASK);
9111                                 reset_code = DRV_MSG_CODE_UNLOAD_REQ_WOL_DIS;
9112
9113                                 bnx2x_fw_command(bp, reset_code);
9114                         }
9115
9116                         /* now it's safe to release the lock */
9117                         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_UNDI);
9118
9119                         bnx2x_undi_int_disable(bp, func);
9120
9121                         /* close input traffic and wait for it */
9122                         /* Do not rcv packets to BRB */
9123                         REG_WR(bp,
9124                               (BP_PORT(bp) ? NIG_REG_LLH1_BRB1_DRV_MASK :
9125                                              NIG_REG_LLH0_BRB1_DRV_MASK), 0x0);
9126                         /* Do not direct rcv packets that are not for MCP to
9127                          * the BRB */
9128                         REG_WR(bp,
9129                                (BP_PORT(bp) ? NIG_REG_LLH1_BRB1_NOT_MCP :
9130                                               NIG_REG_LLH0_BRB1_NOT_MCP), 0x0);
9131                         /* clear AEU */
9132                         REG_WR(bp,
9133                              (BP_PORT(bp) ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
9134                                             MISC_REG_AEU_MASK_ATTN_FUNC_0), 0);
9135                         msleep(10);
9136
9137                         /* save NIG port swap info */
9138                         swap_val = REG_RD(bp, NIG_REG_PORT_SWAP);
9139                         swap_en = REG_RD(bp, NIG_REG_STRAP_OVERRIDE);
9140                         /* reset device */
9141                         REG_WR(bp,
9142                                GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR,
9143                                0xd3ffffff);
9144                         REG_WR(bp,
9145                                GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_CLEAR,
9146                                0x1403);
9147                         /* take the NIG out of reset and restore swap values */
9148                         REG_WR(bp,
9149                                GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET,
9150                                MISC_REGISTERS_RESET_REG_1_RST_NIG);
9151                         REG_WR(bp, NIG_REG_PORT_SWAP, swap_val);
9152                         REG_WR(bp, NIG_REG_STRAP_OVERRIDE, swap_en);
9153
9154                         /* send unload done to the MCP */
9155                         bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
9156
9157                         /* restore our func and fw_seq */
9158                         bp->func = func;
9159                         bp->fw_seq =
9160                                (SHMEM_RD(bp, func_mb[bp->func].drv_mb_header) &
9161                                 DRV_MSG_SEQ_NUMBER_MASK);
9162
9163                 } else
9164                         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_UNDI);
9165         }
9166 }
9167
9168 static void __devinit bnx2x_get_common_hwinfo(struct bnx2x *bp)
9169 {
9170         u32 val, val2, val3, val4, id;
9171         u16 pmc;
9172
9173         /* Get the chip revision id and number. */
9174         /* chip num:16-31, rev:12-15, metal:4-11, bond_id:0-3 */
9175         val = REG_RD(bp, MISC_REG_CHIP_NUM);
9176         id = ((val & 0xffff) << 16);
9177         val = REG_RD(bp, MISC_REG_CHIP_REV);
9178         id |= ((val & 0xf) << 12);
9179         val = REG_RD(bp, MISC_REG_CHIP_METAL);
9180         id |= ((val & 0xff) << 4);
9181         val = REG_RD(bp, MISC_REG_BOND_ID);
9182         id |= (val & 0xf);
9183         bp->common.chip_id = id;
9184         bp->link_params.chip_id = bp->common.chip_id;
9185         BNX2X_DEV_INFO("chip ID is 0x%x\n", id);
9186
9187         val = (REG_RD(bp, 0x2874) & 0x55);
9188         if ((bp->common.chip_id & 0x1) ||
9189             (CHIP_IS_E1(bp) && val) || (CHIP_IS_E1H(bp) && (val == 0x55))) {
9190                 bp->flags |= ONE_PORT_FLAG;
9191                 BNX2X_DEV_INFO("single port device\n");
9192         }
9193
9194         val = REG_RD(bp, MCP_REG_MCPR_NVM_CFG4);
9195         bp->common.flash_size = (NVRAM_1MB_SIZE <<
9196                                  (val & MCPR_NVM_CFG4_FLASH_SIZE));
9197         BNX2X_DEV_INFO("flash_size 0x%x (%d)\n",
9198                        bp->common.flash_size, bp->common.flash_size);
9199
9200         bp->common.shmem_base = REG_RD(bp, MISC_REG_SHARED_MEM_ADDR);
9201         bp->common.shmem2_base = REG_RD(bp, MISC_REG_GENERIC_CR_0);
9202         bp->link_params.shmem_base = bp->common.shmem_base;
9203         BNX2X_DEV_INFO("shmem offset 0x%x  shmem2 offset 0x%x\n",
9204                        bp->common.shmem_base, bp->common.shmem2_base);
9205
9206         if (!bp->common.shmem_base ||
9207             (bp->common.shmem_base < 0xA0000) ||
9208             (bp->common.shmem_base >= 0xC0000)) {
9209                 BNX2X_DEV_INFO("MCP not active\n");
9210                 bp->flags |= NO_MCP_FLAG;
9211                 return;
9212         }
9213
9214         val = SHMEM_RD(bp, validity_map[BP_PORT(bp)]);
9215         if ((val & (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
9216                 != (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
9217                 BNX2X_ERROR("BAD MCP validity signature\n");
9218
9219         bp->common.hw_config = SHMEM_RD(bp, dev_info.shared_hw_config.config);
9220         BNX2X_DEV_INFO("hw_config 0x%08x\n", bp->common.hw_config);
9221
9222         bp->link_params.hw_led_mode = ((bp->common.hw_config &
9223                                         SHARED_HW_CFG_LED_MODE_MASK) >>
9224                                        SHARED_HW_CFG_LED_MODE_SHIFT);
9225
9226         bp->link_params.feature_config_flags = 0;
9227         val = SHMEM_RD(bp, dev_info.shared_feature_config.config);
9228         if (val & SHARED_FEAT_CFG_OVERRIDE_PREEMPHASIS_CFG_ENABLED)
9229                 bp->link_params.feature_config_flags |=
9230                                 FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED;
9231         else
9232                 bp->link_params.feature_config_flags &=
9233                                 ~FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED;
9234
9235         val = SHMEM_RD(bp, dev_info.bc_rev) >> 8;
9236         bp->common.bc_ver = val;
9237         BNX2X_DEV_INFO("bc_ver %X\n", val);
9238         if (val < BNX2X_BC_VER) {
9239                 /* for now only warn
9240                  * later we might need to enforce this */
9241                 BNX2X_ERROR("This driver needs bc_ver %X but found %X, "
9242                             "please upgrade BC\n", BNX2X_BC_VER, val);
9243         }
9244         bp->link_params.feature_config_flags |=
9245                 (val >= REQ_BC_VER_4_VRFY_OPT_MDL) ?
9246                 FEATURE_CONFIG_BC_SUPPORTS_OPT_MDL_VRFY : 0;
9247
9248         if (BP_E1HVN(bp) == 0) {
9249                 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_PMC, &pmc);
9250                 bp->flags |= (pmc & PCI_PM_CAP_PME_D3cold) ? 0 : NO_WOL_FLAG;
9251         } else {
9252                 /* no WOL capability for E1HVN != 0 */
9253                 bp->flags |= NO_WOL_FLAG;
9254         }
9255         BNX2X_DEV_INFO("%sWoL capable\n",
9256                        (bp->flags & NO_WOL_FLAG) ? "not " : "");
9257
9258         val = SHMEM_RD(bp, dev_info.shared_hw_config.part_num);
9259         val2 = SHMEM_RD(bp, dev_info.shared_hw_config.part_num[4]);
9260         val3 = SHMEM_RD(bp, dev_info.shared_hw_config.part_num[8]);
9261         val4 = SHMEM_RD(bp, dev_info.shared_hw_config.part_num[12]);
9262
9263         dev_info(&bp->pdev->dev, "part number %X-%X-%X-%X\n",
9264                  val, val2, val3, val4);
9265 }
9266
9267 static void __devinit bnx2x_link_settings_supported(struct bnx2x *bp,
9268                                                     u32 switch_cfg)
9269 {
9270         int port = BP_PORT(bp);
9271         u32 ext_phy_type;
9272
9273         switch (switch_cfg) {
9274         case SWITCH_CFG_1G:
9275                 BNX2X_DEV_INFO("switch_cfg 0x%x (1G)\n", switch_cfg);
9276
9277                 ext_phy_type =
9278                         SERDES_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
9279                 switch (ext_phy_type) {
9280                 case PORT_HW_CFG_SERDES_EXT_PHY_TYPE_DIRECT:
9281                         BNX2X_DEV_INFO("ext_phy_type 0x%x (Direct)\n",
9282                                        ext_phy_type);
9283
9284                         bp->port.supported |= (SUPPORTED_10baseT_Half |
9285                                                SUPPORTED_10baseT_Full |
9286                                                SUPPORTED_100baseT_Half |
9287                                                SUPPORTED_100baseT_Full |
9288                                                SUPPORTED_1000baseT_Full |
9289                                                SUPPORTED_2500baseX_Full |
9290                                                SUPPORTED_TP |
9291                                                SUPPORTED_FIBRE |
9292                                                SUPPORTED_Autoneg |
9293                                                SUPPORTED_Pause |
9294                                                SUPPORTED_Asym_Pause);
9295                         break;
9296
9297                 case PORT_HW_CFG_SERDES_EXT_PHY_TYPE_BCM5482:
9298                         BNX2X_DEV_INFO("ext_phy_type 0x%x (5482)\n",
9299                                        ext_phy_type);
9300
9301                         bp->port.supported |= (SUPPORTED_10baseT_Half |
9302                                                SUPPORTED_10baseT_Full |
9303                                                SUPPORTED_100baseT_Half |
9304                                                SUPPORTED_100baseT_Full |
9305                                                SUPPORTED_1000baseT_Full |
9306                                                SUPPORTED_TP |
9307                                                SUPPORTED_FIBRE |
9308                                                SUPPORTED_Autoneg |
9309                                                SUPPORTED_Pause |
9310                                                SUPPORTED_Asym_Pause);
9311                         break;
9312
9313                 default:
9314                         BNX2X_ERR("NVRAM config error. "
9315                                   "BAD SerDes ext_phy_config 0x%x\n",
9316                                   bp->link_params.ext_phy_config);
9317                         return;
9318                 }
9319
9320                 bp->port.phy_addr = REG_RD(bp, NIG_REG_SERDES0_CTRL_PHY_ADDR +
9321                                            port*0x10);
9322                 BNX2X_DEV_INFO("phy_addr 0x%x\n", bp->port.phy_addr);
9323                 break;
9324
9325         case SWITCH_CFG_10G:
9326                 BNX2X_DEV_INFO("switch_cfg 0x%x (10G)\n", switch_cfg);
9327
9328                 ext_phy_type =
9329                         XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
9330                 switch (ext_phy_type) {
9331                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT:
9332                         BNX2X_DEV_INFO("ext_phy_type 0x%x (Direct)\n",
9333                                        ext_phy_type);
9334
9335                         bp->port.supported |= (SUPPORTED_10baseT_Half |
9336                                                SUPPORTED_10baseT_Full |
9337                                                SUPPORTED_100baseT_Half |
9338                                                SUPPORTED_100baseT_Full |
9339                                                SUPPORTED_1000baseT_Full |
9340                                                SUPPORTED_2500baseX_Full |
9341                                                SUPPORTED_10000baseT_Full |
9342                                                SUPPORTED_TP |
9343                                                SUPPORTED_FIBRE |
9344                                                SUPPORTED_Autoneg |
9345                                                SUPPORTED_Pause |
9346                                                SUPPORTED_Asym_Pause);
9347                         break;
9348
9349                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8072:
9350                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8072)\n",
9351                                        ext_phy_type);
9352
9353                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9354                                                SUPPORTED_1000baseT_Full |
9355                                                SUPPORTED_FIBRE |
9356                                                SUPPORTED_Autoneg |
9357                                                SUPPORTED_Pause |
9358                                                SUPPORTED_Asym_Pause);
9359                         break;
9360
9361                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8073:
9362                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8073)\n",
9363                                        ext_phy_type);
9364
9365                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9366                                                SUPPORTED_2500baseX_Full |
9367                                                SUPPORTED_1000baseT_Full |
9368                                                SUPPORTED_FIBRE |
9369                                                SUPPORTED_Autoneg |
9370                                                SUPPORTED_Pause |
9371                                                SUPPORTED_Asym_Pause);
9372                         break;
9373
9374                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8705:
9375                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8705)\n",
9376                                        ext_phy_type);
9377
9378                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9379                                                SUPPORTED_FIBRE |
9380                                                SUPPORTED_Pause |
9381                                                SUPPORTED_Asym_Pause);
9382                         break;
9383
9384                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8706:
9385                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8706)\n",
9386                                        ext_phy_type);
9387
9388                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9389                                                SUPPORTED_1000baseT_Full |
9390                                                SUPPORTED_FIBRE |
9391                                                SUPPORTED_Pause |
9392                                                SUPPORTED_Asym_Pause);
9393                         break;
9394
9395                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8726:
9396                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8726)\n",
9397                                        ext_phy_type);
9398
9399                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9400                                                SUPPORTED_1000baseT_Full |
9401                                                SUPPORTED_Autoneg |
9402                                                SUPPORTED_FIBRE |
9403                                                SUPPORTED_Pause |
9404                                                SUPPORTED_Asym_Pause);
9405                         break;
9406
9407                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727:
9408                         BNX2X_DEV_INFO("ext_phy_type 0x%x (8727)\n",
9409                                        ext_phy_type);
9410
9411                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9412                                                SUPPORTED_1000baseT_Full |
9413                                                SUPPORTED_Autoneg |
9414                                                SUPPORTED_FIBRE |
9415                                                SUPPORTED_Pause |
9416                                                SUPPORTED_Asym_Pause);
9417                         break;
9418
9419                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101:
9420                         BNX2X_DEV_INFO("ext_phy_type 0x%x (SFX7101)\n",
9421                                        ext_phy_type);
9422
9423                         bp->port.supported |= (SUPPORTED_10000baseT_Full |
9424                                                SUPPORTED_TP |
9425                                                SUPPORTED_Autoneg |
9426                                                SUPPORTED_Pause |
9427                                                SUPPORTED_Asym_Pause);
9428                         break;
9429
9430                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8481:
9431                         BNX2X_DEV_INFO("ext_phy_type 0x%x (BCM8481)\n",
9432                                        ext_phy_type);
9433
9434                         bp->port.supported |= (SUPPORTED_10baseT_Half |
9435                                                SUPPORTED_10baseT_Full |
9436                                                SUPPORTED_100baseT_Half |
9437                                                SUPPORTED_100baseT_Full |
9438                                                SUPPORTED_1000baseT_Full |
9439                                                SUPPORTED_10000baseT_Full |
9440                                                SUPPORTED_TP |
9441                                                SUPPORTED_Autoneg |
9442                                                SUPPORTED_Pause |
9443                                                SUPPORTED_Asym_Pause);
9444                         break;
9445
9446                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE:
9447                         BNX2X_ERR("XGXS PHY Failure detected 0x%x\n",
9448                                   bp->link_params.ext_phy_config);
9449                         break;
9450
9451                 default:
9452                         BNX2X_ERR("NVRAM config error. "
9453                                   "BAD XGXS ext_phy_config 0x%x\n",
9454                                   bp->link_params.ext_phy_config);
9455                         return;
9456                 }
9457
9458                 bp->port.phy_addr = REG_RD(bp, NIG_REG_XGXS0_CTRL_PHY_ADDR +
9459                                            port*0x18);
9460                 BNX2X_DEV_INFO("phy_addr 0x%x\n", bp->port.phy_addr);
9461
9462                 break;
9463
9464         default:
9465                 BNX2X_ERR("BAD switch_cfg link_config 0x%x\n",
9466                           bp->port.link_config);
9467                 return;
9468         }
9469         bp->link_params.phy_addr = bp->port.phy_addr;
9470
9471         /* mask what we support according to speed_cap_mask */
9472         if (!(bp->link_params.speed_cap_mask &
9473                                 PORT_HW_CFG_SPEED_CAPABILITY_D0_10M_HALF))
9474                 bp->port.supported &= ~SUPPORTED_10baseT_Half;
9475
9476         if (!(bp->link_params.speed_cap_mask &
9477                                 PORT_HW_CFG_SPEED_CAPABILITY_D0_10M_FULL))
9478                 bp->port.supported &= ~SUPPORTED_10baseT_Full;
9479
9480         if (!(bp->link_params.speed_cap_mask &
9481                                 PORT_HW_CFG_SPEED_CAPABILITY_D0_100M_HALF))
9482                 bp->port.supported &= ~SUPPORTED_100baseT_Half;
9483
9484         if (!(bp->link_params.speed_cap_mask &
9485                                 PORT_HW_CFG_SPEED_CAPABILITY_D0_100M_FULL))
9486                 bp->port.supported &= ~SUPPORTED_100baseT_Full;
9487
9488         if (!(bp->link_params.speed_cap_mask &
9489                                         PORT_HW_CFG_SPEED_CAPABILITY_D0_1G))
9490                 bp->port.supported &= ~(SUPPORTED_1000baseT_Half |
9491                                         SUPPORTED_1000baseT_Full);
9492
9493         if (!(bp->link_params.speed_cap_mask &
9494                                         PORT_HW_CFG_SPEED_CAPABILITY_D0_2_5G))
9495                 bp->port.supported &= ~SUPPORTED_2500baseX_Full;
9496
9497         if (!(bp->link_params.speed_cap_mask &
9498                                         PORT_HW_CFG_SPEED_CAPABILITY_D0_10G))
9499                 bp->port.supported &= ~SUPPORTED_10000baseT_Full;
9500
9501         BNX2X_DEV_INFO("supported 0x%x\n", bp->port.supported);
9502 }
9503
9504 static void __devinit bnx2x_link_settings_requested(struct bnx2x *bp)
9505 {
9506         bp->link_params.req_duplex = DUPLEX_FULL;
9507
9508         switch (bp->port.link_config & PORT_FEATURE_LINK_SPEED_MASK) {
9509         case PORT_FEATURE_LINK_SPEED_AUTO:
9510                 if (bp->port.supported & SUPPORTED_Autoneg) {
9511                         bp->link_params.req_line_speed = SPEED_AUTO_NEG;
9512                         bp->port.advertising = bp->port.supported;
9513                 } else {
9514                         u32 ext_phy_type =
9515                             XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
9516
9517                         if ((ext_phy_type ==
9518                              PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8705) ||
9519                             (ext_phy_type ==
9520                              PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8706)) {
9521                                 /* force 10G, no AN */
9522                                 bp->link_params.req_line_speed = SPEED_10000;
9523                                 bp->port.advertising =
9524                                                 (ADVERTISED_10000baseT_Full |
9525                                                  ADVERTISED_FIBRE);
9526                                 break;
9527                         }
9528                         BNX2X_ERR("NVRAM config error. "
9529                                   "Invalid link_config 0x%x"
9530                                   "  Autoneg not supported\n",
9531                                   bp->port.link_config);
9532                         return;
9533                 }
9534                 break;
9535
9536         case PORT_FEATURE_LINK_SPEED_10M_FULL:
9537                 if (bp->port.supported & SUPPORTED_10baseT_Full) {
9538                         bp->link_params.req_line_speed = SPEED_10;
9539                         bp->port.advertising = (ADVERTISED_10baseT_Full |
9540                                                 ADVERTISED_TP);
9541                 } else {
9542                         BNX2X_ERROR("NVRAM config error. "
9543                                     "Invalid link_config 0x%x"
9544                                     "  speed_cap_mask 0x%x\n",
9545                                     bp->port.link_config,
9546                                     bp->link_params.speed_cap_mask);
9547                         return;
9548                 }
9549                 break;
9550
9551         case PORT_FEATURE_LINK_SPEED_10M_HALF:
9552                 if (bp->port.supported & SUPPORTED_10baseT_Half) {
9553                         bp->link_params.req_line_speed = SPEED_10;
9554                         bp->link_params.req_duplex = DUPLEX_HALF;
9555                         bp->port.advertising = (ADVERTISED_10baseT_Half |
9556                                                 ADVERTISED_TP);
9557                 } else {
9558                         BNX2X_ERROR("NVRAM config error. "
9559                                     "Invalid link_config 0x%x"
9560                                     "  speed_cap_mask 0x%x\n",
9561                                     bp->port.link_config,
9562                                     bp->link_params.speed_cap_mask);
9563                         return;
9564                 }
9565                 break;
9566
9567         case PORT_FEATURE_LINK_SPEED_100M_FULL:
9568                 if (bp->port.supported & SUPPORTED_100baseT_Full) {
9569                         bp->link_params.req_line_speed = SPEED_100;
9570                         bp->port.advertising = (ADVERTISED_100baseT_Full |
9571                                                 ADVERTISED_TP);
9572                 } else {
9573                         BNX2X_ERROR("NVRAM config error. "
9574                                     "Invalid link_config 0x%x"
9575                                     "  speed_cap_mask 0x%x\n",
9576                                     bp->port.link_config,
9577                                     bp->link_params.speed_cap_mask);
9578                         return;
9579                 }
9580                 break;
9581
9582         case PORT_FEATURE_LINK_SPEED_100M_HALF:
9583                 if (bp->port.supported & SUPPORTED_100baseT_Half) {
9584                         bp->link_params.req_line_speed = SPEED_100;
9585                         bp->link_params.req_duplex = DUPLEX_HALF;
9586                         bp->port.advertising = (ADVERTISED_100baseT_Half |
9587                                                 ADVERTISED_TP);
9588                 } else {
9589                         BNX2X_ERROR("NVRAM config error. "
9590                                     "Invalid link_config 0x%x"
9591                                     "  speed_cap_mask 0x%x\n",
9592                                     bp->port.link_config,
9593                                     bp->link_params.speed_cap_mask);
9594                         return;
9595                 }
9596                 break;
9597
9598         case PORT_FEATURE_LINK_SPEED_1G:
9599                 if (bp->port.supported & SUPPORTED_1000baseT_Full) {
9600                         bp->link_params.req_line_speed = SPEED_1000;
9601                         bp->port.advertising = (ADVERTISED_1000baseT_Full |
9602                                                 ADVERTISED_TP);
9603                 } else {
9604                         BNX2X_ERROR("NVRAM config error. "
9605                                     "Invalid link_config 0x%x"
9606                                     "  speed_cap_mask 0x%x\n",
9607                                     bp->port.link_config,
9608                                     bp->link_params.speed_cap_mask);
9609                         return;
9610                 }
9611                 break;
9612
9613         case PORT_FEATURE_LINK_SPEED_2_5G:
9614                 if (bp->port.supported & SUPPORTED_2500baseX_Full) {
9615                         bp->link_params.req_line_speed = SPEED_2500;
9616                         bp->port.advertising = (ADVERTISED_2500baseX_Full |
9617                                                 ADVERTISED_TP);
9618                 } else {
9619                         BNX2X_ERROR("NVRAM config error. "
9620                                     "Invalid link_config 0x%x"
9621                                     "  speed_cap_mask 0x%x\n",
9622                                     bp->port.link_config,
9623                                     bp->link_params.speed_cap_mask);
9624                         return;
9625                 }
9626                 break;
9627
9628         case PORT_FEATURE_LINK_SPEED_10G_CX4:
9629         case PORT_FEATURE_LINK_SPEED_10G_KX4:
9630         case PORT_FEATURE_LINK_SPEED_10G_KR:
9631                 if (bp->port.supported & SUPPORTED_10000baseT_Full) {
9632                         bp->link_params.req_line_speed = SPEED_10000;
9633                         bp->port.advertising = (ADVERTISED_10000baseT_Full |
9634                                                 ADVERTISED_FIBRE);
9635                 } else {
9636                         BNX2X_ERROR("NVRAM config error. "
9637                                     "Invalid link_config 0x%x"
9638                                     "  speed_cap_mask 0x%x\n",
9639                                     bp->port.link_config,
9640                                     bp->link_params.speed_cap_mask);
9641                         return;
9642                 }
9643                 break;
9644
9645         default:
9646                 BNX2X_ERROR("NVRAM config error. "
9647                             "BAD link speed link_config 0x%x\n",
9648                             bp->port.link_config);
9649                 bp->link_params.req_line_speed = SPEED_AUTO_NEG;
9650                 bp->port.advertising = bp->port.supported;
9651                 break;
9652         }
9653
9654         bp->link_params.req_flow_ctrl = (bp->port.link_config &
9655                                          PORT_FEATURE_FLOW_CONTROL_MASK);
9656         if ((bp->link_params.req_flow_ctrl == BNX2X_FLOW_CTRL_AUTO) &&
9657             !(bp->port.supported & SUPPORTED_Autoneg))
9658                 bp->link_params.req_flow_ctrl = BNX2X_FLOW_CTRL_NONE;
9659
9660         BNX2X_DEV_INFO("req_line_speed %d  req_duplex %d  req_flow_ctrl 0x%x"
9661                        "  advertising 0x%x\n",
9662                        bp->link_params.req_line_speed,
9663                        bp->link_params.req_duplex,
9664                        bp->link_params.req_flow_ctrl, bp->port.advertising);
9665 }
9666
9667 static void __devinit bnx2x_set_mac_buf(u8 *mac_buf, u32 mac_lo, u16 mac_hi)
9668 {
9669         mac_hi = cpu_to_be16(mac_hi);
9670         mac_lo = cpu_to_be32(mac_lo);
9671         memcpy(mac_buf, &mac_hi, sizeof(mac_hi));
9672         memcpy(mac_buf + sizeof(mac_hi), &mac_lo, sizeof(mac_lo));
9673 }
9674
9675 static void __devinit bnx2x_get_port_hwinfo(struct bnx2x *bp)
9676 {
9677         int port = BP_PORT(bp);
9678         u32 val, val2;
9679         u32 config;
9680         u16 i;
9681         u32 ext_phy_type;
9682
9683         bp->link_params.bp = bp;
9684         bp->link_params.port = port;
9685
9686         bp->link_params.lane_config =
9687                 SHMEM_RD(bp, dev_info.port_hw_config[port].lane_config);
9688         bp->link_params.ext_phy_config =
9689                 SHMEM_RD(bp,
9690                          dev_info.port_hw_config[port].external_phy_config);
9691         /* BCM8727_NOC => BCM8727 no over current */
9692         if (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config) ==
9693             PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727_NOC) {
9694                 bp->link_params.ext_phy_config &=
9695                         ~PORT_HW_CFG_XGXS_EXT_PHY_TYPE_MASK;
9696                 bp->link_params.ext_phy_config |=
9697                         PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727;
9698                 bp->link_params.feature_config_flags |=
9699                         FEATURE_CONFIG_BCM8727_NOC;
9700         }
9701
9702         bp->link_params.speed_cap_mask =
9703                 SHMEM_RD(bp,
9704                          dev_info.port_hw_config[port].speed_capability_mask);
9705
9706         bp->port.link_config =
9707                 SHMEM_RD(bp, dev_info.port_feature_config[port].link_config);
9708
9709         /* Get the 4 lanes xgxs config rx and tx */
9710         for (i = 0; i < 2; i++) {
9711                 val = SHMEM_RD(bp,
9712                            dev_info.port_hw_config[port].xgxs_config_rx[i<<1]);
9713                 bp->link_params.xgxs_config_rx[i << 1] = ((val>>16) & 0xffff);
9714                 bp->link_params.xgxs_config_rx[(i << 1) + 1] = (val & 0xffff);
9715
9716                 val = SHMEM_RD(bp,
9717                            dev_info.port_hw_config[port].xgxs_config_tx[i<<1]);
9718                 bp->link_params.xgxs_config_tx[i << 1] = ((val>>16) & 0xffff);
9719                 bp->link_params.xgxs_config_tx[(i << 1) + 1] = (val & 0xffff);
9720         }
9721
9722         /* If the device is capable of WoL, set the default state according
9723          * to the HW
9724          */
9725         config = SHMEM_RD(bp, dev_info.port_feature_config[port].config);
9726         bp->wol = (!(bp->flags & NO_WOL_FLAG) &&
9727                    (config & PORT_FEATURE_WOL_ENABLED));
9728
9729         BNX2X_DEV_INFO("lane_config 0x%08x  ext_phy_config 0x%08x"
9730                        "  speed_cap_mask 0x%08x  link_config 0x%08x\n",
9731                        bp->link_params.lane_config,
9732                        bp->link_params.ext_phy_config,
9733                        bp->link_params.speed_cap_mask, bp->port.link_config);
9734
9735         bp->link_params.switch_cfg |= (bp->port.link_config &
9736                                        PORT_FEATURE_CONNECTED_SWITCH_MASK);
9737         bnx2x_link_settings_supported(bp, bp->link_params.switch_cfg);
9738
9739         bnx2x_link_settings_requested(bp);
9740
9741         /*
9742          * If connected directly, work with the internal PHY, otherwise, work
9743          * with the external PHY
9744          */
9745         ext_phy_type = XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
9746         if (ext_phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT)
9747                 bp->mdio.prtad = bp->link_params.phy_addr;
9748
9749         else if ((ext_phy_type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE) &&
9750                  (ext_phy_type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_NOT_CONN))
9751                 bp->mdio.prtad =
9752                         XGXS_EXT_PHY_ADDR(bp->link_params.ext_phy_config);
9753
9754         val2 = SHMEM_RD(bp, dev_info.port_hw_config[port].mac_upper);
9755         val = SHMEM_RD(bp, dev_info.port_hw_config[port].mac_lower);
9756         bnx2x_set_mac_buf(bp->dev->dev_addr, val, val2);
9757         memcpy(bp->link_params.mac_addr, bp->dev->dev_addr, ETH_ALEN);
9758         memcpy(bp->dev->perm_addr, bp->dev->dev_addr, ETH_ALEN);
9759
9760 #ifdef BCM_CNIC
9761         val2 = SHMEM_RD(bp, dev_info.port_hw_config[port].iscsi_mac_upper);
9762         val = SHMEM_RD(bp, dev_info.port_hw_config[port].iscsi_mac_lower);
9763         bnx2x_set_mac_buf(bp->iscsi_mac, val, val2);
9764 #endif
9765 }
9766
9767 static int __devinit bnx2x_get_hwinfo(struct bnx2x *bp)
9768 {
9769         int func = BP_FUNC(bp);
9770         u32 val, val2;
9771         int rc = 0;
9772
9773         bnx2x_get_common_hwinfo(bp);
9774
9775         bp->e1hov = 0;
9776         bp->e1hmf = 0;
9777         if (CHIP_IS_E1H(bp) && !BP_NOMCP(bp)) {
9778                 bp->mf_config =
9779                         SHMEM_RD(bp, mf_cfg.func_mf_config[func].config);
9780
9781                 val = (SHMEM_RD(bp, mf_cfg.func_mf_config[FUNC_0].e1hov_tag) &
9782                        FUNC_MF_CFG_E1HOV_TAG_MASK);
9783                 if (val != FUNC_MF_CFG_E1HOV_TAG_DEFAULT)
9784                         bp->e1hmf = 1;
9785                 BNX2X_DEV_INFO("%s function mode\n",
9786                                IS_E1HMF(bp) ? "multi" : "single");
9787
9788                 if (IS_E1HMF(bp)) {
9789                         val = (SHMEM_RD(bp, mf_cfg.func_mf_config[func].
9790                                                                 e1hov_tag) &
9791                                FUNC_MF_CFG_E1HOV_TAG_MASK);
9792                         if (val != FUNC_MF_CFG_E1HOV_TAG_DEFAULT) {
9793                                 bp->e1hov = val;
9794                                 BNX2X_DEV_INFO("E1HOV for func %d is %d "
9795                                                "(0x%04x)\n",
9796                                                func, bp->e1hov, bp->e1hov);
9797                         } else {
9798                                 BNX2X_ERROR("No valid E1HOV for func %d,"
9799                                             "  aborting\n", func);
9800                                 rc = -EPERM;
9801                         }
9802                 } else {
9803                         if (BP_E1HVN(bp)) {
9804                                 BNX2X_ERROR("VN %d in single function mode,"
9805                                             "  aborting\n", BP_E1HVN(bp));
9806                                 rc = -EPERM;
9807                         }
9808                 }
9809         }
9810
9811         if (!BP_NOMCP(bp)) {
9812                 bnx2x_get_port_hwinfo(bp);
9813
9814                 bp->fw_seq = (SHMEM_RD(bp, func_mb[func].drv_mb_header) &
9815                               DRV_MSG_SEQ_NUMBER_MASK);
9816                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
9817         }
9818
9819         if (IS_E1HMF(bp)) {
9820                 val2 = SHMEM_RD(bp, mf_cfg.func_mf_config[func].mac_upper);
9821                 val = SHMEM_RD(bp,  mf_cfg.func_mf_config[func].mac_lower);
9822                 if ((val2 != FUNC_MF_CFG_UPPERMAC_DEFAULT) &&
9823                     (val != FUNC_MF_CFG_LOWERMAC_DEFAULT)) {
9824                         bp->dev->dev_addr[0] = (u8)(val2 >> 8 & 0xff);
9825                         bp->dev->dev_addr[1] = (u8)(val2 & 0xff);
9826                         bp->dev->dev_addr[2] = (u8)(val >> 24 & 0xff);
9827                         bp->dev->dev_addr[3] = (u8)(val >> 16 & 0xff);
9828                         bp->dev->dev_addr[4] = (u8)(val >> 8  & 0xff);
9829                         bp->dev->dev_addr[5] = (u8)(val & 0xff);
9830                         memcpy(bp->link_params.mac_addr, bp->dev->dev_addr,
9831                                ETH_ALEN);
9832                         memcpy(bp->dev->perm_addr, bp->dev->dev_addr,
9833                                ETH_ALEN);
9834                 }
9835
9836                 return rc;
9837         }
9838
9839         if (BP_NOMCP(bp)) {
9840                 /* only supposed to happen on emulation/FPGA */
9841                 BNX2X_ERROR("warning: random MAC workaround active\n");
9842                 random_ether_addr(bp->dev->dev_addr);
9843                 memcpy(bp->dev->perm_addr, bp->dev->dev_addr, ETH_ALEN);
9844         }
9845
9846         return rc;
9847 }
9848
9849 static void __devinit bnx2x_read_fwinfo(struct bnx2x *bp)
9850 {
9851         int cnt, i, block_end, rodi;
9852         char vpd_data[BNX2X_VPD_LEN+1];
9853         char str_id_reg[VENDOR_ID_LEN+1];
9854         char str_id_cap[VENDOR_ID_LEN+1];
9855         u8 len;
9856
9857         cnt = pci_read_vpd(bp->pdev, 0, BNX2X_VPD_LEN, vpd_data);
9858         memset(bp->fw_ver, 0, sizeof(bp->fw_ver));
9859
9860         if (cnt < BNX2X_VPD_LEN)
9861                 goto out_not_found;
9862
9863         i = pci_vpd_find_tag(vpd_data, 0, BNX2X_VPD_LEN,
9864                              PCI_VPD_LRDT_RO_DATA);
9865         if (i < 0)
9866                 goto out_not_found;
9867
9868
9869         block_end = i + PCI_VPD_LRDT_TAG_SIZE +
9870                     pci_vpd_lrdt_size(&vpd_data[i]);
9871
9872         i += PCI_VPD_LRDT_TAG_SIZE;
9873
9874         if (block_end > BNX2X_VPD_LEN)
9875                 goto out_not_found;
9876
9877         rodi = pci_vpd_find_info_keyword(vpd_data, i, block_end,
9878                                    PCI_VPD_RO_KEYWORD_MFR_ID);
9879         if (rodi < 0)
9880                 goto out_not_found;
9881
9882         len = pci_vpd_info_field_size(&vpd_data[rodi]);
9883
9884         if (len != VENDOR_ID_LEN)
9885                 goto out_not_found;
9886
9887         rodi += PCI_VPD_INFO_FLD_HDR_SIZE;
9888
9889         /* vendor specific info */
9890         snprintf(str_id_reg, VENDOR_ID_LEN + 1, "%04x", PCI_VENDOR_ID_DELL);
9891         snprintf(str_id_cap, VENDOR_ID_LEN + 1, "%04X", PCI_VENDOR_ID_DELL);
9892         if (!strncmp(str_id_reg, &vpd_data[rodi], VENDOR_ID_LEN) ||
9893             !strncmp(str_id_cap, &vpd_data[rodi], VENDOR_ID_LEN)) {
9894
9895                 rodi = pci_vpd_find_info_keyword(vpd_data, i, block_end,
9896                                                 PCI_VPD_RO_KEYWORD_VENDOR0);
9897                 if (rodi >= 0) {
9898                         len = pci_vpd_info_field_size(&vpd_data[rodi]);
9899
9900                         rodi += PCI_VPD_INFO_FLD_HDR_SIZE;
9901
9902                         if (len < 32 && (len + rodi) <= BNX2X_VPD_LEN) {
9903                                 memcpy(bp->fw_ver, &vpd_data[rodi], len);
9904                                 bp->fw_ver[len] = ' ';
9905                         }
9906                 }
9907                 return;
9908         }
9909 out_not_found:
9910         return;
9911 }
9912
9913 static int __devinit bnx2x_init_bp(struct bnx2x *bp)
9914 {
9915         int func = BP_FUNC(bp);
9916         int timer_interval;
9917         int rc;
9918
9919         /* Disable interrupt handling until HW is initialized */
9920         atomic_set(&bp->intr_sem, 1);
9921         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
9922
9923         mutex_init(&bp->port.phy_mutex);
9924         mutex_init(&bp->fw_mb_mutex);
9925 #ifdef BCM_CNIC
9926         mutex_init(&bp->cnic_mutex);
9927 #endif
9928
9929         INIT_DELAYED_WORK(&bp->sp_task, bnx2x_sp_task);
9930         INIT_DELAYED_WORK(&bp->reset_task, bnx2x_reset_task);
9931
9932         rc = bnx2x_get_hwinfo(bp);
9933
9934         bnx2x_read_fwinfo(bp);
9935         /* need to reset chip if undi was active */
9936         if (!BP_NOMCP(bp))
9937                 bnx2x_undi_unload(bp);
9938
9939         if (CHIP_REV_IS_FPGA(bp))
9940                 dev_err(&bp->pdev->dev, "FPGA detected\n");
9941
9942         if (BP_NOMCP(bp) && (func == 0))
9943                 dev_err(&bp->pdev->dev, "MCP disabled, "
9944                                         "must load devices in order!\n");
9945
9946         /* Set multi queue mode */
9947         if ((multi_mode != ETH_RSS_MODE_DISABLED) &&
9948             ((int_mode == INT_MODE_INTx) || (int_mode == INT_MODE_MSI))) {
9949                 dev_err(&bp->pdev->dev, "Multi disabled since int_mode "
9950                                         "requested is not MSI-X\n");
9951                 multi_mode = ETH_RSS_MODE_DISABLED;
9952         }
9953         bp->multi_mode = multi_mode;
9954         bp->int_mode = int_mode;
9955
9956         bp->dev->features |= NETIF_F_GRO;
9957
9958         /* Set TPA flags */
9959         if (disable_tpa) {
9960                 bp->flags &= ~TPA_ENABLE_FLAG;
9961                 bp->dev->features &= ~NETIF_F_LRO;
9962         } else {
9963                 bp->flags |= TPA_ENABLE_FLAG;
9964                 bp->dev->features |= NETIF_F_LRO;
9965         }
9966         bp->disable_tpa = disable_tpa;
9967
9968         if (CHIP_IS_E1(bp))
9969                 bp->dropless_fc = 0;
9970         else
9971                 bp->dropless_fc = dropless_fc;
9972
9973         bp->mrrs = mrrs;
9974
9975         bp->tx_ring_size = MAX_TX_AVAIL;
9976         bp->rx_ring_size = MAX_RX_AVAIL;
9977
9978         bp->rx_csum = 1;
9979
9980         /* make sure that the numbers are in the right granularity */
9981         bp->tx_ticks = (50 / (4 * BNX2X_BTR)) * (4 * BNX2X_BTR);
9982         bp->rx_ticks = (25 / (4 * BNX2X_BTR)) * (4 * BNX2X_BTR);
9983
9984         timer_interval = (CHIP_REV_IS_SLOW(bp) ? 5*HZ : HZ);
9985         bp->current_interval = (poll ? poll : timer_interval);
9986
9987         init_timer(&bp->timer);
9988         bp->timer.expires = jiffies + bp->current_interval;
9989         bp->timer.data = (unsigned long) bp;
9990         bp->timer.function = bnx2x_timer;
9991
9992         return rc;
9993 }
9994
9995 /*
9996  * ethtool service functions
9997  */
9998
9999 /* All ethtool functions called with rtnl_lock */
10000
10001 static int bnx2x_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
10002 {
10003         struct bnx2x *bp = netdev_priv(dev);
10004
10005         cmd->supported = bp->port.supported;
10006         cmd->advertising = bp->port.advertising;
10007
10008         if ((bp->state == BNX2X_STATE_OPEN) &&
10009             !(bp->flags & MF_FUNC_DIS) &&
10010             (bp->link_vars.link_up)) {
10011                 cmd->speed = bp->link_vars.line_speed;
10012                 cmd->duplex = bp->link_vars.duplex;
10013                 if (IS_E1HMF(bp)) {
10014                         u16 vn_max_rate;
10015
10016                         vn_max_rate =
10017                                 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
10018                                 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
10019                         if (vn_max_rate < cmd->speed)
10020                                 cmd->speed = vn_max_rate;
10021                 }
10022         } else {
10023                 cmd->speed = -1;
10024                 cmd->duplex = -1;
10025         }
10026
10027         if (bp->link_params.switch_cfg == SWITCH_CFG_10G) {
10028                 u32 ext_phy_type =
10029                         XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
10030
10031                 switch (ext_phy_type) {
10032                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT:
10033                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8072:
10034                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8073:
10035                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8705:
10036                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8706:
10037                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8726:
10038                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8727:
10039                         cmd->port = PORT_FIBRE;
10040                         break;
10041
10042                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101:
10043                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8481:
10044                         cmd->port = PORT_TP;
10045                         break;
10046
10047                 case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE:
10048                         BNX2X_ERR("XGXS PHY Failure detected 0x%x\n",
10049                                   bp->link_params.ext_phy_config);
10050                         break;
10051
10052                 default:
10053                         DP(NETIF_MSG_LINK, "BAD XGXS ext_phy_config 0x%x\n",
10054                            bp->link_params.ext_phy_config);
10055                         break;
10056                 }
10057         } else
10058                 cmd->port = PORT_TP;
10059
10060         cmd->phy_address = bp->mdio.prtad;
10061         cmd->transceiver = XCVR_INTERNAL;
10062
10063         if (bp->link_params.req_line_speed == SPEED_AUTO_NEG)
10064                 cmd->autoneg = AUTONEG_ENABLE;
10065         else
10066                 cmd->autoneg = AUTONEG_DISABLE;
10067
10068         cmd->maxtxpkt = 0;
10069         cmd->maxrxpkt = 0;
10070
10071         DP(NETIF_MSG_LINK, "ethtool_cmd: cmd %d\n"
10072            DP_LEVEL "  supported 0x%x  advertising 0x%x  speed %d\n"
10073            DP_LEVEL "  duplex %d  port %d  phy_address %d  transceiver %d\n"
10074            DP_LEVEL "  autoneg %d  maxtxpkt %d  maxrxpkt %d\n",
10075            cmd->cmd, cmd->supported, cmd->advertising, cmd->speed,
10076            cmd->duplex, cmd->port, cmd->phy_address, cmd->transceiver,
10077            cmd->autoneg, cmd->maxtxpkt, cmd->maxrxpkt);
10078
10079         return 0;
10080 }
10081
10082 static int bnx2x_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
10083 {
10084         struct bnx2x *bp = netdev_priv(dev);
10085         u32 advertising;
10086
10087         if (IS_E1HMF(bp))
10088                 return 0;
10089
10090         DP(NETIF_MSG_LINK, "ethtool_cmd: cmd %d\n"
10091            DP_LEVEL "  supported 0x%x  advertising 0x%x  speed %d\n"
10092            DP_LEVEL "  duplex %d  port %d  phy_address %d  transceiver %d\n"
10093            DP_LEVEL "  autoneg %d  maxtxpkt %d  maxrxpkt %d\n",
10094            cmd->cmd, cmd->supported, cmd->advertising, cmd->speed,
10095            cmd->duplex, cmd->port, cmd->phy_address, cmd->transceiver,
10096            cmd->autoneg, cmd->maxtxpkt, cmd->maxrxpkt);
10097
10098         if (cmd->autoneg == AUTONEG_ENABLE) {
10099                 if (!(bp->port.supported & SUPPORTED_Autoneg)) {
10100                         DP(NETIF_MSG_LINK, "Autoneg not supported\n");
10101                         return -EINVAL;
10102                 }
10103
10104                 /* advertise the requested speed and duplex if supported */
10105                 cmd->advertising &= bp->port.supported;
10106
10107                 bp->link_params.req_line_speed = SPEED_AUTO_NEG;
10108                 bp->link_params.req_duplex = DUPLEX_FULL;
10109                 bp->port.advertising |= (ADVERTISED_Autoneg |
10110                                          cmd->advertising);
10111
10112         } else { /* forced speed */
10113                 /* advertise the requested speed and duplex if supported */
10114                 switch (cmd->speed) {
10115                 case SPEED_10:
10116                         if (cmd->duplex == DUPLEX_FULL) {
10117                                 if (!(bp->port.supported &
10118                                       SUPPORTED_10baseT_Full)) {
10119                                         DP(NETIF_MSG_LINK,
10120                                            "10M full not supported\n");
10121                                         return -EINVAL;
10122                                 }
10123
10124                                 advertising = (ADVERTISED_10baseT_Full |
10125                                                ADVERTISED_TP);
10126                         } else {
10127                                 if (!(bp->port.supported &
10128                                       SUPPORTED_10baseT_Half)) {
10129                                         DP(NETIF_MSG_LINK,
10130                                            "10M half not supported\n");
10131                                         return -EINVAL;
10132                                 }
10133
10134                                 advertising = (ADVERTISED_10baseT_Half |
10135                                                ADVERTISED_TP);
10136                         }
10137                         break;
10138
10139                 case SPEED_100:
10140                         if (cmd->duplex == DUPLEX_FULL) {
10141                                 if (!(bp->port.supported &
10142                                                 SUPPORTED_100baseT_Full)) {
10143                                         DP(NETIF_MSG_LINK,
10144                                            "100M full not supported\n");
10145                                         return -EINVAL;
10146                                 }
10147
10148                                 advertising = (ADVERTISED_100baseT_Full |
10149                                                ADVERTISED_TP);
10150                         } else {
10151                                 if (!(bp->port.supported &
10152                                                 SUPPORTED_100baseT_Half)) {
10153                                         DP(NETIF_MSG_LINK,
10154                                            "100M half not supported\n");
10155                                         return -EINVAL;
10156                                 }
10157
10158                                 advertising = (ADVERTISED_100baseT_Half |
10159                                                ADVERTISED_TP);
10160                         }
10161                         break;
10162
10163                 case SPEED_1000:
10164                         if (cmd->duplex != DUPLEX_FULL) {
10165                                 DP(NETIF_MSG_LINK, "1G half not supported\n");
10166                                 return -EINVAL;
10167                         }
10168
10169                         if (!(bp->port.supported & SUPPORTED_1000baseT_Full)) {
10170                                 DP(NETIF_MSG_LINK, "1G full not supported\n");
10171                                 return -EINVAL;
10172                         }
10173
10174                         advertising = (ADVERTISED_1000baseT_Full |
10175                                        ADVERTISED_TP);
10176                         break;
10177
10178                 case SPEED_2500:
10179                         if (cmd->duplex != DUPLEX_FULL) {
10180                                 DP(NETIF_MSG_LINK,
10181                                    "2.5G half not supported\n");
10182                                 return -EINVAL;
10183                         }
10184
10185                         if (!(bp->port.supported & SUPPORTED_2500baseX_Full)) {
10186                                 DP(NETIF_MSG_LINK,
10187                                    "2.5G full not supported\n");
10188                                 return -EINVAL;
10189                         }
10190
10191                         advertising = (ADVERTISED_2500baseX_Full |
10192                                        ADVERTISED_TP);
10193                         break;
10194
10195                 case SPEED_10000:
10196                         if (cmd->duplex != DUPLEX_FULL) {
10197                                 DP(NETIF_MSG_LINK, "10G half not supported\n");
10198                                 return -EINVAL;
10199                         }
10200
10201                         if (!(bp->port.supported & SUPPORTED_10000baseT_Full)) {
10202                                 DP(NETIF_MSG_LINK, "10G full not supported\n");
10203                                 return -EINVAL;
10204                         }
10205
10206                         advertising = (ADVERTISED_10000baseT_Full |
10207                                        ADVERTISED_FIBRE);
10208                         break;
10209
10210                 default:
10211                         DP(NETIF_MSG_LINK, "Unsupported speed\n");
10212                         return -EINVAL;
10213                 }
10214
10215                 bp->link_params.req_line_speed = cmd->speed;
10216                 bp->link_params.req_duplex = cmd->duplex;
10217                 bp->port.advertising = advertising;
10218         }
10219
10220         DP(NETIF_MSG_LINK, "req_line_speed %d\n"
10221            DP_LEVEL "  req_duplex %d  advertising 0x%x\n",
10222            bp->link_params.req_line_speed, bp->link_params.req_duplex,
10223            bp->port.advertising);
10224
10225         if (netif_running(dev)) {
10226                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
10227                 bnx2x_link_set(bp);
10228         }
10229
10230         return 0;
10231 }
10232
10233 #define IS_E1_ONLINE(info)      (((info) & RI_E1_ONLINE) == RI_E1_ONLINE)
10234 #define IS_E1H_ONLINE(info)     (((info) & RI_E1H_ONLINE) == RI_E1H_ONLINE)
10235
10236 static int bnx2x_get_regs_len(struct net_device *dev)
10237 {
10238         struct bnx2x *bp = netdev_priv(dev);
10239         int regdump_len = 0;
10240         int i;
10241
10242         if (CHIP_IS_E1(bp)) {
10243                 for (i = 0; i < REGS_COUNT; i++)
10244                         if (IS_E1_ONLINE(reg_addrs[i].info))
10245                                 regdump_len += reg_addrs[i].size;
10246
10247                 for (i = 0; i < WREGS_COUNT_E1; i++)
10248                         if (IS_E1_ONLINE(wreg_addrs_e1[i].info))
10249                                 regdump_len += wreg_addrs_e1[i].size *
10250                                         (1 + wreg_addrs_e1[i].read_regs_count);
10251
10252         } else { /* E1H */
10253                 for (i = 0; i < REGS_COUNT; i++)
10254                         if (IS_E1H_ONLINE(reg_addrs[i].info))
10255                                 regdump_len += reg_addrs[i].size;
10256
10257                 for (i = 0; i < WREGS_COUNT_E1H; i++)
10258                         if (IS_E1H_ONLINE(wreg_addrs_e1h[i].info))
10259                                 regdump_len += wreg_addrs_e1h[i].size *
10260                                         (1 + wreg_addrs_e1h[i].read_regs_count);
10261         }
10262         regdump_len *= 4;
10263         regdump_len += sizeof(struct dump_hdr);
10264
10265         return regdump_len;
10266 }
10267
10268 static void bnx2x_get_regs(struct net_device *dev,
10269                            struct ethtool_regs *regs, void *_p)
10270 {
10271         u32 *p = _p, i, j;
10272         struct bnx2x *bp = netdev_priv(dev);
10273         struct dump_hdr dump_hdr = {0};
10274
10275         regs->version = 0;
10276         memset(p, 0, regs->len);
10277
10278         if (!netif_running(bp->dev))
10279                 return;
10280
10281         dump_hdr.hdr_size = (sizeof(struct dump_hdr) / 4) - 1;
10282         dump_hdr.dump_sign = dump_sign_all;
10283         dump_hdr.xstorm_waitp = REG_RD(bp, XSTORM_WAITP_ADDR);
10284         dump_hdr.tstorm_waitp = REG_RD(bp, TSTORM_WAITP_ADDR);
10285         dump_hdr.ustorm_waitp = REG_RD(bp, USTORM_WAITP_ADDR);
10286         dump_hdr.cstorm_waitp = REG_RD(bp, CSTORM_WAITP_ADDR);
10287         dump_hdr.info = CHIP_IS_E1(bp) ? RI_E1_ONLINE : RI_E1H_ONLINE;
10288
10289         memcpy(p, &dump_hdr, sizeof(struct dump_hdr));
10290         p += dump_hdr.hdr_size + 1;
10291
10292         if (CHIP_IS_E1(bp)) {
10293                 for (i = 0; i < REGS_COUNT; i++)
10294                         if (IS_E1_ONLINE(reg_addrs[i].info))
10295                                 for (j = 0; j < reg_addrs[i].size; j++)
10296                                         *p++ = REG_RD(bp,
10297                                                       reg_addrs[i].addr + j*4);
10298
10299         } else { /* E1H */
10300                 for (i = 0; i < REGS_COUNT; i++)
10301                         if (IS_E1H_ONLINE(reg_addrs[i].info))
10302                                 for (j = 0; j < reg_addrs[i].size; j++)
10303                                         *p++ = REG_RD(bp,
10304                                                       reg_addrs[i].addr + j*4);
10305         }
10306 }
10307
10308 #define PHY_FW_VER_LEN                  10
10309
10310 static void bnx2x_get_drvinfo(struct net_device *dev,
10311                               struct ethtool_drvinfo *info)
10312 {
10313         struct bnx2x *bp = netdev_priv(dev);
10314         u8 phy_fw_ver[PHY_FW_VER_LEN];
10315
10316         strcpy(info->driver, DRV_MODULE_NAME);
10317         strcpy(info->version, DRV_MODULE_VERSION);
10318
10319         phy_fw_ver[0] = '\0';
10320         if (bp->port.pmf) {
10321                 bnx2x_acquire_phy_lock(bp);
10322                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
10323                                              (bp->state != BNX2X_STATE_CLOSED),
10324                                              phy_fw_ver, PHY_FW_VER_LEN);
10325                 bnx2x_release_phy_lock(bp);
10326         }
10327
10328         strncpy(info->fw_version, bp->fw_ver, 32);
10329         snprintf(info->fw_version + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
10330                  "bc %d.%d.%d%s%s",
10331                  (bp->common.bc_ver & 0xff0000) >> 16,
10332                  (bp->common.bc_ver & 0xff00) >> 8,
10333                  (bp->common.bc_ver & 0xff),
10334                  ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
10335         strcpy(info->bus_info, pci_name(bp->pdev));
10336         info->n_stats = BNX2X_NUM_STATS;
10337         info->testinfo_len = BNX2X_NUM_TESTS;
10338         info->eedump_len = bp->common.flash_size;
10339         info->regdump_len = bnx2x_get_regs_len(dev);
10340 }
10341
10342 static void bnx2x_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
10343 {
10344         struct bnx2x *bp = netdev_priv(dev);
10345
10346         if (bp->flags & NO_WOL_FLAG) {
10347                 wol->supported = 0;
10348                 wol->wolopts = 0;
10349         } else {
10350                 wol->supported = WAKE_MAGIC;
10351                 if (bp->wol)
10352                         wol->wolopts = WAKE_MAGIC;
10353                 else
10354                         wol->wolopts = 0;
10355         }
10356         memset(&wol->sopass, 0, sizeof(wol->sopass));
10357 }
10358
10359 static int bnx2x_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
10360 {
10361         struct bnx2x *bp = netdev_priv(dev);
10362
10363         if (wol->wolopts & ~WAKE_MAGIC)
10364                 return -EINVAL;
10365
10366         if (wol->wolopts & WAKE_MAGIC) {
10367                 if (bp->flags & NO_WOL_FLAG)
10368                         return -EINVAL;
10369
10370                 bp->wol = 1;
10371         } else
10372                 bp->wol = 0;
10373
10374         return 0;
10375 }
10376
10377 static u32 bnx2x_get_msglevel(struct net_device *dev)
10378 {
10379         struct bnx2x *bp = netdev_priv(dev);
10380
10381         return bp->msg_enable;
10382 }
10383
10384 static void bnx2x_set_msglevel(struct net_device *dev, u32 level)
10385 {
10386         struct bnx2x *bp = netdev_priv(dev);
10387
10388         if (capable(CAP_NET_ADMIN))
10389                 bp->msg_enable = level;
10390 }
10391
10392 static int bnx2x_nway_reset(struct net_device *dev)
10393 {
10394         struct bnx2x *bp = netdev_priv(dev);
10395
10396         if (!bp->port.pmf)
10397                 return 0;
10398
10399         if (netif_running(dev)) {
10400                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
10401                 bnx2x_link_set(bp);
10402         }
10403
10404         return 0;
10405 }
10406
10407 static u32 bnx2x_get_link(struct net_device *dev)
10408 {
10409         struct bnx2x *bp = netdev_priv(dev);
10410
10411         if (bp->flags & MF_FUNC_DIS)
10412                 return 0;
10413
10414         return bp->link_vars.link_up;
10415 }
10416
10417 static int bnx2x_get_eeprom_len(struct net_device *dev)
10418 {
10419         struct bnx2x *bp = netdev_priv(dev);
10420
10421         return bp->common.flash_size;
10422 }
10423
10424 static int bnx2x_acquire_nvram_lock(struct bnx2x *bp)
10425 {
10426         int port = BP_PORT(bp);
10427         int count, i;
10428         u32 val = 0;
10429
10430         /* adjust timeout for emulation/FPGA */
10431         count = NVRAM_TIMEOUT_COUNT;
10432         if (CHIP_REV_IS_SLOW(bp))
10433                 count *= 100;
10434
10435         /* request access to nvram interface */
10436         REG_WR(bp, MCP_REG_MCPR_NVM_SW_ARB,
10437                (MCPR_NVM_SW_ARB_ARB_REQ_SET1 << port));
10438
10439         for (i = 0; i < count*10; i++) {
10440                 val = REG_RD(bp, MCP_REG_MCPR_NVM_SW_ARB);
10441                 if (val & (MCPR_NVM_SW_ARB_ARB_ARB1 << port))
10442                         break;
10443
10444                 udelay(5);
10445         }
10446
10447         if (!(val & (MCPR_NVM_SW_ARB_ARB_ARB1 << port))) {
10448                 DP(BNX2X_MSG_NVM, "cannot get access to nvram interface\n");
10449                 return -EBUSY;
10450         }
10451
10452         return 0;
10453 }
10454
10455 static int bnx2x_release_nvram_lock(struct bnx2x *bp)
10456 {
10457         int port = BP_PORT(bp);
10458         int count, i;
10459         u32 val = 0;
10460
10461         /* adjust timeout for emulation/FPGA */
10462         count = NVRAM_TIMEOUT_COUNT;
10463         if (CHIP_REV_IS_SLOW(bp))
10464                 count *= 100;
10465
10466         /* relinquish nvram interface */
10467         REG_WR(bp, MCP_REG_MCPR_NVM_SW_ARB,
10468                (MCPR_NVM_SW_ARB_ARB_REQ_CLR1 << port));
10469
10470         for (i = 0; i < count*10; i++) {
10471                 val = REG_RD(bp, MCP_REG_MCPR_NVM_SW_ARB);
10472                 if (!(val & (MCPR_NVM_SW_ARB_ARB_ARB1 << port)))
10473                         break;
10474
10475                 udelay(5);
10476         }
10477
10478         if (val & (MCPR_NVM_SW_ARB_ARB_ARB1 << port)) {
10479                 DP(BNX2X_MSG_NVM, "cannot free access to nvram interface\n");
10480                 return -EBUSY;
10481         }
10482
10483         return 0;
10484 }
10485
10486 static void bnx2x_enable_nvram_access(struct bnx2x *bp)
10487 {
10488         u32 val;
10489
10490         val = REG_RD(bp, MCP_REG_MCPR_NVM_ACCESS_ENABLE);
10491
10492         /* enable both bits, even on read */
10493         REG_WR(bp, MCP_REG_MCPR_NVM_ACCESS_ENABLE,
10494                (val | MCPR_NVM_ACCESS_ENABLE_EN |
10495                       MCPR_NVM_ACCESS_ENABLE_WR_EN));
10496 }
10497
10498 static void bnx2x_disable_nvram_access(struct bnx2x *bp)
10499 {
10500         u32 val;
10501
10502         val = REG_RD(bp, MCP_REG_MCPR_NVM_ACCESS_ENABLE);
10503
10504         /* disable both bits, even after read */
10505         REG_WR(bp, MCP_REG_MCPR_NVM_ACCESS_ENABLE,
10506                (val & ~(MCPR_NVM_ACCESS_ENABLE_EN |
10507                         MCPR_NVM_ACCESS_ENABLE_WR_EN)));
10508 }
10509
10510 static int bnx2x_nvram_read_dword(struct bnx2x *bp, u32 offset, __be32 *ret_val,
10511                                   u32 cmd_flags)
10512 {
10513         int count, i, rc;
10514         u32 val;
10515
10516         /* build the command word */
10517         cmd_flags |= MCPR_NVM_COMMAND_DOIT;
10518
10519         /* need to clear DONE bit separately */
10520         REG_WR(bp, MCP_REG_MCPR_NVM_COMMAND, MCPR_NVM_COMMAND_DONE);
10521
10522         /* address of the NVRAM to read from */
10523         REG_WR(bp, MCP_REG_MCPR_NVM_ADDR,
10524                (offset & MCPR_NVM_ADDR_NVM_ADDR_VALUE));
10525
10526         /* issue a read command */
10527         REG_WR(bp, MCP_REG_MCPR_NVM_COMMAND, cmd_flags);
10528
10529         /* adjust timeout for emulation/FPGA */
10530         count = NVRAM_TIMEOUT_COUNT;
10531         if (CHIP_REV_IS_SLOW(bp))
10532                 count *= 100;
10533
10534         /* wait for completion */
10535         *ret_val = 0;
10536         rc = -EBUSY;
10537         for (i = 0; i < count; i++) {
10538                 udelay(5);
10539                 val = REG_RD(bp, MCP_REG_MCPR_NVM_COMMAND);
10540
10541                 if (val & MCPR_NVM_COMMAND_DONE) {
10542                         val = REG_RD(bp, MCP_REG_MCPR_NVM_READ);
10543                         /* we read nvram data in cpu order
10544                          * but ethtool sees it as an array of bytes
10545                          * converting to big-endian will do the work */
10546                         *ret_val = cpu_to_be32(val);
10547                         rc = 0;
10548                         break;
10549                 }
10550         }
10551
10552         return rc;
10553 }
10554
10555 static int bnx2x_nvram_read(struct bnx2x *bp, u32 offset, u8 *ret_buf,
10556                             int buf_size)
10557 {
10558         int rc;
10559         u32 cmd_flags;
10560         __be32 val;
10561
10562         if ((offset & 0x03) || (buf_size & 0x03) || (buf_size == 0)) {
10563                 DP(BNX2X_MSG_NVM,
10564                    "Invalid parameter: offset 0x%x  buf_size 0x%x\n",
10565                    offset, buf_size);
10566                 return -EINVAL;
10567         }
10568
10569         if (offset + buf_size > bp->common.flash_size) {
10570                 DP(BNX2X_MSG_NVM, "Invalid parameter: offset (0x%x) +"
10571                                   " buf_size (0x%x) > flash_size (0x%x)\n",
10572                    offset, buf_size, bp->common.flash_size);
10573                 return -EINVAL;
10574         }
10575
10576         /* request access to nvram interface */
10577         rc = bnx2x_acquire_nvram_lock(bp);
10578         if (rc)
10579                 return rc;
10580
10581         /* enable access to nvram interface */
10582         bnx2x_enable_nvram_access(bp);
10583
10584         /* read the first word(s) */
10585         cmd_flags = MCPR_NVM_COMMAND_FIRST;
10586         while ((buf_size > sizeof(u32)) && (rc == 0)) {
10587                 rc = bnx2x_nvram_read_dword(bp, offset, &val, cmd_flags);
10588                 memcpy(ret_buf, &val, 4);
10589
10590                 /* advance to the next dword */
10591                 offset += sizeof(u32);
10592                 ret_buf += sizeof(u32);
10593                 buf_size -= sizeof(u32);
10594                 cmd_flags = 0;
10595         }
10596
10597         if (rc == 0) {
10598                 cmd_flags |= MCPR_NVM_COMMAND_LAST;
10599                 rc = bnx2x_nvram_read_dword(bp, offset, &val, cmd_flags);
10600                 memcpy(ret_buf, &val, 4);
10601         }
10602
10603         /* disable access to nvram interface */
10604         bnx2x_disable_nvram_access(bp);
10605         bnx2x_release_nvram_lock(bp);
10606
10607         return rc;
10608 }
10609
10610 static int bnx2x_get_eeprom(struct net_device *dev,
10611                             struct ethtool_eeprom *eeprom, u8 *eebuf)
10612 {
10613         struct bnx2x *bp = netdev_priv(dev);
10614         int rc;
10615
10616         if (!netif_running(dev))
10617                 return -EAGAIN;
10618
10619         DP(BNX2X_MSG_NVM, "ethtool_eeprom: cmd %d\n"
10620            DP_LEVEL "  magic 0x%x  offset 0x%x (%d)  len 0x%x (%d)\n",
10621            eeprom->cmd, eeprom->magic, eeprom->offset, eeprom->offset,
10622            eeprom->len, eeprom->len);
10623
10624         /* parameters already validated in ethtool_get_eeprom */
10625
10626         rc = bnx2x_nvram_read(bp, eeprom->offset, eebuf, eeprom->len);
10627
10628         return rc;
10629 }
10630
10631 static int bnx2x_nvram_write_dword(struct bnx2x *bp, u32 offset, u32 val,
10632                                    u32 cmd_flags)
10633 {
10634         int count, i, rc;
10635
10636         /* build the command word */
10637         cmd_flags |= MCPR_NVM_COMMAND_DOIT | MCPR_NVM_COMMAND_WR;
10638
10639         /* need to clear DONE bit separately */
10640         REG_WR(bp, MCP_REG_MCPR_NVM_COMMAND, MCPR_NVM_COMMAND_DONE);
10641
10642         /* write the data */
10643         REG_WR(bp, MCP_REG_MCPR_NVM_WRITE, val);
10644
10645         /* address of the NVRAM to write to */
10646         REG_WR(bp, MCP_REG_MCPR_NVM_ADDR,
10647                (offset & MCPR_NVM_ADDR_NVM_ADDR_VALUE));
10648
10649         /* issue the write command */
10650         REG_WR(bp, MCP_REG_MCPR_NVM_COMMAND, cmd_flags);
10651
10652         /* adjust timeout for emulation/FPGA */
10653         count = NVRAM_TIMEOUT_COUNT;
10654         if (CHIP_REV_IS_SLOW(bp))
10655                 count *= 100;
10656
10657         /* wait for completion */
10658         rc = -EBUSY;
10659         for (i = 0; i < count; i++) {
10660                 udelay(5);
10661                 val = REG_RD(bp, MCP_REG_MCPR_NVM_COMMAND);
10662                 if (val & MCPR_NVM_COMMAND_DONE) {
10663                         rc = 0;
10664                         break;
10665                 }
10666         }
10667
10668         return rc;
10669 }
10670
10671 #define BYTE_OFFSET(offset)             (8 * (offset & 0x03))
10672
10673 static int bnx2x_nvram_write1(struct bnx2x *bp, u32 offset, u8 *data_buf,
10674                               int buf_size)
10675 {
10676         int rc;
10677         u32 cmd_flags;
10678         u32 align_offset;
10679         __be32 val;
10680
10681         if (offset + buf_size > bp->common.flash_size) {
10682                 DP(BNX2X_MSG_NVM, "Invalid parameter: offset (0x%x) +"
10683                                   " buf_size (0x%x) > flash_size (0x%x)\n",
10684                    offset, buf_size, bp->common.flash_size);
10685                 return -EINVAL;
10686         }
10687
10688         /* request access to nvram interface */
10689         rc = bnx2x_acquire_nvram_lock(bp);
10690         if (rc)
10691                 return rc;
10692
10693         /* enable access to nvram interface */
10694         bnx2x_enable_nvram_access(bp);
10695
10696         cmd_flags = (MCPR_NVM_COMMAND_FIRST | MCPR_NVM_COMMAND_LAST);
10697         align_offset = (offset & ~0x03);
10698         rc = bnx2x_nvram_read_dword(bp, align_offset, &val, cmd_flags);
10699
10700         if (rc == 0) {
10701                 val &= ~(0xff << BYTE_OFFSET(offset));
10702                 val |= (*data_buf << BYTE_OFFSET(offset));
10703
10704                 /* nvram data is returned as an array of bytes
10705                  * convert it back to cpu order */
10706                 val = be32_to_cpu(val);
10707
10708                 rc = bnx2x_nvram_write_dword(bp, align_offset, val,
10709                                              cmd_flags);
10710         }
10711
10712         /* disable access to nvram interface */
10713         bnx2x_disable_nvram_access(bp);
10714         bnx2x_release_nvram_lock(bp);
10715
10716         return rc;
10717 }
10718
10719 static int bnx2x_nvram_write(struct bnx2x *bp, u32 offset, u8 *data_buf,
10720                              int buf_size)
10721 {
10722         int rc;
10723         u32 cmd_flags;
10724         u32 val;
10725         u32 written_so_far;
10726
10727         if (buf_size == 1)      /* ethtool */
10728                 return bnx2x_nvram_write1(bp, offset, data_buf, buf_size);
10729
10730         if ((offset & 0x03) || (buf_size & 0x03) || (buf_size == 0)) {
10731                 DP(BNX2X_MSG_NVM,
10732                    "Invalid parameter: offset 0x%x  buf_size 0x%x\n",
10733                    offset, buf_size);
10734                 return -EINVAL;
10735         }
10736
10737         if (offset + buf_size > bp->common.flash_size) {
10738                 DP(BNX2X_MSG_NVM, "Invalid parameter: offset (0x%x) +"
10739                                   " buf_size (0x%x) > flash_size (0x%x)\n",
10740                    offset, buf_size, bp->common.flash_size);
10741                 return -EINVAL;
10742         }
10743
10744         /* request access to nvram interface */
10745         rc = bnx2x_acquire_nvram_lock(bp);
10746         if (rc)
10747                 return rc;
10748
10749         /* enable access to nvram interface */
10750         bnx2x_enable_nvram_access(bp);
10751
10752         written_so_far = 0;
10753         cmd_flags = MCPR_NVM_COMMAND_FIRST;
10754         while ((written_so_far < buf_size) && (rc == 0)) {
10755                 if (written_so_far == (buf_size - sizeof(u32)))
10756                         cmd_flags |= MCPR_NVM_COMMAND_LAST;
10757                 else if (((offset + 4) % NVRAM_PAGE_SIZE) == 0)
10758                         cmd_flags |= MCPR_NVM_COMMAND_LAST;
10759                 else if ((offset % NVRAM_PAGE_SIZE) == 0)
10760                         cmd_flags |= MCPR_NVM_COMMAND_FIRST;
10761
10762                 memcpy(&val, data_buf, 4);
10763
10764                 rc = bnx2x_nvram_write_dword(bp, offset, val, cmd_flags);
10765
10766                 /* advance to the next dword */
10767                 offset += sizeof(u32);
10768                 data_buf += sizeof(u32);
10769                 written_so_far += sizeof(u32);
10770                 cmd_flags = 0;
10771         }
10772
10773         /* disable access to nvram interface */
10774         bnx2x_disable_nvram_access(bp);
10775         bnx2x_release_nvram_lock(bp);
10776
10777         return rc;
10778 }
10779
10780 static int bnx2x_set_eeprom(struct net_device *dev,
10781                             struct ethtool_eeprom *eeprom, u8 *eebuf)
10782 {
10783         struct bnx2x *bp = netdev_priv(dev);
10784         int port = BP_PORT(bp);
10785         int rc = 0;
10786
10787         if (!netif_running(dev))
10788                 return -EAGAIN;
10789
10790         DP(BNX2X_MSG_NVM, "ethtool_eeprom: cmd %d\n"
10791            DP_LEVEL "  magic 0x%x  offset 0x%x (%d)  len 0x%x (%d)\n",
10792            eeprom->cmd, eeprom->magic, eeprom->offset, eeprom->offset,
10793            eeprom->len, eeprom->len);
10794
10795         /* parameters already validated in ethtool_set_eeprom */
10796
10797         /* PHY eeprom can be accessed only by the PMF */
10798         if ((eeprom->magic >= 0x50485900) && (eeprom->magic <= 0x504859FF) &&
10799             !bp->port.pmf)
10800                 return -EINVAL;
10801
10802         if (eeprom->magic == 0x50485950) {
10803                 /* 'PHYP' (0x50485950): prepare phy for FW upgrade */
10804                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
10805
10806                 bnx2x_acquire_phy_lock(bp);
10807                 rc |= bnx2x_link_reset(&bp->link_params,
10808                                        &bp->link_vars, 0);
10809                 if (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config) ==
10810                                         PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101)
10811                         bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_0,
10812                                        MISC_REGISTERS_GPIO_HIGH, port);
10813                 bnx2x_release_phy_lock(bp);
10814                 bnx2x_link_report(bp);
10815
10816         } else if (eeprom->magic == 0x50485952) {
10817                 /* 'PHYR' (0x50485952): re-init link after FW upgrade */
10818                 if (bp->state == BNX2X_STATE_OPEN) {
10819                         bnx2x_acquire_phy_lock(bp);
10820                         rc |= bnx2x_link_reset(&bp->link_params,
10821                                                &bp->link_vars, 1);
10822
10823                         rc |= bnx2x_phy_init(&bp->link_params,
10824                                              &bp->link_vars);
10825                         bnx2x_release_phy_lock(bp);
10826                         bnx2x_calc_fc_adv(bp);
10827                 }
10828         } else if (eeprom->magic == 0x53985943) {
10829                 /* 'PHYC' (0x53985943): PHY FW upgrade completed */
10830                 if (XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config) ==
10831                                        PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101) {
10832                         u8 ext_phy_addr =
10833                              XGXS_EXT_PHY_ADDR(bp->link_params.ext_phy_config);
10834
10835                         /* DSP Remove Download Mode */
10836                         bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_0,
10837                                        MISC_REGISTERS_GPIO_LOW, port);
10838
10839                         bnx2x_acquire_phy_lock(bp);
10840
10841                         bnx2x_sfx7101_sp_sw_reset(bp, port, ext_phy_addr);
10842
10843                         /* wait 0.5 sec to allow it to run */
10844                         msleep(500);
10845                         bnx2x_ext_phy_hw_reset(bp, port);
10846                         msleep(500);
10847                         bnx2x_release_phy_lock(bp);
10848                 }
10849         } else
10850                 rc = bnx2x_nvram_write(bp, eeprom->offset, eebuf, eeprom->len);
10851
10852         return rc;
10853 }
10854
10855 static int bnx2x_get_coalesce(struct net_device *dev,
10856                               struct ethtool_coalesce *coal)
10857 {
10858         struct bnx2x *bp = netdev_priv(dev);
10859
10860         memset(coal, 0, sizeof(struct ethtool_coalesce));
10861
10862         coal->rx_coalesce_usecs = bp->rx_ticks;
10863         coal->tx_coalesce_usecs = bp->tx_ticks;
10864
10865         return 0;
10866 }
10867
10868 static int bnx2x_set_coalesce(struct net_device *dev,
10869                               struct ethtool_coalesce *coal)
10870 {
10871         struct bnx2x *bp = netdev_priv(dev);
10872
10873         bp->rx_ticks = (u16)coal->rx_coalesce_usecs;
10874         if (bp->rx_ticks > BNX2X_MAX_COALESCE_TOUT)
10875                 bp->rx_ticks = BNX2X_MAX_COALESCE_TOUT;
10876
10877         bp->tx_ticks = (u16)coal->tx_coalesce_usecs;
10878         if (bp->tx_ticks > BNX2X_MAX_COALESCE_TOUT)
10879                 bp->tx_ticks = BNX2X_MAX_COALESCE_TOUT;
10880
10881         if (netif_running(dev))
10882                 bnx2x_update_coalesce(bp);
10883
10884         return 0;
10885 }
10886
10887 static void bnx2x_get_ringparam(struct net_device *dev,
10888                                 struct ethtool_ringparam *ering)
10889 {
10890         struct bnx2x *bp = netdev_priv(dev);
10891
10892         ering->rx_max_pending = MAX_RX_AVAIL;
10893         ering->rx_mini_max_pending = 0;
10894         ering->rx_jumbo_max_pending = 0;
10895
10896         ering->rx_pending = bp->rx_ring_size;
10897         ering->rx_mini_pending = 0;
10898         ering->rx_jumbo_pending = 0;
10899
10900         ering->tx_max_pending = MAX_TX_AVAIL;
10901         ering->tx_pending = bp->tx_ring_size;
10902 }
10903
10904 static int bnx2x_set_ringparam(struct net_device *dev,
10905                                struct ethtool_ringparam *ering)
10906 {
10907         struct bnx2x *bp = netdev_priv(dev);
10908         int rc = 0;
10909
10910         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
10911                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
10912                 return -EAGAIN;
10913         }
10914
10915         if ((ering->rx_pending > MAX_RX_AVAIL) ||
10916             (ering->tx_pending > MAX_TX_AVAIL) ||
10917             (ering->tx_pending <= MAX_SKB_FRAGS + 4))
10918                 return -EINVAL;
10919
10920         bp->rx_ring_size = ering->rx_pending;
10921         bp->tx_ring_size = ering->tx_pending;
10922
10923         if (netif_running(dev)) {
10924                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
10925                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
10926         }
10927
10928         return rc;
10929 }
10930
10931 static void bnx2x_get_pauseparam(struct net_device *dev,
10932                                  struct ethtool_pauseparam *epause)
10933 {
10934         struct bnx2x *bp = netdev_priv(dev);
10935
10936         epause->autoneg = (bp->link_params.req_flow_ctrl ==
10937                            BNX2X_FLOW_CTRL_AUTO) &&
10938                           (bp->link_params.req_line_speed == SPEED_AUTO_NEG);
10939
10940         epause->rx_pause = ((bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) ==
10941                             BNX2X_FLOW_CTRL_RX);
10942         epause->tx_pause = ((bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX) ==
10943                             BNX2X_FLOW_CTRL_TX);
10944
10945         DP(NETIF_MSG_LINK, "ethtool_pauseparam: cmd %d\n"
10946            DP_LEVEL "  autoneg %d  rx_pause %d  tx_pause %d\n",
10947            epause->cmd, epause->autoneg, epause->rx_pause, epause->tx_pause);
10948 }
10949
10950 static int bnx2x_set_pauseparam(struct net_device *dev,
10951                                 struct ethtool_pauseparam *epause)
10952 {
10953         struct bnx2x *bp = netdev_priv(dev);
10954
10955         if (IS_E1HMF(bp))
10956                 return 0;
10957
10958         DP(NETIF_MSG_LINK, "ethtool_pauseparam: cmd %d\n"
10959            DP_LEVEL "  autoneg %d  rx_pause %d  tx_pause %d\n",
10960            epause->cmd, epause->autoneg, epause->rx_pause, epause->tx_pause);
10961
10962         bp->link_params.req_flow_ctrl = BNX2X_FLOW_CTRL_AUTO;
10963
10964         if (epause->rx_pause)
10965                 bp->link_params.req_flow_ctrl |= BNX2X_FLOW_CTRL_RX;
10966
10967         if (epause->tx_pause)
10968                 bp->link_params.req_flow_ctrl |= BNX2X_FLOW_CTRL_TX;
10969
10970         if (bp->link_params.req_flow_ctrl == BNX2X_FLOW_CTRL_AUTO)
10971                 bp->link_params.req_flow_ctrl = BNX2X_FLOW_CTRL_NONE;
10972
10973         if (epause->autoneg) {
10974                 if (!(bp->port.supported & SUPPORTED_Autoneg)) {
10975                         DP(NETIF_MSG_LINK, "autoneg not supported\n");
10976                         return -EINVAL;
10977                 }
10978
10979                 if (bp->link_params.req_line_speed == SPEED_AUTO_NEG)
10980                         bp->link_params.req_flow_ctrl = BNX2X_FLOW_CTRL_AUTO;
10981         }
10982
10983         DP(NETIF_MSG_LINK,
10984            "req_flow_ctrl 0x%x\n", bp->link_params.req_flow_ctrl);
10985
10986         if (netif_running(dev)) {
10987                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
10988                 bnx2x_link_set(bp);
10989         }
10990
10991         return 0;
10992 }
10993
10994 static int bnx2x_set_flags(struct net_device *dev, u32 data)
10995 {
10996         struct bnx2x *bp = netdev_priv(dev);
10997         int changed = 0;
10998         int rc = 0;
10999
11000         if (data & ~(ETH_FLAG_LRO | ETH_FLAG_RXHASH))
11001                 return -EINVAL;
11002
11003         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
11004                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
11005                 return -EAGAIN;
11006         }
11007
11008         /* TPA requires Rx CSUM offloading */
11009         if ((data & ETH_FLAG_LRO) && bp->rx_csum) {
11010                 if (!bp->disable_tpa) {
11011                         if (!(dev->features & NETIF_F_LRO)) {
11012                                 dev->features |= NETIF_F_LRO;
11013                                 bp->flags |= TPA_ENABLE_FLAG;
11014                                 changed = 1;
11015                         }
11016                 } else
11017                         rc = -EINVAL;
11018         } else if (dev->features & NETIF_F_LRO) {
11019                 dev->features &= ~NETIF_F_LRO;
11020                 bp->flags &= ~TPA_ENABLE_FLAG;
11021                 changed = 1;
11022         }
11023
11024         if (data & ETH_FLAG_RXHASH)
11025                 dev->features |= NETIF_F_RXHASH;
11026         else
11027                 dev->features &= ~NETIF_F_RXHASH;
11028
11029         if (changed && netif_running(dev)) {
11030                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
11031                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
11032         }
11033
11034         return rc;
11035 }
11036
11037 static u32 bnx2x_get_rx_csum(struct net_device *dev)
11038 {
11039         struct bnx2x *bp = netdev_priv(dev);
11040
11041         return bp->rx_csum;
11042 }
11043
11044 static int bnx2x_set_rx_csum(struct net_device *dev, u32 data)
11045 {
11046         struct bnx2x *bp = netdev_priv(dev);
11047         int rc = 0;
11048
11049         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
11050                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
11051                 return -EAGAIN;
11052         }
11053
11054         bp->rx_csum = data;
11055
11056         /* Disable TPA, when Rx CSUM is disabled. Otherwise all
11057            TPA'ed packets will be discarded due to wrong TCP CSUM */
11058         if (!data) {
11059                 u32 flags = ethtool_op_get_flags(dev);
11060
11061                 rc = bnx2x_set_flags(dev, (flags & ~ETH_FLAG_LRO));
11062         }
11063
11064         return rc;
11065 }
11066
11067 static int bnx2x_set_tso(struct net_device *dev, u32 data)
11068 {
11069         if (data) {
11070                 dev->features |= (NETIF_F_TSO | NETIF_F_TSO_ECN);
11071                 dev->features |= NETIF_F_TSO6;
11072         } else {
11073                 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO_ECN);
11074                 dev->features &= ~NETIF_F_TSO6;
11075         }
11076
11077         return 0;
11078 }
11079
11080 static const struct {
11081         char string[ETH_GSTRING_LEN];
11082 } bnx2x_tests_str_arr[BNX2X_NUM_TESTS] = {
11083         { "register_test (offline)" },
11084         { "memory_test (offline)" },
11085         { "loopback_test (offline)" },
11086         { "nvram_test (online)" },
11087         { "interrupt_test (online)" },
11088         { "link_test (online)" },
11089         { "idle check (online)" }
11090 };
11091
11092 static int bnx2x_test_registers(struct bnx2x *bp)
11093 {
11094         int idx, i, rc = -ENODEV;
11095         u32 wr_val = 0;
11096         int port = BP_PORT(bp);
11097         static const struct {
11098                 u32 offset0;
11099                 u32 offset1;
11100                 u32 mask;
11101         } reg_tbl[] = {
11102 /* 0 */         { BRB1_REG_PAUSE_LOW_THRESHOLD_0,      4, 0x000003ff },
11103                 { DORQ_REG_DB_ADDR0,                   4, 0xffffffff },
11104                 { HC_REG_AGG_INT_0,                    4, 0x000003ff },
11105                 { PBF_REG_MAC_IF0_ENABLE,              4, 0x00000001 },
11106                 { PBF_REG_P0_INIT_CRD,                 4, 0x000007ff },
11107                 { PRS_REG_CID_PORT_0,                  4, 0x00ffffff },
11108                 { PXP2_REG_PSWRQ_CDU0_L2P,             4, 0x000fffff },
11109                 { PXP2_REG_RQ_CDU0_EFIRST_MEM_ADDR,    8, 0x0003ffff },
11110                 { PXP2_REG_PSWRQ_TM0_L2P,              4, 0x000fffff },
11111                 { PXP2_REG_RQ_USDM0_EFIRST_MEM_ADDR,   8, 0x0003ffff },
11112 /* 10 */        { PXP2_REG_PSWRQ_TSDM0_L2P,            4, 0x000fffff },
11113                 { QM_REG_CONNNUM_0,                    4, 0x000fffff },
11114                 { TM_REG_LIN0_MAX_ACTIVE_CID,          4, 0x0003ffff },
11115                 { SRC_REG_KEYRSS0_0,                  40, 0xffffffff },
11116                 { SRC_REG_KEYRSS0_7,                  40, 0xffffffff },
11117                 { XCM_REG_WU_DA_SET_TMR_CNT_FLG_CMD00, 4, 0x00000001 },
11118                 { XCM_REG_WU_DA_CNT_CMD00,             4, 0x00000003 },
11119                 { XCM_REG_GLB_DEL_ACK_MAX_CNT_0,       4, 0x000000ff },
11120                 { NIG_REG_LLH0_T_BIT,                  4, 0x00000001 },
11121                 { NIG_REG_EMAC0_IN_EN,                 4, 0x00000001 },
11122 /* 20 */        { NIG_REG_BMAC0_IN_EN,                 4, 0x00000001 },
11123                 { NIG_REG_XCM0_OUT_EN,                 4, 0x00000001 },
11124                 { NIG_REG_BRB0_OUT_EN,                 4, 0x00000001 },
11125                 { NIG_REG_LLH0_XCM_MASK,               4, 0x00000007 },
11126                 { NIG_REG_LLH0_ACPI_PAT_6_LEN,        68, 0x000000ff },
11127                 { NIG_REG_LLH0_ACPI_PAT_0_CRC,        68, 0xffffffff },
11128                 { NIG_REG_LLH0_DEST_MAC_0_0,         160, 0xffffffff },
11129                 { NIG_REG_LLH0_DEST_IP_0_1,          160, 0xffffffff },
11130                 { NIG_REG_LLH0_IPV4_IPV6_0,          160, 0x00000001 },
11131                 { NIG_REG_LLH0_DEST_UDP_0,           160, 0x0000ffff },
11132 /* 30 */        { NIG_REG_LLH0_DEST_TCP_0,           160, 0x0000ffff },
11133                 { NIG_REG_LLH0_VLAN_ID_0,            160, 0x00000fff },
11134                 { NIG_REG_XGXS_SERDES0_MODE_SEL,       4, 0x00000001 },
11135                 { NIG_REG_LED_CONTROL_OVERRIDE_TRAFFIC_P0, 4, 0x00000001 },
11136                 { NIG_REG_STATUS_INTERRUPT_PORT0,      4, 0x07ffffff },
11137                 { NIG_REG_XGXS0_CTRL_EXTREMOTEMDIOST, 24, 0x00000001 },
11138                 { NIG_REG_SERDES0_CTRL_PHY_ADDR,      16, 0x0000001f },
11139
11140                 { 0xffffffff, 0, 0x00000000 }
11141         };
11142
11143         if (!netif_running(bp->dev))
11144                 return rc;
11145
11146         /* Repeat the test twice:
11147            First by writing 0x00000000, second by writing 0xffffffff */
11148         for (idx = 0; idx < 2; idx++) {
11149
11150                 switch (idx) {
11151                 case 0:
11152                         wr_val = 0;
11153                         break;
11154                 case 1:
11155                         wr_val = 0xffffffff;
11156                         break;
11157                 }
11158
11159                 for (i = 0; reg_tbl[i].offset0 != 0xffffffff; i++) {
11160                         u32 offset, mask, save_val, val;
11161
11162                         offset = reg_tbl[i].offset0 + port*reg_tbl[i].offset1;
11163                         mask = reg_tbl[i].mask;
11164
11165                         save_val = REG_RD(bp, offset);
11166
11167                         REG_WR(bp, offset, (wr_val & mask));
11168                         val = REG_RD(bp, offset);
11169
11170                         /* Restore the original register's value */
11171                         REG_WR(bp, offset, save_val);
11172
11173                         /* verify value is as expected */
11174                         if ((val & mask) != (wr_val & mask)) {
11175                                 DP(NETIF_MSG_PROBE,
11176                                    "offset 0x%x: val 0x%x != 0x%x mask 0x%x\n",
11177                                    offset, val, wr_val, mask);
11178                                 goto test_reg_exit;
11179                         }
11180                 }
11181         }
11182
11183         rc = 0;
11184
11185 test_reg_exit:
11186         return rc;
11187 }
11188
11189 static int bnx2x_test_memory(struct bnx2x *bp)
11190 {
11191         int i, j, rc = -ENODEV;
11192         u32 val;
11193         static const struct {
11194                 u32 offset;
11195                 int size;
11196         } mem_tbl[] = {
11197                 { CCM_REG_XX_DESCR_TABLE,   CCM_REG_XX_DESCR_TABLE_SIZE },
11198                 { CFC_REG_ACTIVITY_COUNTER, CFC_REG_ACTIVITY_COUNTER_SIZE },
11199                 { CFC_REG_LINK_LIST,        CFC_REG_LINK_LIST_SIZE },
11200                 { DMAE_REG_CMD_MEM,         DMAE_REG_CMD_MEM_SIZE },
11201                 { TCM_REG_XX_DESCR_TABLE,   TCM_REG_XX_DESCR_TABLE_SIZE },
11202                 { UCM_REG_XX_DESCR_TABLE,   UCM_REG_XX_DESCR_TABLE_SIZE },
11203                 { XCM_REG_XX_DESCR_TABLE,   XCM_REG_XX_DESCR_TABLE_SIZE },
11204
11205                 { 0xffffffff, 0 }
11206         };
11207         static const struct {
11208                 char *name;
11209                 u32 offset;
11210                 u32 e1_mask;
11211                 u32 e1h_mask;
11212         } prty_tbl[] = {
11213                 { "CCM_PRTY_STS",  CCM_REG_CCM_PRTY_STS,   0x3ffc0, 0 },
11214                 { "CFC_PRTY_STS",  CFC_REG_CFC_PRTY_STS,   0x2,     0x2 },
11215                 { "DMAE_PRTY_STS", DMAE_REG_DMAE_PRTY_STS, 0,       0 },
11216                 { "TCM_PRTY_STS",  TCM_REG_TCM_PRTY_STS,   0x3ffc0, 0 },
11217                 { "UCM_PRTY_STS",  UCM_REG_UCM_PRTY_STS,   0x3ffc0, 0 },
11218                 { "XCM_PRTY_STS",  XCM_REG_XCM_PRTY_STS,   0x3ffc1, 0 },
11219
11220                 { NULL, 0xffffffff, 0, 0 }
11221         };
11222
11223         if (!netif_running(bp->dev))
11224                 return rc;
11225
11226         /* Go through all the memories */
11227         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++)
11228                 for (j = 0; j < mem_tbl[i].size; j++)
11229                         REG_RD(bp, mem_tbl[i].offset + j*4);
11230
11231         /* Check the parity status */
11232         for (i = 0; prty_tbl[i].offset != 0xffffffff; i++) {
11233                 val = REG_RD(bp, prty_tbl[i].offset);
11234                 if ((CHIP_IS_E1(bp) && (val & ~(prty_tbl[i].e1_mask))) ||
11235                     (CHIP_IS_E1H(bp) && (val & ~(prty_tbl[i].e1h_mask)))) {
11236                         DP(NETIF_MSG_HW,
11237                            "%s is 0x%x\n", prty_tbl[i].name, val);
11238                         goto test_mem_exit;
11239                 }
11240         }
11241
11242         rc = 0;
11243
11244 test_mem_exit:
11245         return rc;
11246 }
11247
11248 static void bnx2x_wait_for_link(struct bnx2x *bp, u8 link_up)
11249 {
11250         int cnt = 1000;
11251
11252         if (link_up)
11253                 while (bnx2x_link_test(bp) && cnt--)
11254                         msleep(10);
11255 }
11256
11257 static int bnx2x_run_loopback(struct bnx2x *bp, int loopback_mode, u8 link_up)
11258 {
11259         unsigned int pkt_size, num_pkts, i;
11260         struct sk_buff *skb;
11261         unsigned char *packet;
11262         struct bnx2x_fastpath *fp_rx = &bp->fp[0];
11263         struct bnx2x_fastpath *fp_tx = &bp->fp[0];
11264         u16 tx_start_idx, tx_idx;
11265         u16 rx_start_idx, rx_idx;
11266         u16 pkt_prod, bd_prod;
11267         struct sw_tx_bd *tx_buf;
11268         struct eth_tx_start_bd *tx_start_bd;
11269         struct eth_tx_parse_bd *pbd = NULL;
11270         dma_addr_t mapping;
11271         union eth_rx_cqe *cqe;
11272         u8 cqe_fp_flags;
11273         struct sw_rx_bd *rx_buf;
11274         u16 len;
11275         int rc = -ENODEV;
11276
11277         /* check the loopback mode */
11278         switch (loopback_mode) {
11279         case BNX2X_PHY_LOOPBACK:
11280                 if (bp->link_params.loopback_mode != LOOPBACK_XGXS_10)
11281                         return -EINVAL;
11282                 break;
11283         case BNX2X_MAC_LOOPBACK:
11284                 bp->link_params.loopback_mode = LOOPBACK_BMAC;
11285                 bnx2x_phy_init(&bp->link_params, &bp->link_vars);
11286                 break;
11287         default:
11288                 return -EINVAL;
11289         }
11290
11291         /* prepare the loopback packet */
11292         pkt_size = (((bp->dev->mtu < ETH_MAX_PACKET_SIZE) ?
11293                      bp->dev->mtu : ETH_MAX_PACKET_SIZE) + ETH_HLEN);
11294         skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
11295         if (!skb) {
11296                 rc = -ENOMEM;
11297                 goto test_loopback_exit;
11298         }
11299         packet = skb_put(skb, pkt_size);
11300         memcpy(packet, bp->dev->dev_addr, ETH_ALEN);
11301         memset(packet + ETH_ALEN, 0, ETH_ALEN);
11302         memset(packet + 2*ETH_ALEN, 0x77, (ETH_HLEN - 2*ETH_ALEN));
11303         for (i = ETH_HLEN; i < pkt_size; i++)
11304                 packet[i] = (unsigned char) (i & 0xff);
11305
11306         /* send the loopback packet */
11307         num_pkts = 0;
11308         tx_start_idx = le16_to_cpu(*fp_tx->tx_cons_sb);
11309         rx_start_idx = le16_to_cpu(*fp_rx->rx_cons_sb);
11310
11311         pkt_prod = fp_tx->tx_pkt_prod++;
11312         tx_buf = &fp_tx->tx_buf_ring[TX_BD(pkt_prod)];
11313         tx_buf->first_bd = fp_tx->tx_bd_prod;
11314         tx_buf->skb = skb;
11315         tx_buf->flags = 0;
11316
11317         bd_prod = TX_BD(fp_tx->tx_bd_prod);
11318         tx_start_bd = &fp_tx->tx_desc_ring[bd_prod].start_bd;
11319         mapping = dma_map_single(&bp->pdev->dev, skb->data,
11320                                  skb_headlen(skb), DMA_TO_DEVICE);
11321         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
11322         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
11323         tx_start_bd->nbd = cpu_to_le16(2); /* start + pbd */
11324         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
11325         tx_start_bd->vlan = cpu_to_le16(pkt_prod);
11326         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
11327         tx_start_bd->general_data = ((UNICAST_ADDRESS <<
11328                                 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT) | 1);
11329
11330         /* turn on parsing and get a BD */
11331         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
11332         pbd = &fp_tx->tx_desc_ring[bd_prod].parse_bd;
11333
11334         memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
11335
11336         wmb();
11337
11338         fp_tx->tx_db.data.prod += 2;
11339         barrier();
11340         DOORBELL(bp, fp_tx->index, fp_tx->tx_db.raw);
11341
11342         mmiowb();
11343
11344         num_pkts++;
11345         fp_tx->tx_bd_prod += 2; /* start + pbd */
11346
11347         udelay(100);
11348
11349         tx_idx = le16_to_cpu(*fp_tx->tx_cons_sb);
11350         if (tx_idx != tx_start_idx + num_pkts)
11351                 goto test_loopback_exit;
11352
11353         rx_idx = le16_to_cpu(*fp_rx->rx_cons_sb);
11354         if (rx_idx != rx_start_idx + num_pkts)
11355                 goto test_loopback_exit;
11356
11357         cqe = &fp_rx->rx_comp_ring[RCQ_BD(fp_rx->rx_comp_cons)];
11358         cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
11359         if (CQE_TYPE(cqe_fp_flags) || (cqe_fp_flags & ETH_RX_ERROR_FALGS))
11360                 goto test_loopback_rx_exit;
11361
11362         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
11363         if (len != pkt_size)
11364                 goto test_loopback_rx_exit;
11365
11366         rx_buf = &fp_rx->rx_buf_ring[RX_BD(fp_rx->rx_bd_cons)];
11367         skb = rx_buf->skb;
11368         skb_reserve(skb, cqe->fast_path_cqe.placement_offset);
11369         for (i = ETH_HLEN; i < pkt_size; i++)
11370                 if (*(skb->data + i) != (unsigned char) (i & 0xff))
11371                         goto test_loopback_rx_exit;
11372
11373         rc = 0;
11374
11375 test_loopback_rx_exit:
11376
11377         fp_rx->rx_bd_cons = NEXT_RX_IDX(fp_rx->rx_bd_cons);
11378         fp_rx->rx_bd_prod = NEXT_RX_IDX(fp_rx->rx_bd_prod);
11379         fp_rx->rx_comp_cons = NEXT_RCQ_IDX(fp_rx->rx_comp_cons);
11380         fp_rx->rx_comp_prod = NEXT_RCQ_IDX(fp_rx->rx_comp_prod);
11381
11382         /* Update producers */
11383         bnx2x_update_rx_prod(bp, fp_rx, fp_rx->rx_bd_prod, fp_rx->rx_comp_prod,
11384                              fp_rx->rx_sge_prod);
11385
11386 test_loopback_exit:
11387         bp->link_params.loopback_mode = LOOPBACK_NONE;
11388
11389         return rc;
11390 }
11391
11392 static int bnx2x_test_loopback(struct bnx2x *bp, u8 link_up)
11393 {
11394         int rc = 0, res;
11395
11396         if (BP_NOMCP(bp))
11397                 return rc;
11398
11399         if (!netif_running(bp->dev))
11400                 return BNX2X_LOOPBACK_FAILED;
11401
11402         bnx2x_netif_stop(bp, 1);
11403         bnx2x_acquire_phy_lock(bp);
11404
11405         res = bnx2x_run_loopback(bp, BNX2X_PHY_LOOPBACK, link_up);
11406         if (res) {
11407                 DP(NETIF_MSG_PROBE, "  PHY loopback failed  (res %d)\n", res);
11408                 rc |= BNX2X_PHY_LOOPBACK_FAILED;
11409         }
11410
11411         res = bnx2x_run_loopback(bp, BNX2X_MAC_LOOPBACK, link_up);
11412         if (res) {
11413                 DP(NETIF_MSG_PROBE, "  MAC loopback failed  (res %d)\n", res);
11414                 rc |= BNX2X_MAC_LOOPBACK_FAILED;
11415         }
11416
11417         bnx2x_release_phy_lock(bp);
11418         bnx2x_netif_start(bp);
11419
11420         return rc;
11421 }
11422
11423 #define CRC32_RESIDUAL                  0xdebb20e3
11424
11425 static int bnx2x_test_nvram(struct bnx2x *bp)
11426 {
11427         static const struct {
11428                 int offset;
11429                 int size;
11430         } nvram_tbl[] = {
11431                 {     0,  0x14 }, /* bootstrap */
11432                 {  0x14,  0xec }, /* dir */
11433                 { 0x100, 0x350 }, /* manuf_info */
11434                 { 0x450,  0xf0 }, /* feature_info */
11435                 { 0x640,  0x64 }, /* upgrade_key_info */
11436                 { 0x6a4,  0x64 },
11437                 { 0x708,  0x70 }, /* manuf_key_info */
11438                 { 0x778,  0x70 },
11439                 {     0,     0 }
11440         };
11441         __be32 buf[0x350 / 4];
11442         u8 *data = (u8 *)buf;
11443         int i, rc;
11444         u32 magic, crc;
11445
11446         if (BP_NOMCP(bp))
11447                 return 0;
11448
11449         rc = bnx2x_nvram_read(bp, 0, data, 4);
11450         if (rc) {
11451                 DP(NETIF_MSG_PROBE, "magic value read (rc %d)\n", rc);
11452                 goto test_nvram_exit;
11453         }
11454
11455         magic = be32_to_cpu(buf[0]);
11456         if (magic != 0x669955aa) {
11457                 DP(NETIF_MSG_PROBE, "magic value (0x%08x)\n", magic);
11458                 rc = -ENODEV;
11459                 goto test_nvram_exit;
11460         }
11461
11462         for (i = 0; nvram_tbl[i].size; i++) {
11463
11464                 rc = bnx2x_nvram_read(bp, nvram_tbl[i].offset, data,
11465                                       nvram_tbl[i].size);
11466                 if (rc) {
11467                         DP(NETIF_MSG_PROBE,
11468                            "nvram_tbl[%d] read data (rc %d)\n", i, rc);
11469                         goto test_nvram_exit;
11470                 }
11471
11472                 crc = ether_crc_le(nvram_tbl[i].size, data);
11473                 if (crc != CRC32_RESIDUAL) {
11474                         DP(NETIF_MSG_PROBE,
11475                            "nvram_tbl[%d] crc value (0x%08x)\n", i, crc);
11476                         rc = -ENODEV;
11477                         goto test_nvram_exit;
11478                 }
11479         }
11480
11481 test_nvram_exit:
11482         return rc;
11483 }
11484
11485 static int bnx2x_test_intr(struct bnx2x *bp)
11486 {
11487         struct mac_configuration_cmd *config = bnx2x_sp(bp, mac_config);
11488         int i, rc;
11489
11490         if (!netif_running(bp->dev))
11491                 return -ENODEV;
11492
11493         config->hdr.length = 0;
11494         if (CHIP_IS_E1(bp))
11495                 /* use last unicast entries */
11496                 config->hdr.offset = (BP_PORT(bp) ? 63 : 31);
11497         else
11498                 config->hdr.offset = BP_FUNC(bp);
11499         config->hdr.client_id = bp->fp->cl_id;
11500         config->hdr.reserved1 = 0;
11501
11502         bp->set_mac_pending++;
11503         smp_wmb();
11504         rc = bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_SET_MAC, 0,
11505                            U64_HI(bnx2x_sp_mapping(bp, mac_config)),
11506                            U64_LO(bnx2x_sp_mapping(bp, mac_config)), 0);
11507         if (rc == 0) {
11508                 for (i = 0; i < 10; i++) {
11509                         if (!bp->set_mac_pending)
11510                                 break;
11511                         smp_rmb();
11512                         msleep_interruptible(10);
11513                 }
11514                 if (i == 10)
11515                         rc = -ENODEV;
11516         }
11517
11518         return rc;
11519 }
11520
11521 static void bnx2x_self_test(struct net_device *dev,
11522                             struct ethtool_test *etest, u64 *buf)
11523 {
11524         struct bnx2x *bp = netdev_priv(dev);
11525
11526         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
11527                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
11528                 etest->flags |= ETH_TEST_FL_FAILED;
11529                 return;
11530         }
11531
11532         memset(buf, 0, sizeof(u64) * BNX2X_NUM_TESTS);
11533
11534         if (!netif_running(dev))
11535                 return;
11536
11537         /* offline tests are not supported in MF mode */
11538         if (IS_E1HMF(bp))
11539                 etest->flags &= ~ETH_TEST_FL_OFFLINE;
11540
11541         if (etest->flags & ETH_TEST_FL_OFFLINE) {
11542                 int port = BP_PORT(bp);
11543                 u32 val;
11544                 u8 link_up;
11545
11546                 /* save current value of input enable for TX port IF */
11547                 val = REG_RD(bp, NIG_REG_EGRESS_UMP0_IN_EN + port*4);
11548                 /* disable input for TX port IF */
11549                 REG_WR(bp, NIG_REG_EGRESS_UMP0_IN_EN + port*4, 0);
11550
11551                 link_up = (bnx2x_link_test(bp) == 0);
11552                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
11553                 bnx2x_nic_load(bp, LOAD_DIAG);
11554                 /* wait until link state is restored */
11555                 bnx2x_wait_for_link(bp, link_up);
11556
11557                 if (bnx2x_test_registers(bp) != 0) {
11558                         buf[0] = 1;
11559                         etest->flags |= ETH_TEST_FL_FAILED;
11560                 }
11561                 if (bnx2x_test_memory(bp) != 0) {
11562                         buf[1] = 1;
11563                         etest->flags |= ETH_TEST_FL_FAILED;
11564                 }
11565                 buf[2] = bnx2x_test_loopback(bp, link_up);
11566                 if (buf[2] != 0)
11567                         etest->flags |= ETH_TEST_FL_FAILED;
11568
11569                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
11570
11571                 /* restore input for TX port IF */
11572                 REG_WR(bp, NIG_REG_EGRESS_UMP0_IN_EN + port*4, val);
11573
11574                 bnx2x_nic_load(bp, LOAD_NORMAL);
11575                 /* wait until link state is restored */
11576                 bnx2x_wait_for_link(bp, link_up);
11577         }
11578         if (bnx2x_test_nvram(bp) != 0) {
11579                 buf[3] = 1;
11580                 etest->flags |= ETH_TEST_FL_FAILED;
11581         }
11582         if (bnx2x_test_intr(bp) != 0) {
11583                 buf[4] = 1;
11584                 etest->flags |= ETH_TEST_FL_FAILED;
11585         }
11586         if (bp->port.pmf)
11587                 if (bnx2x_link_test(bp) != 0) {
11588                         buf[5] = 1;
11589                         etest->flags |= ETH_TEST_FL_FAILED;
11590                 }
11591
11592 #ifdef BNX2X_EXTRA_DEBUG
11593         bnx2x_panic_dump(bp);
11594 #endif
11595 }
11596
11597 static const struct {
11598         long offset;
11599         int size;
11600         u8 string[ETH_GSTRING_LEN];
11601 } bnx2x_q_stats_arr[BNX2X_NUM_Q_STATS] = {
11602 /* 1 */ { Q_STATS_OFFSET32(total_bytes_received_hi), 8, "[%d]: rx_bytes" },
11603         { Q_STATS_OFFSET32(error_bytes_received_hi),
11604                                                 8, "[%d]: rx_error_bytes" },
11605         { Q_STATS_OFFSET32(total_unicast_packets_received_hi),
11606                                                 8, "[%d]: rx_ucast_packets" },
11607         { Q_STATS_OFFSET32(total_multicast_packets_received_hi),
11608                                                 8, "[%d]: rx_mcast_packets" },
11609         { Q_STATS_OFFSET32(total_broadcast_packets_received_hi),
11610                                                 8, "[%d]: rx_bcast_packets" },
11611         { Q_STATS_OFFSET32(no_buff_discard_hi), 8, "[%d]: rx_discards" },
11612         { Q_STATS_OFFSET32(rx_err_discard_pkt),
11613                                          4, "[%d]: rx_phy_ip_err_discards"},
11614         { Q_STATS_OFFSET32(rx_skb_alloc_failed),
11615                                          4, "[%d]: rx_skb_alloc_discard" },
11616         { Q_STATS_OFFSET32(hw_csum_err), 4, "[%d]: rx_csum_offload_errors" },
11617
11618 /* 10 */{ Q_STATS_OFFSET32(total_bytes_transmitted_hi), 8, "[%d]: tx_bytes" },
11619         { Q_STATS_OFFSET32(total_unicast_packets_transmitted_hi),
11620                                                 8, "[%d]: tx_ucast_packets" },
11621         { Q_STATS_OFFSET32(total_multicast_packets_transmitted_hi),
11622                                                 8, "[%d]: tx_mcast_packets" },
11623         { Q_STATS_OFFSET32(total_broadcast_packets_transmitted_hi),
11624                                                 8, "[%d]: tx_bcast_packets" }
11625 };
11626
11627 static const struct {
11628         long offset;
11629         int size;
11630         u32 flags;
11631 #define STATS_FLAGS_PORT                1
11632 #define STATS_FLAGS_FUNC                2
11633 #define STATS_FLAGS_BOTH                (STATS_FLAGS_FUNC | STATS_FLAGS_PORT)
11634         u8 string[ETH_GSTRING_LEN];
11635 } bnx2x_stats_arr[BNX2X_NUM_STATS] = {
11636 /* 1 */ { STATS_OFFSET32(total_bytes_received_hi),
11637                                 8, STATS_FLAGS_BOTH, "rx_bytes" },
11638         { STATS_OFFSET32(error_bytes_received_hi),
11639                                 8, STATS_FLAGS_BOTH, "rx_error_bytes" },
11640         { STATS_OFFSET32(total_unicast_packets_received_hi),
11641                                 8, STATS_FLAGS_BOTH, "rx_ucast_packets" },
11642         { STATS_OFFSET32(total_multicast_packets_received_hi),
11643                                 8, STATS_FLAGS_BOTH, "rx_mcast_packets" },
11644         { STATS_OFFSET32(total_broadcast_packets_received_hi),
11645                                 8, STATS_FLAGS_BOTH, "rx_bcast_packets" },
11646         { STATS_OFFSET32(rx_stat_dot3statsfcserrors_hi),
11647                                 8, STATS_FLAGS_PORT, "rx_crc_errors" },
11648         { STATS_OFFSET32(rx_stat_dot3statsalignmenterrors_hi),
11649                                 8, STATS_FLAGS_PORT, "rx_align_errors" },
11650         { STATS_OFFSET32(rx_stat_etherstatsundersizepkts_hi),
11651                                 8, STATS_FLAGS_PORT, "rx_undersize_packets" },
11652         { STATS_OFFSET32(etherstatsoverrsizepkts_hi),
11653                                 8, STATS_FLAGS_PORT, "rx_oversize_packets" },
11654 /* 10 */{ STATS_OFFSET32(rx_stat_etherstatsfragments_hi),
11655                                 8, STATS_FLAGS_PORT, "rx_fragments" },
11656         { STATS_OFFSET32(rx_stat_etherstatsjabbers_hi),
11657                                 8, STATS_FLAGS_PORT, "rx_jabbers" },
11658         { STATS_OFFSET32(no_buff_discard_hi),
11659                                 8, STATS_FLAGS_BOTH, "rx_discards" },
11660         { STATS_OFFSET32(mac_filter_discard),
11661                                 4, STATS_FLAGS_PORT, "rx_filtered_packets" },
11662         { STATS_OFFSET32(xxoverflow_discard),
11663                                 4, STATS_FLAGS_PORT, "rx_fw_discards" },
11664         { STATS_OFFSET32(brb_drop_hi),
11665                                 8, STATS_FLAGS_PORT, "rx_brb_discard" },
11666         { STATS_OFFSET32(brb_truncate_hi),
11667                                 8, STATS_FLAGS_PORT, "rx_brb_truncate" },
11668         { STATS_OFFSET32(pause_frames_received_hi),
11669                                 8, STATS_FLAGS_PORT, "rx_pause_frames" },
11670         { STATS_OFFSET32(rx_stat_maccontrolframesreceived_hi),
11671                                 8, STATS_FLAGS_PORT, "rx_mac_ctrl_frames" },
11672         { STATS_OFFSET32(nig_timer_max),
11673                         4, STATS_FLAGS_PORT, "rx_constant_pause_events" },
11674 /* 20 */{ STATS_OFFSET32(rx_err_discard_pkt),
11675                                 4, STATS_FLAGS_BOTH, "rx_phy_ip_err_discards"},
11676         { STATS_OFFSET32(rx_skb_alloc_failed),
11677                                 4, STATS_FLAGS_BOTH, "rx_skb_alloc_discard" },
11678         { STATS_OFFSET32(hw_csum_err),
11679                                 4, STATS_FLAGS_BOTH, "rx_csum_offload_errors" },
11680
11681         { STATS_OFFSET32(total_bytes_transmitted_hi),
11682                                 8, STATS_FLAGS_BOTH, "tx_bytes" },
11683         { STATS_OFFSET32(tx_stat_ifhcoutbadoctets_hi),
11684                                 8, STATS_FLAGS_PORT, "tx_error_bytes" },
11685         { STATS_OFFSET32(total_unicast_packets_transmitted_hi),
11686                                 8, STATS_FLAGS_BOTH, "tx_ucast_packets" },
11687         { STATS_OFFSET32(total_multicast_packets_transmitted_hi),
11688                                 8, STATS_FLAGS_BOTH, "tx_mcast_packets" },
11689         { STATS_OFFSET32(total_broadcast_packets_transmitted_hi),
11690                                 8, STATS_FLAGS_BOTH, "tx_bcast_packets" },
11691         { STATS_OFFSET32(tx_stat_dot3statsinternalmactransmiterrors_hi),
11692                                 8, STATS_FLAGS_PORT, "tx_mac_errors" },
11693         { STATS_OFFSET32(rx_stat_dot3statscarriersenseerrors_hi),
11694                                 8, STATS_FLAGS_PORT, "tx_carrier_errors" },
11695 /* 30 */{ STATS_OFFSET32(tx_stat_dot3statssinglecollisionframes_hi),
11696                                 8, STATS_FLAGS_PORT, "tx_single_collisions" },
11697         { STATS_OFFSET32(tx_stat_dot3statsmultiplecollisionframes_hi),
11698                                 8, STATS_FLAGS_PORT, "tx_multi_collisions" },
11699         { STATS_OFFSET32(tx_stat_dot3statsdeferredtransmissions_hi),
11700                                 8, STATS_FLAGS_PORT, "tx_deferred" },
11701         { STATS_OFFSET32(tx_stat_dot3statsexcessivecollisions_hi),
11702                                 8, STATS_FLAGS_PORT, "tx_excess_collisions" },
11703         { STATS_OFFSET32(tx_stat_dot3statslatecollisions_hi),
11704                                 8, STATS_FLAGS_PORT, "tx_late_collisions" },
11705         { STATS_OFFSET32(tx_stat_etherstatscollisions_hi),
11706                                 8, STATS_FLAGS_PORT, "tx_total_collisions" },
11707         { STATS_OFFSET32(tx_stat_etherstatspkts64octets_hi),
11708                                 8, STATS_FLAGS_PORT, "tx_64_byte_packets" },
11709         { STATS_OFFSET32(tx_stat_etherstatspkts65octetsto127octets_hi),
11710                         8, STATS_FLAGS_PORT, "tx_65_to_127_byte_packets" },
11711         { STATS_OFFSET32(tx_stat_etherstatspkts128octetsto255octets_hi),
11712                         8, STATS_FLAGS_PORT, "tx_128_to_255_byte_packets" },
11713         { STATS_OFFSET32(tx_stat_etherstatspkts256octetsto511octets_hi),
11714                         8, STATS_FLAGS_PORT, "tx_256_to_511_byte_packets" },
11715 /* 40 */{ STATS_OFFSET32(tx_stat_etherstatspkts512octetsto1023octets_hi),
11716                         8, STATS_FLAGS_PORT, "tx_512_to_1023_byte_packets" },
11717         { STATS_OFFSET32(etherstatspkts1024octetsto1522octets_hi),
11718                         8, STATS_FLAGS_PORT, "tx_1024_to_1522_byte_packets" },
11719         { STATS_OFFSET32(etherstatspktsover1522octets_hi),
11720                         8, STATS_FLAGS_PORT, "tx_1523_to_9022_byte_packets" },
11721         { STATS_OFFSET32(pause_frames_sent_hi),
11722                                 8, STATS_FLAGS_PORT, "tx_pause_frames" }
11723 };
11724
11725 #define IS_PORT_STAT(i) \
11726         ((bnx2x_stats_arr[i].flags & STATS_FLAGS_BOTH) == STATS_FLAGS_PORT)
11727 #define IS_FUNC_STAT(i)         (bnx2x_stats_arr[i].flags & STATS_FLAGS_FUNC)
11728 #define IS_E1HMF_MODE_STAT(bp) \
11729                         (IS_E1HMF(bp) && !(bp->msg_enable & BNX2X_MSG_STATS))
11730
11731 static int bnx2x_get_sset_count(struct net_device *dev, int stringset)
11732 {
11733         struct bnx2x *bp = netdev_priv(dev);
11734         int i, num_stats;
11735
11736         switch (stringset) {
11737         case ETH_SS_STATS:
11738                 if (is_multi(bp)) {
11739                         num_stats = BNX2X_NUM_Q_STATS * bp->num_queues;
11740                         if (!IS_E1HMF_MODE_STAT(bp))
11741                                 num_stats += BNX2X_NUM_STATS;
11742                 } else {
11743                         if (IS_E1HMF_MODE_STAT(bp)) {
11744                                 num_stats = 0;
11745                                 for (i = 0; i < BNX2X_NUM_STATS; i++)
11746                                         if (IS_FUNC_STAT(i))
11747                                                 num_stats++;
11748                         } else
11749                                 num_stats = BNX2X_NUM_STATS;
11750                 }
11751                 return num_stats;
11752
11753         case ETH_SS_TEST:
11754                 return BNX2X_NUM_TESTS;
11755
11756         default:
11757                 return -EINVAL;
11758         }
11759 }
11760
11761 static void bnx2x_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
11762 {
11763         struct bnx2x *bp = netdev_priv(dev);
11764         int i, j, k;
11765
11766         switch (stringset) {
11767         case ETH_SS_STATS:
11768                 if (is_multi(bp)) {
11769                         k = 0;
11770                         for_each_queue(bp, i) {
11771                                 for (j = 0; j < BNX2X_NUM_Q_STATS; j++)
11772                                         sprintf(buf + (k + j)*ETH_GSTRING_LEN,
11773                                                 bnx2x_q_stats_arr[j].string, i);
11774                                 k += BNX2X_NUM_Q_STATS;
11775                         }
11776                         if (IS_E1HMF_MODE_STAT(bp))
11777                                 break;
11778                         for (j = 0; j < BNX2X_NUM_STATS; j++)
11779                                 strcpy(buf + (k + j)*ETH_GSTRING_LEN,
11780                                        bnx2x_stats_arr[j].string);
11781                 } else {
11782                         for (i = 0, j = 0; i < BNX2X_NUM_STATS; i++) {
11783                                 if (IS_E1HMF_MODE_STAT(bp) && IS_PORT_STAT(i))
11784                                         continue;
11785                                 strcpy(buf + j*ETH_GSTRING_LEN,
11786                                        bnx2x_stats_arr[i].string);
11787                                 j++;
11788                         }
11789                 }
11790                 break;
11791
11792         case ETH_SS_TEST:
11793                 memcpy(buf, bnx2x_tests_str_arr, sizeof(bnx2x_tests_str_arr));
11794                 break;
11795         }
11796 }
11797
11798 static void bnx2x_get_ethtool_stats(struct net_device *dev,
11799                                     struct ethtool_stats *stats, u64 *buf)
11800 {
11801         struct bnx2x *bp = netdev_priv(dev);
11802         u32 *hw_stats, *offset;
11803         int i, j, k;
11804
11805         if (is_multi(bp)) {
11806                 k = 0;
11807                 for_each_queue(bp, i) {
11808                         hw_stats = (u32 *)&bp->fp[i].eth_q_stats;
11809                         for (j = 0; j < BNX2X_NUM_Q_STATS; j++) {
11810                                 if (bnx2x_q_stats_arr[j].size == 0) {
11811                                         /* skip this counter */
11812                                         buf[k + j] = 0;
11813                                         continue;
11814                                 }
11815                                 offset = (hw_stats +
11816                                           bnx2x_q_stats_arr[j].offset);
11817                                 if (bnx2x_q_stats_arr[j].size == 4) {
11818                                         /* 4-byte counter */
11819                                         buf[k + j] = (u64) *offset;
11820                                         continue;
11821                                 }
11822                                 /* 8-byte counter */
11823                                 buf[k + j] = HILO_U64(*offset, *(offset + 1));
11824                         }
11825                         k += BNX2X_NUM_Q_STATS;
11826                 }
11827                 if (IS_E1HMF_MODE_STAT(bp))
11828                         return;
11829                 hw_stats = (u32 *)&bp->eth_stats;
11830                 for (j = 0; j < BNX2X_NUM_STATS; j++) {
11831                         if (bnx2x_stats_arr[j].size == 0) {
11832                                 /* skip this counter */
11833                                 buf[k + j] = 0;
11834                                 continue;
11835                         }
11836                         offset = (hw_stats + bnx2x_stats_arr[j].offset);
11837                         if (bnx2x_stats_arr[j].size == 4) {
11838                                 /* 4-byte counter */
11839                                 buf[k + j] = (u64) *offset;
11840                                 continue;
11841                         }
11842                         /* 8-byte counter */
11843                         buf[k + j] = HILO_U64(*offset, *(offset + 1));
11844                 }
11845         } else {
11846                 hw_stats = (u32 *)&bp->eth_stats;
11847                 for (i = 0, j = 0; i < BNX2X_NUM_STATS; i++) {
11848                         if (IS_E1HMF_MODE_STAT(bp) && IS_PORT_STAT(i))
11849                                 continue;
11850                         if (bnx2x_stats_arr[i].size == 0) {
11851                                 /* skip this counter */
11852                                 buf[j] = 0;
11853                                 j++;
11854                                 continue;
11855                         }
11856                         offset = (hw_stats + bnx2x_stats_arr[i].offset);
11857                         if (bnx2x_stats_arr[i].size == 4) {
11858                                 /* 4-byte counter */
11859                                 buf[j] = (u64) *offset;
11860                                 j++;
11861                                 continue;
11862                         }
11863                         /* 8-byte counter */
11864                         buf[j] = HILO_U64(*offset, *(offset + 1));
11865                         j++;
11866                 }
11867         }
11868 }
11869
11870 static int bnx2x_phys_id(struct net_device *dev, u32 data)
11871 {
11872         struct bnx2x *bp = netdev_priv(dev);
11873         int i;
11874
11875         if (!netif_running(dev))
11876                 return 0;
11877
11878         if (!bp->port.pmf)
11879                 return 0;
11880
11881         if (data == 0)
11882                 data = 2;
11883
11884         for (i = 0; i < (data * 2); i++) {
11885                 if ((i % 2) == 0)
11886                         bnx2x_set_led(&bp->link_params, LED_MODE_OPER,
11887                                       SPEED_1000);
11888                 else
11889                         bnx2x_set_led(&bp->link_params, LED_MODE_OFF, 0);
11890
11891                 msleep_interruptible(500);
11892                 if (signal_pending(current))
11893                         break;
11894         }
11895
11896         if (bp->link_vars.link_up)
11897                 bnx2x_set_led(&bp->link_params, LED_MODE_OPER,
11898                               bp->link_vars.line_speed);
11899
11900         return 0;
11901 }
11902
11903 static const struct ethtool_ops bnx2x_ethtool_ops = {
11904         .get_settings           = bnx2x_get_settings,
11905         .set_settings           = bnx2x_set_settings,
11906         .get_drvinfo            = bnx2x_get_drvinfo,
11907         .get_regs_len           = bnx2x_get_regs_len,
11908         .get_regs               = bnx2x_get_regs,
11909         .get_wol                = bnx2x_get_wol,
11910         .set_wol                = bnx2x_set_wol,
11911         .get_msglevel           = bnx2x_get_msglevel,
11912         .set_msglevel           = bnx2x_set_msglevel,
11913         .nway_reset             = bnx2x_nway_reset,
11914         .get_link               = bnx2x_get_link,
11915         .get_eeprom_len         = bnx2x_get_eeprom_len,
11916         .get_eeprom             = bnx2x_get_eeprom,
11917         .set_eeprom             = bnx2x_set_eeprom,
11918         .get_coalesce           = bnx2x_get_coalesce,
11919         .set_coalesce           = bnx2x_set_coalesce,
11920         .get_ringparam          = bnx2x_get_ringparam,
11921         .set_ringparam          = bnx2x_set_ringparam,
11922         .get_pauseparam         = bnx2x_get_pauseparam,
11923         .set_pauseparam         = bnx2x_set_pauseparam,
11924         .get_rx_csum            = bnx2x_get_rx_csum,
11925         .set_rx_csum            = bnx2x_set_rx_csum,
11926         .get_tx_csum            = ethtool_op_get_tx_csum,
11927         .set_tx_csum            = ethtool_op_set_tx_hw_csum,
11928         .set_flags              = bnx2x_set_flags,
11929         .get_flags              = ethtool_op_get_flags,
11930         .get_sg                 = ethtool_op_get_sg,
11931         .set_sg                 = ethtool_op_set_sg,
11932         .get_tso                = ethtool_op_get_tso,
11933         .set_tso                = bnx2x_set_tso,
11934         .self_test              = bnx2x_self_test,
11935         .get_sset_count         = bnx2x_get_sset_count,
11936         .get_strings            = bnx2x_get_strings,
11937         .phys_id                = bnx2x_phys_id,
11938         .get_ethtool_stats      = bnx2x_get_ethtool_stats,
11939 };
11940
11941 /* end of ethtool_ops */
11942
11943 /****************************************************************************
11944 * General service functions
11945 ****************************************************************************/
11946
11947 static int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
11948 {
11949         u16 pmcsr;
11950
11951         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
11952
11953         switch (state) {
11954         case PCI_D0:
11955                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
11956                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
11957                                        PCI_PM_CTRL_PME_STATUS));
11958
11959                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
11960                         /* delay required during transition out of D3hot */
11961                         msleep(20);
11962                 break;
11963
11964         case PCI_D3hot:
11965                 /* If there are other clients above don't
11966                    shut down the power */
11967                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
11968                         return 0;
11969                 /* Don't shut down the power for emulation and FPGA */
11970                 if (CHIP_REV_IS_SLOW(bp))
11971                         return 0;
11972
11973                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
11974                 pmcsr |= 3;
11975
11976                 if (bp->wol)
11977                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
11978
11979                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
11980                                       pmcsr);
11981
11982                 /* No more memory access after this point until
11983                 * device is brought back to D0.
11984                 */
11985                 break;
11986
11987         default:
11988                 return -EINVAL;
11989         }
11990         return 0;
11991 }
11992
11993 static inline int bnx2x_has_rx_work(struct bnx2x_fastpath *fp)
11994 {
11995         u16 rx_cons_sb;
11996
11997         /* Tell compiler that status block fields can change */
11998         barrier();
11999         rx_cons_sb = le16_to_cpu(*fp->rx_cons_sb);
12000         if ((rx_cons_sb & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
12001                 rx_cons_sb++;
12002         return (fp->rx_comp_cons != rx_cons_sb);
12003 }
12004
12005 /*
12006  * net_device service functions
12007  */
12008
12009 static int bnx2x_poll(struct napi_struct *napi, int budget)
12010 {
12011         int work_done = 0;
12012         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
12013                                                  napi);
12014         struct bnx2x *bp = fp->bp;
12015
12016         while (1) {
12017 #ifdef BNX2X_STOP_ON_ERROR
12018                 if (unlikely(bp->panic)) {
12019                         napi_complete(napi);
12020                         return 0;
12021                 }
12022 #endif
12023
12024                 if (bnx2x_has_tx_work(fp))
12025                         bnx2x_tx_int(fp);
12026
12027                 if (bnx2x_has_rx_work(fp)) {
12028                         work_done += bnx2x_rx_int(fp, budget - work_done);
12029
12030                         /* must not complete if we consumed full budget */
12031                         if (work_done >= budget)
12032                                 break;
12033                 }
12034
12035                 /* Fall out from the NAPI loop if needed */
12036                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
12037                         bnx2x_update_fpsb_idx(fp);
12038                 /* bnx2x_has_rx_work() reads the status block, thus we need
12039                  * to ensure that status block indices have been actually read
12040                  * (bnx2x_update_fpsb_idx) prior to this check
12041                  * (bnx2x_has_rx_work) so that we won't write the "newer"
12042                  * value of the status block to IGU (if there was a DMA right
12043                  * after bnx2x_has_rx_work and if there is no rmb, the memory
12044                  * reading (bnx2x_update_fpsb_idx) may be postponed to right
12045                  * before bnx2x_ack_sb). In this case there will never be
12046                  * another interrupt until there is another update of the
12047                  * status block, while there is still unhandled work.
12048                  */
12049                         rmb();
12050
12051                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
12052                                 napi_complete(napi);
12053                                 /* Re-enable interrupts */
12054                                 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
12055                                              le16_to_cpu(fp->fp_c_idx),
12056                                              IGU_INT_NOP, 1);
12057                                 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
12058                                              le16_to_cpu(fp->fp_u_idx),
12059                                              IGU_INT_ENABLE, 1);
12060                                 break;
12061                         }
12062                 }
12063         }
12064
12065         return work_done;
12066 }
12067
12068
12069 /* we split the first BD into headers and data BDs
12070  * to ease the pain of our fellow microcode engineers
12071  * we use one mapping for both BDs
12072  * So far this has only been observed to happen
12073  * in Other Operating Systems(TM)
12074  */
12075 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
12076                                    struct bnx2x_fastpath *fp,
12077                                    struct sw_tx_bd *tx_buf,
12078                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
12079                                    u16 bd_prod, int nbd)
12080 {
12081         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
12082         struct eth_tx_bd *d_tx_bd;
12083         dma_addr_t mapping;
12084         int old_len = le16_to_cpu(h_tx_bd->nbytes);
12085
12086         /* first fix first BD */
12087         h_tx_bd->nbd = cpu_to_le16(nbd);
12088         h_tx_bd->nbytes = cpu_to_le16(hlen);
12089
12090         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
12091            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
12092            h_tx_bd->addr_lo, h_tx_bd->nbd);
12093
12094         /* now get a new data BD
12095          * (after the pbd) and fill it */
12096         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
12097         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
12098
12099         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
12100                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
12101
12102         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
12103         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
12104         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
12105
12106         /* this marks the BD as one that has no individual mapping */
12107         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
12108
12109         DP(NETIF_MSG_TX_QUEUED,
12110            "TSO split data size is %d (%x:%x)\n",
12111            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
12112
12113         /* update tx_bd */
12114         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
12115
12116         return bd_prod;
12117 }
12118
12119 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
12120 {
12121         if (fix > 0)
12122                 csum = (u16) ~csum_fold(csum_sub(csum,
12123                                 csum_partial(t_header - fix, fix, 0)));
12124
12125         else if (fix < 0)
12126                 csum = (u16) ~csum_fold(csum_add(csum,
12127                                 csum_partial(t_header, -fix, 0)));
12128
12129         return swab16(csum);
12130 }
12131
12132 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
12133 {
12134         u32 rc;
12135
12136         if (skb->ip_summed != CHECKSUM_PARTIAL)
12137                 rc = XMIT_PLAIN;
12138
12139         else {
12140                 if (skb->protocol == htons(ETH_P_IPV6)) {
12141                         rc = XMIT_CSUM_V6;
12142                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
12143                                 rc |= XMIT_CSUM_TCP;
12144
12145                 } else {
12146                         rc = XMIT_CSUM_V4;
12147                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
12148                                 rc |= XMIT_CSUM_TCP;
12149                 }
12150         }
12151
12152         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
12153                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
12154
12155         else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
12156                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
12157
12158         return rc;
12159 }
12160
12161 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
12162 /* check if packet requires linearization (packet is too fragmented)
12163    no need to check fragmentation if page size > 8K (there will be no
12164    violation to FW restrictions) */
12165 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
12166                              u32 xmit_type)
12167 {
12168         int to_copy = 0;
12169         int hlen = 0;
12170         int first_bd_sz = 0;
12171
12172         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
12173         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
12174
12175                 if (xmit_type & XMIT_GSO) {
12176                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
12177                         /* Check if LSO packet needs to be copied:
12178                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
12179                         int wnd_size = MAX_FETCH_BD - 3;
12180                         /* Number of windows to check */
12181                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
12182                         int wnd_idx = 0;
12183                         int frag_idx = 0;
12184                         u32 wnd_sum = 0;
12185
12186                         /* Headers length */
12187                         hlen = (int)(skb_transport_header(skb) - skb->data) +
12188                                 tcp_hdrlen(skb);
12189
12190                         /* Amount of data (w/o headers) on linear part of SKB*/
12191                         first_bd_sz = skb_headlen(skb) - hlen;
12192
12193                         wnd_sum  = first_bd_sz;
12194
12195                         /* Calculate the first sum - it's special */
12196                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
12197                                 wnd_sum +=
12198                                         skb_shinfo(skb)->frags[frag_idx].size;
12199
12200                         /* If there was data on linear skb data - check it */
12201                         if (first_bd_sz > 0) {
12202                                 if (unlikely(wnd_sum < lso_mss)) {
12203                                         to_copy = 1;
12204                                         goto exit_lbl;
12205                                 }
12206
12207                                 wnd_sum -= first_bd_sz;
12208                         }
12209
12210                         /* Others are easier: run through the frag list and
12211                            check all windows */
12212                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
12213                                 wnd_sum +=
12214                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
12215
12216                                 if (unlikely(wnd_sum < lso_mss)) {
12217                                         to_copy = 1;
12218                                         break;
12219                                 }
12220                                 wnd_sum -=
12221                                         skb_shinfo(skb)->frags[wnd_idx].size;
12222                         }
12223                 } else {
12224                         /* in non-LSO too fragmented packet should always
12225                            be linearized */
12226                         to_copy = 1;
12227                 }
12228         }
12229
12230 exit_lbl:
12231         if (unlikely(to_copy))
12232                 DP(NETIF_MSG_TX_QUEUED,
12233                    "Linearization IS REQUIRED for %s packet. "
12234                    "num_frags %d  hlen %d  first_bd_sz %d\n",
12235                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
12236                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
12237
12238         return to_copy;
12239 }
12240 #endif
12241
12242 /* called with netif_tx_lock
12243  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
12244  * netif_wake_queue()
12245  */
12246 static netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
12247 {
12248         struct bnx2x *bp = netdev_priv(dev);
12249         struct bnx2x_fastpath *fp;
12250         struct netdev_queue *txq;
12251         struct sw_tx_bd *tx_buf;
12252         struct eth_tx_start_bd *tx_start_bd;
12253         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
12254         struct eth_tx_parse_bd *pbd = NULL;
12255         u16 pkt_prod, bd_prod;
12256         int nbd, fp_index;
12257         dma_addr_t mapping;
12258         u32 xmit_type = bnx2x_xmit_type(bp, skb);
12259         int i;
12260         u8 hlen = 0;
12261         __le16 pkt_size = 0;
12262         struct ethhdr *eth;
12263         u8 mac_type = UNICAST_ADDRESS;
12264
12265 #ifdef BNX2X_STOP_ON_ERROR
12266         if (unlikely(bp->panic))
12267                 return NETDEV_TX_BUSY;
12268 #endif
12269
12270         fp_index = skb_get_queue_mapping(skb);
12271         txq = netdev_get_tx_queue(dev, fp_index);
12272
12273         fp = &bp->fp[fp_index];
12274
12275         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
12276                 fp->eth_q_stats.driver_xoff++;
12277                 netif_tx_stop_queue(txq);
12278                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
12279                 return NETDEV_TX_BUSY;
12280         }
12281
12282         DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x  protocol %x  protocol(%x,%x)"
12283            "  gso type %x  xmit_type %x\n",
12284            skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
12285            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
12286
12287         eth = (struct ethhdr *)skb->data;
12288
12289         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
12290         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
12291                 if (is_broadcast_ether_addr(eth->h_dest))
12292                         mac_type = BROADCAST_ADDRESS;
12293                 else
12294                         mac_type = MULTICAST_ADDRESS;
12295         }
12296
12297 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
12298         /* First, check if we need to linearize the skb (due to FW
12299            restrictions). No need to check fragmentation if page size > 8K
12300            (there will be no violation to FW restrictions) */
12301         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
12302                 /* Statistics of linearization */
12303                 bp->lin_cnt++;
12304                 if (skb_linearize(skb) != 0) {
12305                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
12306                            "silently dropping this SKB\n");
12307                         dev_kfree_skb_any(skb);
12308                         return NETDEV_TX_OK;
12309                 }
12310         }
12311 #endif
12312
12313         /*
12314         Please read carefully. First we use one BD which we mark as start,
12315         then we have a parsing info BD (used for TSO or xsum),
12316         and only then we have the rest of the TSO BDs.
12317         (don't forget to mark the last one as last,
12318         and to unmap only AFTER you write to the BD ...)
12319         And above all, all pdb sizes are in words - NOT DWORDS!
12320         */
12321
12322         pkt_prod = fp->tx_pkt_prod++;
12323         bd_prod = TX_BD(fp->tx_bd_prod);
12324
12325         /* get a tx_buf and first BD */
12326         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
12327         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
12328
12329         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
12330         tx_start_bd->general_data =  (mac_type <<
12331                                         ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
12332         /* header nbd */
12333         tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
12334
12335         /* remember the first BD of the packet */
12336         tx_buf->first_bd = fp->tx_bd_prod;
12337         tx_buf->skb = skb;
12338         tx_buf->flags = 0;
12339
12340         DP(NETIF_MSG_TX_QUEUED,
12341            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
12342            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
12343
12344 #ifdef BCM_VLAN
12345         if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
12346             (bp->flags & HW_VLAN_TX_FLAG)) {
12347                 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
12348                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
12349         } else
12350 #endif
12351                 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
12352
12353         /* turn on parsing and get a BD */
12354         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
12355         pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
12356
12357         memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
12358
12359         if (xmit_type & XMIT_CSUM) {
12360                 hlen = (skb_network_header(skb) - skb->data) / 2;
12361
12362                 /* for now NS flag is not used in Linux */
12363                 pbd->global_data =
12364                         (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
12365                                  ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
12366
12367                 pbd->ip_hlen = (skb_transport_header(skb) -
12368                                 skb_network_header(skb)) / 2;
12369
12370                 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
12371
12372                 pbd->total_hlen = cpu_to_le16(hlen);
12373                 hlen = hlen*2;
12374
12375                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
12376
12377                 if (xmit_type & XMIT_CSUM_V4)
12378                         tx_start_bd->bd_flags.as_bitfield |=
12379                                                 ETH_TX_BD_FLAGS_IP_CSUM;
12380                 else
12381                         tx_start_bd->bd_flags.as_bitfield |=
12382                                                 ETH_TX_BD_FLAGS_IPV6;
12383
12384                 if (xmit_type & XMIT_CSUM_TCP) {
12385                         pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
12386
12387                 } else {
12388                         s8 fix = SKB_CS_OFF(skb); /* signed! */
12389
12390                         pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
12391
12392                         DP(NETIF_MSG_TX_QUEUED,
12393                            "hlen %d  fix %d  csum before fix %x\n",
12394                            le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
12395
12396                         /* HW bug: fixup the CSUM */
12397                         pbd->tcp_pseudo_csum =
12398                                 bnx2x_csum_fix(skb_transport_header(skb),
12399                                                SKB_CS(skb), fix);
12400
12401                         DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
12402                            pbd->tcp_pseudo_csum);
12403                 }
12404         }
12405
12406         mapping = dma_map_single(&bp->pdev->dev, skb->data,
12407                                  skb_headlen(skb), DMA_TO_DEVICE);
12408
12409         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
12410         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
12411         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
12412         tx_start_bd->nbd = cpu_to_le16(nbd);
12413         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
12414         pkt_size = tx_start_bd->nbytes;
12415
12416         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
12417            "  nbytes %d  flags %x  vlan %x\n",
12418            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
12419            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
12420            tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
12421
12422         if (xmit_type & XMIT_GSO) {
12423
12424                 DP(NETIF_MSG_TX_QUEUED,
12425                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
12426                    skb->len, hlen, skb_headlen(skb),
12427                    skb_shinfo(skb)->gso_size);
12428
12429                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
12430
12431                 if (unlikely(skb_headlen(skb) > hlen))
12432                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
12433                                                  hlen, bd_prod, ++nbd);
12434
12435                 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
12436                 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
12437                 pbd->tcp_flags = pbd_tcp_flags(skb);
12438
12439                 if (xmit_type & XMIT_GSO_V4) {
12440                         pbd->ip_id = swab16(ip_hdr(skb)->id);
12441                         pbd->tcp_pseudo_csum =
12442                                 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
12443                                                           ip_hdr(skb)->daddr,
12444                                                           0, IPPROTO_TCP, 0));
12445
12446                 } else
12447                         pbd->tcp_pseudo_csum =
12448                                 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
12449                                                         &ipv6_hdr(skb)->daddr,
12450                                                         0, IPPROTO_TCP, 0));
12451
12452                 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
12453         }
12454         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
12455
12456         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
12457                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
12458
12459                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
12460                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
12461                 if (total_pkt_bd == NULL)
12462                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
12463
12464                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
12465                                        frag->page_offset,
12466                                        frag->size, DMA_TO_DEVICE);
12467
12468                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
12469                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
12470                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
12471                 le16_add_cpu(&pkt_size, frag->size);
12472
12473                 DP(NETIF_MSG_TX_QUEUED,
12474                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
12475                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
12476                    le16_to_cpu(tx_data_bd->nbytes));
12477         }
12478
12479         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
12480
12481         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
12482
12483         /* now send a tx doorbell, counting the next BD
12484          * if the packet contains or ends with it
12485          */
12486         if (TX_BD_POFF(bd_prod) < nbd)
12487                 nbd++;
12488
12489         if (total_pkt_bd != NULL)
12490                 total_pkt_bd->total_pkt_bytes = pkt_size;
12491
12492         if (pbd)
12493                 DP(NETIF_MSG_TX_QUEUED,
12494                    "PBD @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
12495                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
12496                    pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
12497                    pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
12498                    pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
12499
12500         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
12501
12502         /*
12503          * Make sure that the BD data is updated before updating the producer
12504          * since FW might read the BD right after the producer is updated.
12505          * This is only applicable for weak-ordered memory model archs such
12506          * as IA-64. The following barrier is also mandatory since FW will
12507          * assumes packets must have BDs.
12508          */
12509         wmb();
12510
12511         fp->tx_db.data.prod += nbd;
12512         barrier();
12513         DOORBELL(bp, fp->index, fp->tx_db.raw);
12514
12515         mmiowb();
12516
12517         fp->tx_bd_prod += nbd;
12518
12519         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
12520                 netif_tx_stop_queue(txq);
12521
12522                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
12523                  * ordering of set_bit() in netif_tx_stop_queue() and read of
12524                  * fp->bd_tx_cons */
12525                 smp_mb();
12526
12527                 fp->eth_q_stats.driver_xoff++;
12528                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
12529                         netif_tx_wake_queue(txq);
12530         }
12531         fp->tx_pkt++;
12532
12533         return NETDEV_TX_OK;
12534 }
12535
12536 /* called with rtnl_lock */
12537 static int bnx2x_open(struct net_device *dev)
12538 {
12539         struct bnx2x *bp = netdev_priv(dev);
12540
12541         netif_carrier_off(dev);
12542
12543         bnx2x_set_power_state(bp, PCI_D0);
12544
12545         if (!bnx2x_reset_is_done(bp)) {
12546                 do {
12547                         /* Reset MCP mail box sequence if there is on going
12548                          * recovery
12549                          */
12550                         bp->fw_seq = 0;
12551
12552                         /* If it's the first function to load and reset done
12553                          * is still not cleared it may mean that. We don't
12554                          * check the attention state here because it may have
12555                          * already been cleared by a "common" reset but we
12556                          * shell proceed with "process kill" anyway.
12557                          */
12558                         if ((bnx2x_get_load_cnt(bp) == 0) &&
12559                                 bnx2x_trylock_hw_lock(bp,
12560                                 HW_LOCK_RESOURCE_RESERVED_08) &&
12561                                 (!bnx2x_leader_reset(bp))) {
12562                                 DP(NETIF_MSG_HW, "Recovered in open\n");
12563                                 break;
12564                         }
12565
12566                         bnx2x_set_power_state(bp, PCI_D3hot);
12567
12568                         printk(KERN_ERR"%s: Recovery flow hasn't been properly"
12569                         " completed yet. Try again later. If u still see this"
12570                         " message after a few retries then power cycle is"
12571                         " required.\n", bp->dev->name);
12572
12573                         return -EAGAIN;
12574                 } while (0);
12575         }
12576
12577         bp->recovery_state = BNX2X_RECOVERY_DONE;
12578
12579         return bnx2x_nic_load(bp, LOAD_OPEN);
12580 }
12581
12582 /* called with rtnl_lock */
12583 static int bnx2x_close(struct net_device *dev)
12584 {
12585         struct bnx2x *bp = netdev_priv(dev);
12586
12587         /* Unload the driver, release IRQs */
12588         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
12589         bnx2x_set_power_state(bp, PCI_D3hot);
12590
12591         return 0;
12592 }
12593
12594 /* called with netif_tx_lock from dev_mcast.c */
12595 static void bnx2x_set_rx_mode(struct net_device *dev)
12596 {
12597         struct bnx2x *bp = netdev_priv(dev);
12598         u32 rx_mode = BNX2X_RX_MODE_NORMAL;
12599         int port = BP_PORT(bp);
12600
12601         if (bp->state != BNX2X_STATE_OPEN) {
12602                 DP(NETIF_MSG_IFUP, "state is %x, returning\n", bp->state);
12603                 return;
12604         }
12605
12606         DP(NETIF_MSG_IFUP, "dev->flags = %x\n", dev->flags);
12607
12608         if (dev->flags & IFF_PROMISC)
12609                 rx_mode = BNX2X_RX_MODE_PROMISC;
12610
12611         else if ((dev->flags & IFF_ALLMULTI) ||
12612                  ((netdev_mc_count(dev) > BNX2X_MAX_MULTICAST) &&
12613                   CHIP_IS_E1(bp)))
12614                 rx_mode = BNX2X_RX_MODE_ALLMULTI;
12615
12616         else { /* some multicasts */
12617                 if (CHIP_IS_E1(bp)) {
12618                         int i, old, offset;
12619                         struct netdev_hw_addr *ha;
12620                         struct mac_configuration_cmd *config =
12621                                                 bnx2x_sp(bp, mcast_config);
12622
12623                         i = 0;
12624                         netdev_for_each_mc_addr(ha, dev) {
12625                                 config->config_table[i].
12626                                         cam_entry.msb_mac_addr =
12627                                         swab16(*(u16 *)&ha->addr[0]);
12628                                 config->config_table[i].
12629                                         cam_entry.middle_mac_addr =
12630                                         swab16(*(u16 *)&ha->addr[2]);
12631                                 config->config_table[i].
12632                                         cam_entry.lsb_mac_addr =
12633                                         swab16(*(u16 *)&ha->addr[4]);
12634                                 config->config_table[i].cam_entry.flags =
12635                                                         cpu_to_le16(port);
12636                                 config->config_table[i].
12637                                         target_table_entry.flags = 0;
12638                                 config->config_table[i].target_table_entry.
12639                                         clients_bit_vector =
12640                                                 cpu_to_le32(1 << BP_L_ID(bp));
12641                                 config->config_table[i].
12642                                         target_table_entry.vlan_id = 0;
12643
12644                                 DP(NETIF_MSG_IFUP,
12645                                    "setting MCAST[%d] (%04x:%04x:%04x)\n", i,
12646                                    config->config_table[i].
12647                                                 cam_entry.msb_mac_addr,
12648                                    config->config_table[i].
12649                                                 cam_entry.middle_mac_addr,
12650                                    config->config_table[i].
12651                                                 cam_entry.lsb_mac_addr);
12652                                 i++;
12653                         }
12654                         old = config->hdr.length;
12655                         if (old > i) {
12656                                 for (; i < old; i++) {
12657                                         if (CAM_IS_INVALID(config->
12658                                                            config_table[i])) {
12659                                                 /* already invalidated */
12660                                                 break;
12661                                         }
12662                                         /* invalidate */
12663                                         CAM_INVALIDATE(config->
12664                                                        config_table[i]);
12665                                 }
12666                         }
12667
12668                         if (CHIP_REV_IS_SLOW(bp))
12669                                 offset = BNX2X_MAX_EMUL_MULTI*(1 + port);
12670                         else
12671                                 offset = BNX2X_MAX_MULTICAST*(1 + port);
12672
12673                         config->hdr.length = i;
12674                         config->hdr.offset = offset;
12675                         config->hdr.client_id = bp->fp->cl_id;
12676                         config->hdr.reserved1 = 0;
12677
12678                         bp->set_mac_pending++;
12679                         smp_wmb();
12680
12681                         bnx2x_sp_post(bp, RAMROD_CMD_ID_ETH_SET_MAC, 0,
12682                                    U64_HI(bnx2x_sp_mapping(bp, mcast_config)),
12683                                    U64_LO(bnx2x_sp_mapping(bp, mcast_config)),
12684                                       0);
12685                 } else { /* E1H */
12686                         /* Accept one or more multicasts */
12687                         struct netdev_hw_addr *ha;
12688                         u32 mc_filter[MC_HASH_SIZE];
12689                         u32 crc, bit, regidx;
12690                         int i;
12691
12692                         memset(mc_filter, 0, 4 * MC_HASH_SIZE);
12693
12694                         netdev_for_each_mc_addr(ha, dev) {
12695                                 DP(NETIF_MSG_IFUP, "Adding mcast MAC: %pM\n",
12696                                    ha->addr);
12697
12698                                 crc = crc32c_le(0, ha->addr, ETH_ALEN);
12699                                 bit = (crc >> 24) & 0xff;
12700                                 regidx = bit >> 5;
12701                                 bit &= 0x1f;
12702                                 mc_filter[regidx] |= (1 << bit);
12703                         }
12704
12705                         for (i = 0; i < MC_HASH_SIZE; i++)
12706                                 REG_WR(bp, MC_HASH_OFFSET(bp, i),
12707                                        mc_filter[i]);
12708                 }
12709         }
12710
12711         bp->rx_mode = rx_mode;
12712         bnx2x_set_storm_rx_mode(bp);
12713 }
12714
12715 /* called with rtnl_lock */
12716 static int bnx2x_change_mac_addr(struct net_device *dev, void *p)
12717 {
12718         struct sockaddr *addr = p;
12719         struct bnx2x *bp = netdev_priv(dev);
12720
12721         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
12722                 return -EINVAL;
12723
12724         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
12725         if (netif_running(dev)) {
12726                 if (CHIP_IS_E1(bp))
12727                         bnx2x_set_eth_mac_addr_e1(bp, 1);
12728                 else
12729                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
12730         }
12731
12732         return 0;
12733 }
12734
12735 /* called with rtnl_lock */
12736 static int bnx2x_mdio_read(struct net_device *netdev, int prtad,
12737                            int devad, u16 addr)
12738 {
12739         struct bnx2x *bp = netdev_priv(netdev);
12740         u16 value;
12741         int rc;
12742         u32 phy_type = XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
12743
12744         DP(NETIF_MSG_LINK, "mdio_read: prtad 0x%x, devad 0x%x, addr 0x%x\n",
12745            prtad, devad, addr);
12746
12747         if (prtad != bp->mdio.prtad) {
12748                 DP(NETIF_MSG_LINK, "prtad missmatch (cmd:0x%x != bp:0x%x)\n",
12749                    prtad, bp->mdio.prtad);
12750                 return -EINVAL;
12751         }
12752
12753         /* The HW expects different devad if CL22 is used */
12754         devad = (devad == MDIO_DEVAD_NONE) ? DEFAULT_PHY_DEV_ADDR : devad;
12755
12756         bnx2x_acquire_phy_lock(bp);
12757         rc = bnx2x_cl45_read(bp, BP_PORT(bp), phy_type, prtad,
12758                              devad, addr, &value);
12759         bnx2x_release_phy_lock(bp);
12760         DP(NETIF_MSG_LINK, "mdio_read_val 0x%x rc = 0x%x\n", value, rc);
12761
12762         if (!rc)
12763                 rc = value;
12764         return rc;
12765 }
12766
12767 /* called with rtnl_lock */
12768 static int bnx2x_mdio_write(struct net_device *netdev, int prtad, int devad,
12769                             u16 addr, u16 value)
12770 {
12771         struct bnx2x *bp = netdev_priv(netdev);
12772         u32 ext_phy_type = XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);
12773         int rc;
12774
12775         DP(NETIF_MSG_LINK, "mdio_write: prtad 0x%x, devad 0x%x, addr 0x%x,"
12776                            " value 0x%x\n", prtad, devad, addr, value);
12777
12778         if (prtad != bp->mdio.prtad) {
12779                 DP(NETIF_MSG_LINK, "prtad missmatch (cmd:0x%x != bp:0x%x)\n",
12780                    prtad, bp->mdio.prtad);
12781                 return -EINVAL;
12782         }
12783
12784         /* The HW expects different devad if CL22 is used */
12785         devad = (devad == MDIO_DEVAD_NONE) ? DEFAULT_PHY_DEV_ADDR : devad;
12786
12787         bnx2x_acquire_phy_lock(bp);
12788         rc = bnx2x_cl45_write(bp, BP_PORT(bp), ext_phy_type, prtad,
12789                               devad, addr, value);
12790         bnx2x_release_phy_lock(bp);
12791         return rc;
12792 }
12793
12794 /* called with rtnl_lock */
12795 static int bnx2x_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
12796 {
12797         struct bnx2x *bp = netdev_priv(dev);
12798         struct mii_ioctl_data *mdio = if_mii(ifr);
12799
12800         DP(NETIF_MSG_LINK, "ioctl: phy id 0x%x, reg 0x%x, val_in 0x%x\n",
12801            mdio->phy_id, mdio->reg_num, mdio->val_in);
12802
12803         if (!netif_running(dev))
12804                 return -EAGAIN;
12805
12806         return mdio_mii_ioctl(&bp->mdio, mdio, cmd);
12807 }
12808
12809 /* called with rtnl_lock */
12810 static int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
12811 {
12812         struct bnx2x *bp = netdev_priv(dev);
12813         int rc = 0;
12814
12815         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
12816                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
12817                 return -EAGAIN;
12818         }
12819
12820         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
12821             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
12822                 return -EINVAL;
12823
12824         /* This does not race with packet allocation
12825          * because the actual alloc size is
12826          * only updated as part of load
12827          */
12828         dev->mtu = new_mtu;
12829
12830         if (netif_running(dev)) {
12831                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
12832                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
12833         }
12834
12835         return rc;
12836 }
12837
12838 static void bnx2x_tx_timeout(struct net_device *dev)
12839 {
12840         struct bnx2x *bp = netdev_priv(dev);
12841
12842 #ifdef BNX2X_STOP_ON_ERROR
12843         if (!bp->panic)
12844                 bnx2x_panic();
12845 #endif
12846         /* This allows the netif to be shutdown gracefully before resetting */
12847         schedule_delayed_work(&bp->reset_task, 0);
12848 }
12849
12850 #ifdef BCM_VLAN
12851 /* called with rtnl_lock */
12852 static void bnx2x_vlan_rx_register(struct net_device *dev,
12853                                    struct vlan_group *vlgrp)
12854 {
12855         struct bnx2x *bp = netdev_priv(dev);
12856
12857         bp->vlgrp = vlgrp;
12858
12859         /* Set flags according to the required capabilities */
12860         bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
12861
12862         if (dev->features & NETIF_F_HW_VLAN_TX)
12863                 bp->flags |= HW_VLAN_TX_FLAG;
12864
12865         if (dev->features & NETIF_F_HW_VLAN_RX)
12866                 bp->flags |= HW_VLAN_RX_FLAG;
12867
12868         if (netif_running(dev))
12869                 bnx2x_set_client_config(bp);
12870 }
12871
12872 #endif
12873
12874 #ifdef CONFIG_NET_POLL_CONTROLLER
12875 static void poll_bnx2x(struct net_device *dev)
12876 {
12877         struct bnx2x *bp = netdev_priv(dev);
12878
12879         disable_irq(bp->pdev->irq);
12880         bnx2x_interrupt(bp->pdev->irq, dev);
12881         enable_irq(bp->pdev->irq);
12882 }
12883 #endif
12884
12885 static const struct net_device_ops bnx2x_netdev_ops = {
12886         .ndo_open               = bnx2x_open,
12887         .ndo_stop               = bnx2x_close,
12888         .ndo_start_xmit         = bnx2x_start_xmit,
12889         .ndo_set_multicast_list = bnx2x_set_rx_mode,
12890         .ndo_set_mac_address    = bnx2x_change_mac_addr,
12891         .ndo_validate_addr      = eth_validate_addr,
12892         .ndo_do_ioctl           = bnx2x_ioctl,
12893         .ndo_change_mtu         = bnx2x_change_mtu,
12894         .ndo_tx_timeout         = bnx2x_tx_timeout,
12895 #ifdef BCM_VLAN
12896         .ndo_vlan_rx_register   = bnx2x_vlan_rx_register,
12897 #endif
12898 #ifdef CONFIG_NET_POLL_CONTROLLER
12899         .ndo_poll_controller    = poll_bnx2x,
12900 #endif
12901 };
12902
12903 static int __devinit bnx2x_init_dev(struct pci_dev *pdev,
12904                                     struct net_device *dev)
12905 {
12906         struct bnx2x *bp;
12907         int rc;
12908
12909         SET_NETDEV_DEV(dev, &pdev->dev);
12910         bp = netdev_priv(dev);
12911
12912         bp->dev = dev;
12913         bp->pdev = pdev;
12914         bp->flags = 0;
12915         bp->func = PCI_FUNC(pdev->devfn);
12916
12917         rc = pci_enable_device(pdev);
12918         if (rc) {
12919                 dev_err(&bp->pdev->dev,
12920                         "Cannot enable PCI device, aborting\n");
12921                 goto err_out;
12922         }
12923
12924         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12925                 dev_err(&bp->pdev->dev,
12926                         "Cannot find PCI device base address, aborting\n");
12927                 rc = -ENODEV;
12928                 goto err_out_disable;
12929         }
12930
12931         if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
12932                 dev_err(&bp->pdev->dev, "Cannot find second PCI device"
12933                        " base address, aborting\n");
12934                 rc = -ENODEV;
12935                 goto err_out_disable;
12936         }
12937
12938         if (atomic_read(&pdev->enable_cnt) == 1) {
12939                 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12940                 if (rc) {
12941                         dev_err(&bp->pdev->dev,
12942                                 "Cannot obtain PCI resources, aborting\n");
12943                         goto err_out_disable;
12944                 }
12945
12946                 pci_set_master(pdev);
12947                 pci_save_state(pdev);
12948         }
12949
12950         bp->pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
12951         if (bp->pm_cap == 0) {
12952                 dev_err(&bp->pdev->dev,
12953                         "Cannot find power management capability, aborting\n");
12954                 rc = -EIO;
12955                 goto err_out_release;
12956         }
12957
12958         bp->pcie_cap = pci_find_capability(pdev, PCI_CAP_ID_EXP);
12959         if (bp->pcie_cap == 0) {
12960                 dev_err(&bp->pdev->dev,
12961                         "Cannot find PCI Express capability, aborting\n");
12962                 rc = -EIO;
12963                 goto err_out_release;
12964         }
12965
12966         if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) == 0) {
12967                 bp->flags |= USING_DAC_FLAG;
12968                 if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)) != 0) {
12969                         dev_err(&bp->pdev->dev, "dma_set_coherent_mask"
12970                                " failed, aborting\n");
12971                         rc = -EIO;
12972                         goto err_out_release;
12973                 }
12974
12975         } else if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12976                 dev_err(&bp->pdev->dev,
12977                         "System does not support DMA, aborting\n");
12978                 rc = -EIO;
12979                 goto err_out_release;
12980         }
12981
12982         dev->mem_start = pci_resource_start(pdev, 0);
12983         dev->base_addr = dev->mem_start;
12984         dev->mem_end = pci_resource_end(pdev, 0);
12985
12986         dev->irq = pdev->irq;
12987
12988         bp->regview = pci_ioremap_bar(pdev, 0);
12989         if (!bp->regview) {
12990                 dev_err(&bp->pdev->dev,
12991                         "Cannot map register space, aborting\n");
12992                 rc = -ENOMEM;
12993                 goto err_out_release;
12994         }
12995
12996         bp->doorbells = ioremap_nocache(pci_resource_start(pdev, 2),
12997                                         min_t(u64, BNX2X_DB_SIZE,
12998                                               pci_resource_len(pdev, 2)));
12999         if (!bp->doorbells) {
13000                 dev_err(&bp->pdev->dev,
13001                         "Cannot map doorbell space, aborting\n");
13002                 rc = -ENOMEM;
13003                 goto err_out_unmap;
13004         }
13005
13006         bnx2x_set_power_state(bp, PCI_D0);
13007
13008         /* clean indirect addresses */
13009         pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS,
13010                                PCICFG_VENDOR_ID_OFFSET);
13011         REG_WR(bp, PXP2_REG_PGL_ADDR_88_F0 + BP_PORT(bp)*16, 0);
13012         REG_WR(bp, PXP2_REG_PGL_ADDR_8C_F0 + BP_PORT(bp)*16, 0);
13013         REG_WR(bp, PXP2_REG_PGL_ADDR_90_F0 + BP_PORT(bp)*16, 0);
13014         REG_WR(bp, PXP2_REG_PGL_ADDR_94_F0 + BP_PORT(bp)*16, 0);
13015
13016         /* Reset the load counter */
13017         bnx2x_clear_load_cnt(bp);
13018
13019         dev->watchdog_timeo = TX_TIMEOUT;
13020
13021         dev->netdev_ops = &bnx2x_netdev_ops;
13022         dev->ethtool_ops = &bnx2x_ethtool_ops;
13023         dev->features |= NETIF_F_SG;
13024         dev->features |= NETIF_F_HW_CSUM;
13025         if (bp->flags & USING_DAC_FLAG)
13026                 dev->features |= NETIF_F_HIGHDMA;
13027         dev->features |= (NETIF_F_TSO | NETIF_F_TSO_ECN);
13028         dev->features |= NETIF_F_TSO6;
13029 #ifdef BCM_VLAN
13030         dev->features |= (NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX);
13031         bp->flags |= (HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
13032
13033         dev->vlan_features |= NETIF_F_SG;
13034         dev->vlan_features |= NETIF_F_HW_CSUM;
13035         if (bp->flags & USING_DAC_FLAG)
13036                 dev->vlan_features |= NETIF_F_HIGHDMA;
13037         dev->vlan_features |= (NETIF_F_TSO | NETIF_F_TSO_ECN);
13038         dev->vlan_features |= NETIF_F_TSO6;
13039 #endif
13040
13041         /* get_port_hwinfo() will set prtad and mmds properly */
13042         bp->mdio.prtad = MDIO_PRTAD_NONE;
13043         bp->mdio.mmds = 0;
13044         bp->mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
13045         bp->mdio.dev = dev;
13046         bp->mdio.mdio_read = bnx2x_mdio_read;
13047         bp->mdio.mdio_write = bnx2x_mdio_write;
13048
13049         return 0;
13050
13051 err_out_unmap:
13052         if (bp->regview) {
13053                 iounmap(bp->regview);
13054                 bp->regview = NULL;
13055         }
13056         if (bp->doorbells) {
13057                 iounmap(bp->doorbells);
13058                 bp->doorbells = NULL;
13059         }
13060
13061 err_out_release:
13062         if (atomic_read(&pdev->enable_cnt) == 1)
13063                 pci_release_regions(pdev);
13064
13065 err_out_disable:
13066         pci_disable_device(pdev);
13067         pci_set_drvdata(pdev, NULL);
13068
13069 err_out:
13070         return rc;
13071 }
13072
13073 static void __devinit bnx2x_get_pcie_width_speed(struct bnx2x *bp,
13074                                                  int *width, int *speed)
13075 {
13076         u32 val = REG_RD(bp, PCICFG_OFFSET + PCICFG_LINK_CONTROL);
13077
13078         *width = (val & PCICFG_LINK_WIDTH) >> PCICFG_LINK_WIDTH_SHIFT;
13079
13080         /* return value of 1=2.5GHz 2=5GHz */
13081         *speed = (val & PCICFG_LINK_SPEED) >> PCICFG_LINK_SPEED_SHIFT;
13082 }
13083
13084 static int __devinit bnx2x_check_firmware(struct bnx2x *bp)
13085 {
13086         const struct firmware *firmware = bp->firmware;
13087         struct bnx2x_fw_file_hdr *fw_hdr;
13088         struct bnx2x_fw_file_section *sections;
13089         u32 offset, len, num_ops;
13090         u16 *ops_offsets;
13091         int i;
13092         const u8 *fw_ver;
13093
13094         if (firmware->size < sizeof(struct bnx2x_fw_file_hdr))
13095                 return -EINVAL;
13096
13097         fw_hdr = (struct bnx2x_fw_file_hdr *)firmware->data;
13098         sections = (struct bnx2x_fw_file_section *)fw_hdr;
13099
13100         /* Make sure none of the offsets and sizes make us read beyond
13101          * the end of the firmware data */
13102         for (i = 0; i < sizeof(*fw_hdr) / sizeof(*sections); i++) {
13103                 offset = be32_to_cpu(sections[i].offset);
13104                 len = be32_to_cpu(sections[i].len);
13105                 if (offset + len > firmware->size) {
13106                         dev_err(&bp->pdev->dev,
13107                                 "Section %d length is out of bounds\n", i);
13108                         return -EINVAL;
13109                 }
13110         }
13111
13112         /* Likewise for the init_ops offsets */
13113         offset = be32_to_cpu(fw_hdr->init_ops_offsets.offset);
13114         ops_offsets = (u16 *)(firmware->data + offset);
13115         num_ops = be32_to_cpu(fw_hdr->init_ops.len) / sizeof(struct raw_op);
13116
13117         for (i = 0; i < be32_to_cpu(fw_hdr->init_ops_offsets.len) / 2; i++) {
13118                 if (be16_to_cpu(ops_offsets[i]) > num_ops) {
13119                         dev_err(&bp->pdev->dev,
13120                                 "Section offset %d is out of bounds\n", i);
13121                         return -EINVAL;
13122                 }
13123         }
13124
13125         /* Check FW version */
13126         offset = be32_to_cpu(fw_hdr->fw_version.offset);
13127         fw_ver = firmware->data + offset;
13128         if ((fw_ver[0] != BCM_5710_FW_MAJOR_VERSION) ||
13129             (fw_ver[1] != BCM_5710_FW_MINOR_VERSION) ||
13130             (fw_ver[2] != BCM_5710_FW_REVISION_VERSION) ||
13131             (fw_ver[3] != BCM_5710_FW_ENGINEERING_VERSION)) {
13132                 dev_err(&bp->pdev->dev,
13133                         "Bad FW version:%d.%d.%d.%d. Should be %d.%d.%d.%d\n",
13134                        fw_ver[0], fw_ver[1], fw_ver[2],
13135                        fw_ver[3], BCM_5710_FW_MAJOR_VERSION,
13136                        BCM_5710_FW_MINOR_VERSION,
13137                        BCM_5710_FW_REVISION_VERSION,
13138                        BCM_5710_FW_ENGINEERING_VERSION);
13139                 return -EINVAL;
13140         }
13141
13142         return 0;
13143 }
13144
13145 static inline void be32_to_cpu_n(const u8 *_source, u8 *_target, u32 n)
13146 {
13147         const __be32 *source = (const __be32 *)_source;
13148         u32 *target = (u32 *)_target;
13149         u32 i;
13150
13151         for (i = 0; i < n/4; i++)
13152                 target[i] = be32_to_cpu(source[i]);
13153 }
13154
13155 /*
13156    Ops array is stored in the following format:
13157    {op(8bit), offset(24bit, big endian), data(32bit, big endian)}
13158  */
13159 static inline void bnx2x_prep_ops(const u8 *_source, u8 *_target, u32 n)
13160 {
13161         const __be32 *source = (const __be32 *)_source;
13162         struct raw_op *target = (struct raw_op *)_target;
13163         u32 i, j, tmp;
13164
13165         for (i = 0, j = 0; i < n/8; i++, j += 2) {
13166                 tmp = be32_to_cpu(source[j]);
13167                 target[i].op = (tmp >> 24) & 0xff;
13168                 target[i].offset = tmp & 0xffffff;
13169                 target[i].raw_data = be32_to_cpu(source[j + 1]);
13170         }
13171 }
13172
13173 static inline void be16_to_cpu_n(const u8 *_source, u8 *_target, u32 n)
13174 {
13175         const __be16 *source = (const __be16 *)_source;
13176         u16 *target = (u16 *)_target;
13177         u32 i;
13178
13179         for (i = 0; i < n/2; i++)
13180                 target[i] = be16_to_cpu(source[i]);
13181 }
13182
13183 #define BNX2X_ALLOC_AND_SET(arr, lbl, func)                             \
13184 do {                                                                    \
13185         u32 len = be32_to_cpu(fw_hdr->arr.len);                         \
13186         bp->arr = kmalloc(len, GFP_KERNEL);                             \
13187         if (!bp->arr) {                                                 \
13188                 pr_err("Failed to allocate %d bytes for "#arr"\n", len); \
13189                 goto lbl;                                               \
13190         }                                                               \
13191         func(bp->firmware->data + be32_to_cpu(fw_hdr->arr.offset),      \
13192              (u8 *)bp->arr, len);                                       \
13193 } while (0)
13194
13195 static int __devinit bnx2x_init_firmware(struct bnx2x *bp, struct device *dev)
13196 {
13197         const char *fw_file_name;
13198         struct bnx2x_fw_file_hdr *fw_hdr;
13199         int rc;
13200
13201         if (CHIP_IS_E1(bp))
13202                 fw_file_name = FW_FILE_NAME_E1;
13203         else if (CHIP_IS_E1H(bp))
13204                 fw_file_name = FW_FILE_NAME_E1H;
13205         else {
13206                 dev_err(dev, "Unsupported chip revision\n");
13207                 return -EINVAL;
13208         }
13209
13210         dev_info(dev, "Loading %s\n", fw_file_name);
13211
13212         rc = request_firmware(&bp->firmware, fw_file_name, dev);
13213         if (rc) {
13214                 dev_err(dev, "Can't load firmware file %s\n", fw_file_name);
13215                 goto request_firmware_exit;
13216         }
13217
13218         rc = bnx2x_check_firmware(bp);
13219         if (rc) {
13220                 dev_err(dev, "Corrupt firmware file %s\n", fw_file_name);
13221                 goto request_firmware_exit;
13222         }
13223
13224         fw_hdr = (struct bnx2x_fw_file_hdr *)bp->firmware->data;
13225
13226         /* Initialize the pointers to the init arrays */
13227         /* Blob */
13228         BNX2X_ALLOC_AND_SET(init_data, request_firmware_exit, be32_to_cpu_n);
13229
13230         /* Opcodes */
13231         BNX2X_ALLOC_AND_SET(init_ops, init_ops_alloc_err, bnx2x_prep_ops);
13232
13233         /* Offsets */
13234         BNX2X_ALLOC_AND_SET(init_ops_offsets, init_offsets_alloc_err,
13235                             be16_to_cpu_n);
13236
13237         /* STORMs firmware */
13238         INIT_TSEM_INT_TABLE_DATA(bp) = bp->firmware->data +
13239                         be32_to_cpu(fw_hdr->tsem_int_table_data.offset);
13240         INIT_TSEM_PRAM_DATA(bp)      = bp->firmware->data +
13241                         be32_to_cpu(fw_hdr->tsem_pram_data.offset);
13242         INIT_USEM_INT_TABLE_DATA(bp) = bp->firmware->data +
13243                         be32_to_cpu(fw_hdr->usem_int_table_data.offset);
13244         INIT_USEM_PRAM_DATA(bp)      = bp->firmware->data +
13245                         be32_to_cpu(fw_hdr->usem_pram_data.offset);
13246         INIT_XSEM_INT_TABLE_DATA(bp) = bp->firmware->data +
13247                         be32_to_cpu(fw_hdr->xsem_int_table_data.offset);
13248         INIT_XSEM_PRAM_DATA(bp)      = bp->firmware->data +
13249                         be32_to_cpu(fw_hdr->xsem_pram_data.offset);
13250         INIT_CSEM_INT_TABLE_DATA(bp) = bp->firmware->data +
13251                         be32_to_cpu(fw_hdr->csem_int_table_data.offset);
13252         INIT_CSEM_PRAM_DATA(bp)      = bp->firmware->data +
13253                         be32_to_cpu(fw_hdr->csem_pram_data.offset);
13254
13255         return 0;
13256
13257 init_offsets_alloc_err:
13258         kfree(bp->init_ops);
13259 init_ops_alloc_err:
13260         kfree(bp->init_data);
13261 request_firmware_exit:
13262         release_firmware(bp->firmware);
13263
13264         return rc;
13265 }
13266
13267
13268 static int __devinit bnx2x_init_one(struct pci_dev *pdev,
13269                                     const struct pci_device_id *ent)
13270 {
13271         struct net_device *dev = NULL;
13272         struct bnx2x *bp;
13273         int pcie_width, pcie_speed;
13274         int rc;
13275
13276         /* dev zeroed in init_etherdev */
13277         dev = alloc_etherdev_mq(sizeof(*bp), MAX_CONTEXT);
13278         if (!dev) {
13279                 dev_err(&pdev->dev, "Cannot allocate net device\n");
13280                 return -ENOMEM;
13281         }
13282
13283         bp = netdev_priv(dev);
13284         bp->msg_enable = debug;
13285
13286         pci_set_drvdata(pdev, dev);
13287
13288         rc = bnx2x_init_dev(pdev, dev);
13289         if (rc < 0) {
13290                 free_netdev(dev);
13291                 return rc;
13292         }
13293
13294         rc = bnx2x_init_bp(bp);
13295         if (rc)
13296                 goto init_one_exit;
13297
13298         /* Set init arrays */
13299         rc = bnx2x_init_firmware(bp, &pdev->dev);
13300         if (rc) {
13301                 dev_err(&pdev->dev, "Error loading firmware\n");
13302                 goto init_one_exit;
13303         }
13304
13305         rc = register_netdev(dev);
13306         if (rc) {
13307                 dev_err(&pdev->dev, "Cannot register net device\n");
13308                 goto init_one_exit;
13309         }
13310
13311         bnx2x_get_pcie_width_speed(bp, &pcie_width, &pcie_speed);
13312         netdev_info(dev, "%s (%c%d) PCI-E x%d %s found at mem %lx,"
13313                " IRQ %d, ", board_info[ent->driver_data].name,
13314                (CHIP_REV(bp) >> 12) + 'A', (CHIP_METAL(bp) >> 4),
13315                pcie_width, (pcie_speed == 2) ? "5GHz (Gen2)" : "2.5GHz",
13316                dev->base_addr, bp->pdev->irq);
13317         pr_cont("node addr %pM\n", dev->dev_addr);
13318
13319         return 0;
13320
13321 init_one_exit:
13322         if (bp->regview)
13323                 iounmap(bp->regview);
13324
13325         if (bp->doorbells)
13326                 iounmap(bp->doorbells);
13327
13328         free_netdev(dev);
13329
13330         if (atomic_read(&pdev->enable_cnt) == 1)
13331                 pci_release_regions(pdev);
13332
13333         pci_disable_device(pdev);
13334         pci_set_drvdata(pdev, NULL);
13335
13336         return rc;
13337 }
13338
13339 static void __devexit bnx2x_remove_one(struct pci_dev *pdev)
13340 {
13341         struct net_device *dev = pci_get_drvdata(pdev);
13342         struct bnx2x *bp;
13343
13344         if (!dev) {
13345                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
13346                 return;
13347         }
13348         bp = netdev_priv(dev);
13349
13350         unregister_netdev(dev);
13351
13352         /* Make sure RESET task is not scheduled before continuing */
13353         cancel_delayed_work_sync(&bp->reset_task);
13354
13355         kfree(bp->init_ops_offsets);
13356         kfree(bp->init_ops);
13357         kfree(bp->init_data);
13358         release_firmware(bp->firmware);
13359
13360         if (bp->regview)
13361                 iounmap(bp->regview);
13362
13363         if (bp->doorbells)
13364                 iounmap(bp->doorbells);
13365
13366         free_netdev(dev);
13367
13368         if (atomic_read(&pdev->enable_cnt) == 1)
13369                 pci_release_regions(pdev);
13370
13371         pci_disable_device(pdev);
13372         pci_set_drvdata(pdev, NULL);
13373 }
13374
13375 static int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
13376 {
13377         struct net_device *dev = pci_get_drvdata(pdev);
13378         struct bnx2x *bp;
13379
13380         if (!dev) {
13381                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
13382                 return -ENODEV;
13383         }
13384         bp = netdev_priv(dev);
13385
13386         rtnl_lock();
13387
13388         pci_save_state(pdev);
13389
13390         if (!netif_running(dev)) {
13391                 rtnl_unlock();
13392                 return 0;
13393         }
13394
13395         netif_device_detach(dev);
13396
13397         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
13398
13399         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
13400
13401         rtnl_unlock();
13402
13403         return 0;
13404 }
13405
13406 static int bnx2x_resume(struct pci_dev *pdev)
13407 {
13408         struct net_device *dev = pci_get_drvdata(pdev);
13409         struct bnx2x *bp;
13410         int rc;
13411
13412         if (!dev) {
13413                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
13414                 return -ENODEV;
13415         }
13416         bp = netdev_priv(dev);
13417
13418         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
13419                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
13420                 return -EAGAIN;
13421         }
13422
13423         rtnl_lock();
13424
13425         pci_restore_state(pdev);
13426
13427         if (!netif_running(dev)) {
13428                 rtnl_unlock();
13429                 return 0;
13430         }
13431
13432         bnx2x_set_power_state(bp, PCI_D0);
13433         netif_device_attach(dev);
13434
13435         rc = bnx2x_nic_load(bp, LOAD_OPEN);
13436
13437         rtnl_unlock();
13438
13439         return rc;
13440 }
13441
13442 static int bnx2x_eeh_nic_unload(struct bnx2x *bp)
13443 {
13444         int i;
13445
13446         bp->state = BNX2X_STATE_ERROR;
13447
13448         bp->rx_mode = BNX2X_RX_MODE_NONE;
13449
13450         bnx2x_netif_stop(bp, 0);
13451         netif_carrier_off(bp->dev);
13452
13453         del_timer_sync(&bp->timer);
13454         bp->stats_state = STATS_STATE_DISABLED;
13455         DP(BNX2X_MSG_STATS, "stats_state - DISABLED\n");
13456
13457         /* Release IRQs */
13458         bnx2x_free_irq(bp, false);
13459
13460         if (CHIP_IS_E1(bp)) {
13461                 struct mac_configuration_cmd *config =
13462                                                 bnx2x_sp(bp, mcast_config);
13463
13464                 for (i = 0; i < config->hdr.length; i++)
13465                         CAM_INVALIDATE(config->config_table[i]);
13466         }
13467
13468         /* Free SKBs, SGEs, TPA pool and driver internals */
13469         bnx2x_free_skbs(bp);
13470         for_each_queue(bp, i)
13471                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
13472         for_each_queue(bp, i)
13473                 netif_napi_del(&bnx2x_fp(bp, i, napi));
13474         bnx2x_free_mem(bp);
13475
13476         bp->state = BNX2X_STATE_CLOSED;
13477
13478         return 0;
13479 }
13480
13481 static void bnx2x_eeh_recover(struct bnx2x *bp)
13482 {
13483         u32 val;
13484
13485         mutex_init(&bp->port.phy_mutex);
13486
13487         bp->common.shmem_base = REG_RD(bp, MISC_REG_SHARED_MEM_ADDR);
13488         bp->link_params.shmem_base = bp->common.shmem_base;
13489         BNX2X_DEV_INFO("shmem offset is 0x%x\n", bp->common.shmem_base);
13490
13491         if (!bp->common.shmem_base ||
13492             (bp->common.shmem_base < 0xA0000) ||
13493             (bp->common.shmem_base >= 0xC0000)) {
13494                 BNX2X_DEV_INFO("MCP not active\n");
13495                 bp->flags |= NO_MCP_FLAG;
13496                 return;
13497         }
13498
13499         val = SHMEM_RD(bp, validity_map[BP_PORT(bp)]);
13500         if ((val & (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
13501                 != (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB))
13502                 BNX2X_ERR("BAD MCP validity signature\n");
13503
13504         if (!BP_NOMCP(bp)) {
13505                 bp->fw_seq = (SHMEM_RD(bp, func_mb[BP_FUNC(bp)].drv_mb_header)
13506                               & DRV_MSG_SEQ_NUMBER_MASK);
13507                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
13508         }
13509 }
13510
13511 /**
13512  * bnx2x_io_error_detected - called when PCI error is detected
13513  * @pdev: Pointer to PCI device
13514  * @state: The current pci connection state
13515  *
13516  * This function is called after a PCI bus error affecting
13517  * this device has been detected.
13518  */
13519 static pci_ers_result_t bnx2x_io_error_detected(struct pci_dev *pdev,
13520                                                 pci_channel_state_t state)
13521 {
13522         struct net_device *dev = pci_get_drvdata(pdev);
13523         struct bnx2x *bp = netdev_priv(dev);
13524
13525         rtnl_lock();
13526
13527         netif_device_detach(dev);
13528
13529         if (state == pci_channel_io_perm_failure) {
13530                 rtnl_unlock();
13531                 return PCI_ERS_RESULT_DISCONNECT;
13532         }
13533
13534         if (netif_running(dev))
13535                 bnx2x_eeh_nic_unload(bp);
13536
13537         pci_disable_device(pdev);
13538
13539         rtnl_unlock();
13540
13541         /* Request a slot reset */
13542         return PCI_ERS_RESULT_NEED_RESET;
13543 }
13544
13545 /**
13546  * bnx2x_io_slot_reset - called after the PCI bus has been reset
13547  * @pdev: Pointer to PCI device
13548  *
13549  * Restart the card from scratch, as if from a cold-boot.
13550  */
13551 static pci_ers_result_t bnx2x_io_slot_reset(struct pci_dev *pdev)
13552 {
13553         struct net_device *dev = pci_get_drvdata(pdev);
13554         struct bnx2x *bp = netdev_priv(dev);
13555
13556         rtnl_lock();
13557
13558         if (pci_enable_device(pdev)) {
13559                 dev_err(&pdev->dev,
13560                         "Cannot re-enable PCI device after reset\n");
13561                 rtnl_unlock();
13562                 return PCI_ERS_RESULT_DISCONNECT;
13563         }
13564
13565         pci_set_master(pdev);
13566         pci_restore_state(pdev);
13567
13568         if (netif_running(dev))
13569                 bnx2x_set_power_state(bp, PCI_D0);
13570
13571         rtnl_unlock();
13572
13573         return PCI_ERS_RESULT_RECOVERED;
13574 }
13575
13576 /**
13577  * bnx2x_io_resume - called when traffic can start flowing again
13578  * @pdev: Pointer to PCI device
13579  *
13580  * This callback is called when the error recovery driver tells us that
13581  * its OK to resume normal operation.
13582  */
13583 static void bnx2x_io_resume(struct pci_dev *pdev)
13584 {
13585         struct net_device *dev = pci_get_drvdata(pdev);
13586         struct bnx2x *bp = netdev_priv(dev);
13587
13588         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
13589                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
13590                 return;
13591         }
13592
13593         rtnl_lock();
13594
13595         bnx2x_eeh_recover(bp);
13596
13597         if (netif_running(dev))
13598                 bnx2x_nic_load(bp, LOAD_NORMAL);
13599
13600         netif_device_attach(dev);
13601
13602         rtnl_unlock();
13603 }
13604
13605 static struct pci_error_handlers bnx2x_err_handler = {
13606         .error_detected = bnx2x_io_error_detected,
13607         .slot_reset     = bnx2x_io_slot_reset,
13608         .resume         = bnx2x_io_resume,
13609 };
13610
13611 static struct pci_driver bnx2x_pci_driver = {
13612         .name        = DRV_MODULE_NAME,
13613         .id_table    = bnx2x_pci_tbl,
13614         .probe       = bnx2x_init_one,
13615         .remove      = __devexit_p(bnx2x_remove_one),
13616         .suspend     = bnx2x_suspend,
13617         .resume      = bnx2x_resume,
13618         .err_handler = &bnx2x_err_handler,
13619 };
13620
13621 static int __init bnx2x_init(void)
13622 {
13623         int ret;
13624
13625         pr_info("%s", version);
13626
13627         bnx2x_wq = create_singlethread_workqueue("bnx2x");
13628         if (bnx2x_wq == NULL) {
13629                 pr_err("Cannot create workqueue\n");
13630                 return -ENOMEM;
13631         }
13632
13633         ret = pci_register_driver(&bnx2x_pci_driver);
13634         if (ret) {
13635                 pr_err("Cannot register driver\n");
13636                 destroy_workqueue(bnx2x_wq);
13637         }
13638         return ret;
13639 }
13640
13641 static void __exit bnx2x_cleanup(void)
13642 {
13643         pci_unregister_driver(&bnx2x_pci_driver);
13644
13645         destroy_workqueue(bnx2x_wq);
13646 }
13647
13648 module_init(bnx2x_init);
13649 module_exit(bnx2x_cleanup);
13650
13651 #ifdef BCM_CNIC
13652
13653 /* count denotes the number of new completions we have seen */
13654 static void bnx2x_cnic_sp_post(struct bnx2x *bp, int count)
13655 {
13656         struct eth_spe *spe;
13657
13658 #ifdef BNX2X_STOP_ON_ERROR
13659         if (unlikely(bp->panic))
13660                 return;
13661 #endif
13662
13663         spin_lock_bh(&bp->spq_lock);
13664         bp->cnic_spq_pending -= count;
13665
13666         for (; bp->cnic_spq_pending < bp->cnic_eth_dev.max_kwqe_pending;
13667              bp->cnic_spq_pending++) {
13668
13669                 if (!bp->cnic_kwq_pending)
13670                         break;
13671
13672                 spe = bnx2x_sp_get_next(bp);
13673                 *spe = *bp->cnic_kwq_cons;
13674
13675                 bp->cnic_kwq_pending--;
13676
13677                 DP(NETIF_MSG_TIMER, "pending on SPQ %d, on KWQ %d count %d\n",
13678                    bp->cnic_spq_pending, bp->cnic_kwq_pending, count);
13679
13680                 if (bp->cnic_kwq_cons == bp->cnic_kwq_last)
13681                         bp->cnic_kwq_cons = bp->cnic_kwq;
13682                 else
13683                         bp->cnic_kwq_cons++;
13684         }
13685         bnx2x_sp_prod_update(bp);
13686         spin_unlock_bh(&bp->spq_lock);
13687 }
13688
13689 static int bnx2x_cnic_sp_queue(struct net_device *dev,
13690                                struct kwqe_16 *kwqes[], u32 count)
13691 {
13692         struct bnx2x *bp = netdev_priv(dev);
13693         int i;
13694
13695 #ifdef BNX2X_STOP_ON_ERROR
13696         if (unlikely(bp->panic))
13697                 return -EIO;
13698 #endif
13699
13700         spin_lock_bh(&bp->spq_lock);
13701
13702         for (i = 0; i < count; i++) {
13703                 struct eth_spe *spe = (struct eth_spe *)kwqes[i];
13704
13705                 if (bp->cnic_kwq_pending == MAX_SP_DESC_CNT)
13706                         break;
13707
13708                 *bp->cnic_kwq_prod = *spe;
13709
13710                 bp->cnic_kwq_pending++;
13711
13712                 DP(NETIF_MSG_TIMER, "L5 SPQE %x %x %x:%x pos %d\n",
13713                    spe->hdr.conn_and_cmd_data, spe->hdr.type,
13714                    spe->data.mac_config_addr.hi,
13715                    spe->data.mac_config_addr.lo,
13716                    bp->cnic_kwq_pending);
13717
13718                 if (bp->cnic_kwq_prod == bp->cnic_kwq_last)
13719                         bp->cnic_kwq_prod = bp->cnic_kwq;
13720                 else
13721                         bp->cnic_kwq_prod++;
13722         }
13723
13724         spin_unlock_bh(&bp->spq_lock);
13725
13726         if (bp->cnic_spq_pending < bp->cnic_eth_dev.max_kwqe_pending)
13727                 bnx2x_cnic_sp_post(bp, 0);
13728
13729         return i;
13730 }
13731
13732 static int bnx2x_cnic_ctl_send(struct bnx2x *bp, struct cnic_ctl_info *ctl)
13733 {
13734         struct cnic_ops *c_ops;
13735         int rc = 0;
13736
13737         mutex_lock(&bp->cnic_mutex);
13738         c_ops = bp->cnic_ops;
13739         if (c_ops)
13740                 rc = c_ops->cnic_ctl(bp->cnic_data, ctl);
13741         mutex_unlock(&bp->cnic_mutex);
13742
13743         return rc;
13744 }
13745
13746 static int bnx2x_cnic_ctl_send_bh(struct bnx2x *bp, struct cnic_ctl_info *ctl)
13747 {
13748         struct cnic_ops *c_ops;
13749         int rc = 0;
13750
13751         rcu_read_lock();
13752         c_ops = rcu_dereference(bp->cnic_ops);
13753         if (c_ops)
13754                 rc = c_ops->cnic_ctl(bp->cnic_data, ctl);
13755         rcu_read_unlock();
13756
13757         return rc;
13758 }
13759
13760 /*
13761  * for commands that have no data
13762  */
13763 static int bnx2x_cnic_notify(struct bnx2x *bp, int cmd)
13764 {
13765         struct cnic_ctl_info ctl = {0};
13766
13767         ctl.cmd = cmd;
13768
13769         return bnx2x_cnic_ctl_send(bp, &ctl);
13770 }
13771
13772 static void bnx2x_cnic_cfc_comp(struct bnx2x *bp, int cid)
13773 {
13774         struct cnic_ctl_info ctl;
13775
13776         /* first we tell CNIC and only then we count this as a completion */
13777         ctl.cmd = CNIC_CTL_COMPLETION_CMD;
13778         ctl.data.comp.cid = cid;
13779
13780         bnx2x_cnic_ctl_send_bh(bp, &ctl);
13781         bnx2x_cnic_sp_post(bp, 1);
13782 }
13783
13784 static int bnx2x_drv_ctl(struct net_device *dev, struct drv_ctl_info *ctl)
13785 {
13786         struct bnx2x *bp = netdev_priv(dev);
13787         int rc = 0;
13788
13789         switch (ctl->cmd) {
13790         case DRV_CTL_CTXTBL_WR_CMD: {
13791                 u32 index = ctl->data.io.offset;
13792                 dma_addr_t addr = ctl->data.io.dma_addr;
13793
13794                 bnx2x_ilt_wr(bp, index, addr);
13795                 break;
13796         }
13797
13798         case DRV_CTL_COMPLETION_CMD: {
13799                 int count = ctl->data.comp.comp_count;
13800
13801                 bnx2x_cnic_sp_post(bp, count);
13802                 break;
13803         }
13804
13805         /* rtnl_lock is held.  */
13806         case DRV_CTL_START_L2_CMD: {
13807                 u32 cli = ctl->data.ring.client_id;
13808
13809                 bp->rx_mode_cl_mask |= (1 << cli);
13810                 bnx2x_set_storm_rx_mode(bp);
13811                 break;
13812         }
13813
13814         /* rtnl_lock is held.  */
13815         case DRV_CTL_STOP_L2_CMD: {
13816                 u32 cli = ctl->data.ring.client_id;
13817
13818                 bp->rx_mode_cl_mask &= ~(1 << cli);
13819                 bnx2x_set_storm_rx_mode(bp);
13820                 break;
13821         }
13822
13823         default:
13824                 BNX2X_ERR("unknown command %x\n", ctl->cmd);
13825                 rc = -EINVAL;
13826         }
13827
13828         return rc;
13829 }
13830
13831 static void bnx2x_setup_cnic_irq_info(struct bnx2x *bp)
13832 {
13833         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
13834
13835         if (bp->flags & USING_MSIX_FLAG) {
13836                 cp->drv_state |= CNIC_DRV_STATE_USING_MSIX;
13837                 cp->irq_arr[0].irq_flags |= CNIC_IRQ_FL_MSIX;
13838                 cp->irq_arr[0].vector = bp->msix_table[1].vector;
13839         } else {
13840                 cp->drv_state &= ~CNIC_DRV_STATE_USING_MSIX;
13841                 cp->irq_arr[0].irq_flags &= ~CNIC_IRQ_FL_MSIX;
13842         }
13843         cp->irq_arr[0].status_blk = bp->cnic_sb;
13844         cp->irq_arr[0].status_blk_num = CNIC_SB_ID(bp);
13845         cp->irq_arr[1].status_blk = bp->def_status_blk;
13846         cp->irq_arr[1].status_blk_num = DEF_SB_ID;
13847
13848         cp->num_irq = 2;
13849 }
13850
13851 static int bnx2x_register_cnic(struct net_device *dev, struct cnic_ops *ops,
13852                                void *data)
13853 {
13854         struct bnx2x *bp = netdev_priv(dev);
13855         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
13856
13857         if (ops == NULL)
13858                 return -EINVAL;
13859
13860         if (atomic_read(&bp->intr_sem) != 0)
13861                 return -EBUSY;
13862
13863         bp->cnic_kwq = kzalloc(PAGE_SIZE, GFP_KERNEL);
13864         if (!bp->cnic_kwq)
13865                 return -ENOMEM;
13866
13867         bp->cnic_kwq_cons = bp->cnic_kwq;
13868         bp->cnic_kwq_prod = bp->cnic_kwq;
13869         bp->cnic_kwq_last = bp->cnic_kwq + MAX_SP_DESC_CNT;
13870
13871         bp->cnic_spq_pending = 0;
13872         bp->cnic_kwq_pending = 0;
13873
13874         bp->cnic_data = data;
13875
13876         cp->num_irq = 0;
13877         cp->drv_state = CNIC_DRV_STATE_REGD;
13878
13879         bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping, CNIC_SB_ID(bp));
13880
13881         bnx2x_setup_cnic_irq_info(bp);
13882         bnx2x_set_iscsi_eth_mac_addr(bp, 1);
13883         bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
13884         rcu_assign_pointer(bp->cnic_ops, ops);
13885
13886         return 0;
13887 }
13888
13889 static int bnx2x_unregister_cnic(struct net_device *dev)
13890 {
13891         struct bnx2x *bp = netdev_priv(dev);
13892         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
13893
13894         mutex_lock(&bp->cnic_mutex);
13895         if (bp->cnic_flags & BNX2X_CNIC_FLAG_MAC_SET) {
13896                 bp->cnic_flags &= ~BNX2X_CNIC_FLAG_MAC_SET;
13897                 bnx2x_set_iscsi_eth_mac_addr(bp, 0);
13898         }
13899         cp->drv_state = 0;
13900         rcu_assign_pointer(bp->cnic_ops, NULL);
13901         mutex_unlock(&bp->cnic_mutex);
13902         synchronize_rcu();
13903         kfree(bp->cnic_kwq);
13904         bp->cnic_kwq = NULL;
13905
13906         return 0;
13907 }
13908
13909 struct cnic_eth_dev *bnx2x_cnic_probe(struct net_device *dev)
13910 {
13911         struct bnx2x *bp = netdev_priv(dev);
13912         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
13913
13914         cp->drv_owner = THIS_MODULE;
13915         cp->chip_id = CHIP_ID(bp);
13916         cp->pdev = bp->pdev;
13917         cp->io_base = bp->regview;
13918         cp->io_base2 = bp->doorbells;
13919         cp->max_kwqe_pending = 8;
13920         cp->ctx_blk_size = CNIC_CTX_PER_ILT * sizeof(union cdu_context);
13921         cp->ctx_tbl_offset = FUNC_ILT_BASE(BP_FUNC(bp)) + 1;
13922         cp->ctx_tbl_len = CNIC_ILT_LINES;
13923         cp->starting_cid = BCM_CNIC_CID_START;
13924         cp->drv_submit_kwqes_16 = bnx2x_cnic_sp_queue;
13925         cp->drv_ctl = bnx2x_drv_ctl;
13926         cp->drv_register_cnic = bnx2x_register_cnic;
13927         cp->drv_unregister_cnic = bnx2x_unregister_cnic;
13928
13929         return cp;
13930 }
13931 EXPORT_SYMBOL(bnx2x_cnic_probe);
13932
13933 #endif /* BCM_CNIC */
13934