drivers/net/sfc/falcon.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2006-2008 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/bitops.h>
  12 #include <linux/delay.h>
  13 #include <linux/pci.h>
  14 #include <linux/module.h>
  15 #include <linux/seq_file.h>
  16 #include <linux/i2c.h>
  17 #include <linux/mii.h>
  18 #include "net_driver.h"
  19 #include "bitfield.h"
  20 #include "efx.h"
  21 #include "mac.h"
  22 #include "spi.h"
  23 #include "falcon.h"
  24 #include "regs.h"
  25 #include "io.h"
  26 #include "mdio_10g.h"
  27 #include "phy.h"
  28 #include "workarounds.h"
  29
  30 /* Hardware control for SFC4000 (aka Falcon). */
  31
  32 /**************************************************************************
  33  *
  34  * Configurable values
  35  *
  36  **************************************************************************
  37  */
  38
  39 /* This is set to 16 for a good reason.  In summary, if larger than
  40  * 16, the descriptor cache holds more than a default socket
  41  * buffer's worth of packets (for UDP we can only have at most one
  42  * socket buffer's worth outstanding).  This combined with the fact
  43  * that we only get 1 TX event per descriptor cache means the NIC
  44  * goes idle.
  45  */
  46 #define TX_DC_ENTRIES 16
  47 #define TX_DC_ENTRIES_ORDER 1
  48
  49 #define RX_DC_ENTRIES 64
  50 #define RX_DC_ENTRIES_ORDER 3
  51
  52 static const unsigned int
  53 /* "Large" EEPROM device: Atmel AT25640 or similar
  54  * 8 KB, 16-bit address, 32 B write block */
  55 large_eeprom_type = ((13 << SPI_DEV_TYPE_SIZE_LBN)
  56                      | (2 << SPI_DEV_TYPE_ADDR_LEN_LBN)
  57                      | (5 << SPI_DEV_TYPE_BLOCK_SIZE_LBN)),
  58 /* Default flash device: Atmel AT25F1024
  59  * 128 KB, 24-bit address, 32 KB erase block, 256 B write block */
  60 default_flash_type = ((17 << SPI_DEV_TYPE_SIZE_LBN)
  61                       | (3 << SPI_DEV_TYPE_ADDR_LEN_LBN)
  62                       | (0x52 << SPI_DEV_TYPE_ERASE_CMD_LBN)
  63                       | (15 << SPI_DEV_TYPE_ERASE_SIZE_LBN)
  64                       | (8 << SPI_DEV_TYPE_BLOCK_SIZE_LBN));
  65
  66 /* RX FIFO XOFF watermark
  67  *
  68  * When the amount of the RX FIFO increases used increases past this
  69  * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A)
  70  * This also has an effect on RX/TX arbitration
  71  */
  72 int efx_nic_rx_xoff_thresh = -1;
  73 module_param_named(rx_xoff_thresh_bytes, efx_nic_rx_xoff_thresh, int, 0644);
  74 MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold");
  75
  76 /* RX FIFO XON watermark
  77  *
  78  * When the amount of the RX FIFO used decreases below this
  79  * watermark send XON. Only used if TX flow control is enabled (ethtool -A)
  80  * This also has an effect on RX/TX arbitration
  81  */
  82 int efx_nic_rx_xon_thresh = -1;
  83 module_param_named(rx_xon_thresh_bytes, efx_nic_rx_xon_thresh, int, 0644);
  84 MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold");
  85
  86 /* If EFX_MAX_INT_ERRORS internal errors occur within
  87  * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
  88  * disable it.
  89  */
  90 #define EFX_INT_ERROR_EXPIRE 3600
  91 #define EFX_MAX_INT_ERRORS 5
  92
  93 /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
  94  */
  95 #define EFX_FLUSH_INTERVAL 10
  96 #define EFX_FLUSH_POLL_COUNT 100
  97
  98 /**************************************************************************
  99  *
 100  * Falcon constants
 101  *
 102  **************************************************************************
 103  */
 104
 105 /* Size and alignment of special buffers (4KB) */
 106 #define EFX_BUF_SIZE 4096
 107
 108 /* Depth of RX flush request fifo */
 109 #define EFX_RX_FLUSH_COUNT 4
 110
 111 /**************************************************************************
 112  *
 113  * Solarstorm hardware access
 114  *
 115  **************************************************************************/
 116
 117 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
 118                                      unsigned int index)
 119 {
 120         efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
 121                         value, index);
 122 }
 123
 124 /* Read the current event from the event queue */
 125 static inline efx_qword_t *efx_event(struct efx_channel *channel,
 126                                      unsigned int index)
 127 {
 128         return (((efx_qword_t *) (channel->eventq.addr)) + index);
 129 }
 130
 131 /* See if an event is present
 132  *
 133  * We check both the high and low dword of the event for all ones.  We
 134  * wrote all ones when we cleared the event, and no valid event can
 135  * have all ones in either its high or low dwords.  This approach is
 136  * robust against reordering.
 137  *
 138  * Note that using a single 64-bit comparison is incorrect; even
 139  * though the CPU read will be atomic, the DMA write may not be.
 140  */
 141 static inline int efx_event_present(efx_qword_t *event)
 142 {
 143         return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 144                   EFX_DWORD_IS_ALL_ONES(event->dword[1])));
 145 }
 146
 147 /**************************************************************************
 148  *
 149  * I2C bus - this is a bit-bashing interface using GPIO pins
 150  * Note that it uses the output enables to tristate the outputs
 151  * SDA is the data pin and SCL is the clock
 152  *
 153  **************************************************************************
 154  */
 155 static void falcon_setsda(void *data, int state)
 156 {
 157         struct efx_nic *efx = (struct efx_nic *)data;
 158         efx_oword_t reg;
 159
 160         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
 161         EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO3_OEN, !state);
 162         efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
 163 }
 164
 165 static void falcon_setscl(void *data, int state)
 166 {
 167         struct efx_nic *efx = (struct efx_nic *)data;
 168         efx_oword_t reg;
 169
 170         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
 171         EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO0_OEN, !state);
 172         efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
 173 }
 174
 175 static int falcon_getsda(void *data)
 176 {
 177         struct efx_nic *efx = (struct efx_nic *)data;
 178         efx_oword_t reg;
 179
 180         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
 181         return EFX_OWORD_FIELD(reg, FRF_AB_GPIO3_IN);
 182 }
 183
 184 static int falcon_getscl(void *data)
 185 {
 186         struct efx_nic *efx = (struct efx_nic *)data;
 187         efx_oword_t reg;
 188
 189         efx_reado(efx, &reg, FR_AB_GPIO_CTL);
 190         return EFX_OWORD_FIELD(reg, FRF_AB_GPIO0_IN);
 191 }
 192
 193 static struct i2c_algo_bit_data falcon_i2c_bit_operations = {
 194         .setsda         = falcon_setsda,
 195         .setscl         = falcon_setscl,
 196         .getsda         = falcon_getsda,
 197         .getscl         = falcon_getscl,
 198         .udelay         = 5,
 199         /* Wait up to 50 ms for slave to let us pull SCL high */
 200         .timeout        = DIV_ROUND_UP(HZ, 20),
 201 };
 202
 203 /**************************************************************************
 204  *
 205  * Special buffer handling
 206  * Special buffers are used for event queues and the TX and RX
 207  * descriptor rings.
 208  *
 209  *************************************************************************/
 210
 211 /*
 212  * Initialise a special buffer
 213  *
 214  * This will define a buffer (previously allocated via
 215  * efx_alloc_special_buffer()) in the buffer table, allowing
 216  * it to be used for event queues, descriptor rings etc.
 217  */
 218 static void
 219 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 220 {
 221         efx_qword_t buf_desc;
 222         int index;
 223         dma_addr_t dma_addr;
 224         int i;
 225
 226         EFX_BUG_ON_PARANOID(!buffer->addr);
 227
 228         /* Write buffer descriptors to NIC */
 229         for (i = 0; i < buffer->entries; i++) {
 230                 index = buffer->index + i;
 231                 dma_addr = buffer->dma_addr + (i * 4096);
 232                 EFX_LOG(efx, "mapping special buffer %d at %llx\n",
 233                         index, (unsigned long long)dma_addr);
 234                 EFX_POPULATE_QWORD_3(buf_desc,
 235                                      FRF_AZ_BUF_ADR_REGION, 0,
 236                                      FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
 237                                      FRF_AZ_BUF_OWNER_ID_FBUF, 0);
 238                 efx_write_buf_tbl(efx, &buf_desc, index);
 239         }
 240 }
 241
 242 /* Unmaps a buffer and clears the buffer table entries */
 243 static void
 244 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 245 {
 246         efx_oword_t buf_tbl_upd;
 247         unsigned int start = buffer->index;
 248         unsigned int end = (buffer->index + buffer->entries - 1);
 249
 250         if (!buffer->entries)
 251                 return;
 252
 253         EFX_LOG(efx, "unmapping special buffers %d-%d\n",
 254                 buffer->index, buffer->index + buffer->entries - 1);
 255
 256         EFX_POPULATE_OWORD_4(buf_tbl_upd,
 257                              FRF_AZ_BUF_UPD_CMD, 0,
 258                              FRF_AZ_BUF_CLR_CMD, 1,
 259                              FRF_AZ_BUF_CLR_END_ID, end,
 260                              FRF_AZ_BUF_CLR_START_ID, start);
 261         efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
 262 }
 263
 264 /*
 265  * Allocate a new special buffer
 266  *
 267  * This allocates memory for a new buffer, clears it and allocates a
 268  * new buffer ID range.  It does not write into the buffer table.
 269  *
 270  * This call will allocate 4KB buffers, since 8KB buffers can't be
 271  * used for event queues and descriptor rings.
 272  */
 273 static int efx_alloc_special_buffer(struct efx_nic *efx,
 274                                     struct efx_special_buffer *buffer,
 275                                     unsigned int len)
 276 {
 277         len = ALIGN(len, EFX_BUF_SIZE);
 278
 279         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
 280                                             &buffer->dma_addr);
 281         if (!buffer->addr)
 282                 return -ENOMEM;
 283         buffer->len = len;
 284         buffer->entries = len / EFX_BUF_SIZE;
 285         BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
 286
 287         /* All zeros is a potentially valid event so memset to 0xff */
 288         memset(buffer->addr, 0xff, len);
 289
 290         /* Select new buffer ID */
 291         buffer->index = efx->next_buffer_table;
 292         efx->next_buffer_table += buffer->entries;
 293
 294         EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x "
 295                 "(virt %p phys %llx)\n", buffer->index,
 296                 buffer->index + buffer->entries - 1,
 297                 (u64)buffer->dma_addr, len,
 298                 buffer->addr, (u64)virt_to_phys(buffer->addr));
 299
 300         return 0;
 301 }
 302
 303 static void
 304 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 305 {
 306         if (!buffer->addr)
 307                 return;
 308
 309         EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x "
 310                 "(virt %p phys %llx)\n", buffer->index,
 311                 buffer->index + buffer->entries - 1,
 312                 (u64)buffer->dma_addr, buffer->len,
 313                 buffer->addr, (u64)virt_to_phys(buffer->addr));
 314
 315         pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr,
 316                             buffer->dma_addr);
 317         buffer->addr = NULL;
 318         buffer->entries = 0;
 319 }
 320
 321 /**************************************************************************
 322  *
 323  * Generic buffer handling
 324  * These buffers are used for interrupt status and MAC stats
 325  *
 326  **************************************************************************/
 327
 328 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 329                          unsigned int len)
 330 {
 331         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
 332                                             &buffer->dma_addr);
 333         if (!buffer->addr)
 334                 return -ENOMEM;
 335         buffer->len = len;
 336         memset(buffer->addr, 0, len);
 337         return 0;
 338 }
 339
 340 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
 341 {
 342         if (buffer->addr) {
 343                 pci_free_consistent(efx->pci_dev, buffer->len,
 344                                     buffer->addr, buffer->dma_addr);
 345                 buffer->addr = NULL;
 346         }
 347 }
 348
 349 /**************************************************************************
 350  *
 351  * TX path
 352  *
 353  **************************************************************************/
 354
 355 /* Returns a pointer to the specified transmit descriptor in the TX
 356  * descriptor queue belonging to the specified channel.
 357  */
 358 static inline efx_qword_t *
 359 efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 360 {
 361         return (((efx_qword_t *) (tx_queue->txd.addr)) + index);
 362 }
 363
 364 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
 365 static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
 366 {
 367         unsigned write_ptr;
 368         efx_dword_t reg;
 369
 370         write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
 371         EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
 372         efx_writed_page(tx_queue->efx, &reg,
 373                         FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
 374 }
 375
 376
 377 /* For each entry inserted into the software descriptor ring, create a
 378  * descriptor in the hardware TX descriptor ring (in host memory), and
 379  * write a doorbell.
 380  */
 381 void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
 382 {
 383
 384         struct efx_tx_buffer *buffer;
 385         efx_qword_t *txd;
 386         unsigned write_ptr;
 387
 388         BUG_ON(tx_queue->write_count == tx_queue->insert_count);
 389
 390         do {
 391                 write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
 392                 buffer = &tx_queue->buffer[write_ptr];
 393                 txd = efx_tx_desc(tx_queue, write_ptr);
 394                 ++tx_queue->write_count;
 395
 396                 /* Create TX descriptor ring entry */
 397                 EFX_POPULATE_QWORD_4(*txd,
 398                                      FSF_AZ_TX_KER_CONT, buffer->continuation,
 399                                      FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
 400                                      FSF_AZ_TX_KER_BUF_REGION, 0,
 401                                      FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
 402         } while (tx_queue->write_count != tx_queue->insert_count);
 403
 404         wmb(); /* Ensure descriptors are written before they are fetched */
 405         efx_notify_tx_desc(tx_queue);
 406 }
 407
 408 /* Allocate hardware resources for a TX queue */
 409 int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
 410 {
 411         struct efx_nic *efx = tx_queue->efx;
 412         BUILD_BUG_ON(EFX_TXQ_SIZE < 512 || EFX_TXQ_SIZE > 4096 ||
 413                      EFX_TXQ_SIZE & EFX_TXQ_MASK);
 414         return efx_alloc_special_buffer(efx, &tx_queue->txd,
 415                                         EFX_TXQ_SIZE * sizeof(efx_qword_t));
 416 }
 417
 418 void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 419 {
 420         efx_oword_t tx_desc_ptr;
 421         struct efx_nic *efx = tx_queue->efx;
 422
 423         tx_queue->flushed = FLUSH_NONE;
 424
 425         /* Pin TX descriptor ring */
 426         efx_init_special_buffer(efx, &tx_queue->txd);
 427
 428         /* Push TX descriptor ring to card */
 429         EFX_POPULATE_OWORD_10(tx_desc_ptr,
 430                               FRF_AZ_TX_DESCQ_EN, 1,
 431                               FRF_AZ_TX_ISCSI_DDIG_EN, 0,
 432                               FRF_AZ_TX_ISCSI_HDIG_EN, 0,
 433                               FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
 434                               FRF_AZ_TX_DESCQ_EVQ_ID,
 435                               tx_queue->channel->channel,
 436                               FRF_AZ_TX_DESCQ_OWNER_ID, 0,
 437                               FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
 438                               FRF_AZ_TX_DESCQ_SIZE,
 439                               __ffs(tx_queue->txd.entries),
 440                               FRF_AZ_TX_DESCQ_TYPE, 0,
 441                               FRF_BZ_TX_NON_IP_DROP_DIS, 1);
 442
 443         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 444                 int csum = tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM;
 445                 EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
 446                 EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_TCP_CHKSM_DIS,
 447                                     !csum);
 448         }
 449
 450         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 451                          tx_queue->queue);
 452
 453         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
 454                 efx_oword_t reg;
 455
 456                 /* Only 128 bits in this register */
 457                 BUILD_BUG_ON(EFX_TX_QUEUE_COUNT >= 128);
 458
 459                 efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 460                 if (tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM)
 461                         clear_bit_le(tx_queue->queue, (void *)&reg);
 462                 else
 463                         set_bit_le(tx_queue->queue, (void *)&reg);
 464                 efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 465         }
 466 }
 467
 468 static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
 469 {
 470         struct efx_nic *efx = tx_queue->efx;
 471         efx_oword_t tx_flush_descq;
 472
 473         tx_queue->flushed = FLUSH_PENDING;
 474
 475         /* Post a flush command */
 476         EFX_POPULATE_OWORD_2(tx_flush_descq,
 477                              FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 478                              FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
 479         efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
 480 }
 481
 482 void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
 483 {
 484         struct efx_nic *efx = tx_queue->efx;
 485         efx_oword_t tx_desc_ptr;
 486
 487         /* The queue should have been flushed */
 488         WARN_ON(tx_queue->flushed != FLUSH_DONE);
 489
 490         /* Remove TX descriptor ring from card */
 491         EFX_ZERO_OWORD(tx_desc_ptr);
 492         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 493                          tx_queue->queue);
 494
 495         /* Unpin TX descriptor ring */
 496         efx_fini_special_buffer(efx, &tx_queue->txd);
 497 }
 498
 499 /* Free buffers backing TX queue */
 500 void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
 501 {
 502         efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
 503 }
 504
 505 /**************************************************************************
 506  *
 507  * RX path
 508  *
 509  **************************************************************************/
 510
 511 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
 512 static inline efx_qword_t *
 513 efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
 514 {
 515         return (((efx_qword_t *) (rx_queue->rxd.addr)) + index);
 516 }
 517
 518 /* This creates an entry in the RX descriptor queue */
 519 static inline void
 520 efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
 521 {
 522         struct efx_rx_buffer *rx_buf;
 523         efx_qword_t *rxd;
 524
 525         rxd = efx_rx_desc(rx_queue, index);
 526         rx_buf = efx_rx_buffer(rx_queue, index);
 527         EFX_POPULATE_QWORD_3(*rxd,
 528                              FSF_AZ_RX_KER_BUF_SIZE,
 529                              rx_buf->len -
 530                              rx_queue->efx->type->rx_buffer_padding,
 531                              FSF_AZ_RX_KER_BUF_REGION, 0,
 532                              FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
 533 }
 534
 535 /* This writes to the RX_DESC_WPTR register for the specified receive
 536  * descriptor ring.
 537  */
 538 void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
 539 {
 540         efx_dword_t reg;
 541         unsigned write_ptr;
 542
 543         while (rx_queue->notified_count != rx_queue->added_count) {
 544                 efx_build_rx_desc(rx_queue,
 545                                   rx_queue->notified_count &
 546                                   EFX_RXQ_MASK);
 547                 ++rx_queue->notified_count;
 548         }
 549
 550         wmb();
 551         write_ptr = rx_queue->added_count & EFX_RXQ_MASK;
 552         EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
 553         efx_writed_page(rx_queue->efx, &reg,
 554                         FR_AZ_RX_DESC_UPD_DWORD_P0, rx_queue->queue);
 555 }
 556
 557 int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
 558 {
 559         struct efx_nic *efx = rx_queue->efx;
 560         BUILD_BUG_ON(EFX_RXQ_SIZE < 512 || EFX_RXQ_SIZE > 4096 ||
 561                      EFX_RXQ_SIZE & EFX_RXQ_MASK);
 562         return efx_alloc_special_buffer(efx, &rx_queue->rxd,
 563                                         EFX_RXQ_SIZE * sizeof(efx_qword_t));
 564 }
 565
 566 void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
 567 {
 568         efx_oword_t rx_desc_ptr;
 569         struct efx_nic *efx = rx_queue->efx;
 570         bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
 571         bool iscsi_digest_en = is_b0;
 572
 573         EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n",
 574                 rx_queue->queue, rx_queue->rxd.index,
 575                 rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 576
 577         rx_queue->flushed = FLUSH_NONE;
 578
 579         /* Pin RX descriptor ring */
 580         efx_init_special_buffer(efx, &rx_queue->rxd);
 581
 582         /* Push RX descriptor ring to card */
 583         EFX_POPULATE_OWORD_10(rx_desc_ptr,
 584                               FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
 585                               FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
 586                               FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
 587                               FRF_AZ_RX_DESCQ_EVQ_ID,
 588                               rx_queue->channel->channel,
 589                               FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 590                               FRF_AZ_RX_DESCQ_LABEL, rx_queue->queue,
 591                               FRF_AZ_RX_DESCQ_SIZE,
 592                               __ffs(rx_queue->rxd.entries),
 593                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
 594                               /* For >=B0 this is scatter so disable */
 595                               FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
 596                               FRF_AZ_RX_DESCQ_EN, 1);
 597         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 598                          rx_queue->queue);
 599 }
 600
 601 static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
 602 {
 603         struct efx_nic *efx = rx_queue->efx;
 604         efx_oword_t rx_flush_descq;
 605
 606         rx_queue->flushed = FLUSH_PENDING;
 607
 608         /* Post a flush command */
 609         EFX_POPULATE_OWORD_2(rx_flush_descq,
 610                              FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 611                              FRF_AZ_RX_FLUSH_DESCQ, rx_queue->queue);
 612         efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
 613 }
 614
 615 void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
 616 {
 617         efx_oword_t rx_desc_ptr;
 618         struct efx_nic *efx = rx_queue->efx;
 619
 620         /* The queue should already have been flushed */
 621         WARN_ON(rx_queue->flushed != FLUSH_DONE);
 622
 623         /* Remove RX descriptor ring from card */
 624         EFX_ZERO_OWORD(rx_desc_ptr);
 625         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 626                          rx_queue->queue);
 627
 628         /* Unpin RX descriptor ring */
 629         efx_fini_special_buffer(efx, &rx_queue->rxd);
 630 }
 631
 632 /* Free buffers backing RX queue */
 633 void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
 634 {
 635         efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
 636 }
 637
 638 /**************************************************************************
 639  *
 640  * Event queue processing
 641  * Event queues are processed by per-channel tasklets.
 642  *
 643  **************************************************************************/
 644
 645 /* Update a channel's event queue's read pointer (RPTR) register
 646  *
 647  * This writes the EVQ_RPTR_REG register for the specified channel's
 648  * event queue.
 649  *
 650  * Note that EVQ_RPTR_REG contains the index of the "last read" event,
 651  * whereas channel->eventq_read_ptr contains the index of the "next to
 652  * read" event.
 653  */
 654 void efx_nic_eventq_read_ack(struct efx_channel *channel)
 655 {
 656         efx_dword_t reg;
 657         struct efx_nic *efx = channel->efx;
 658
 659         EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr);
 660         efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
 661                             channel->channel);
 662 }
 663
 664 /* Use HW to insert a SW defined event */
 665 void efx_generate_event(struct efx_channel *channel, efx_qword_t *event)
 666 {
 667         efx_oword_t drv_ev_reg;
 668
 669         BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
 670                      FRF_AZ_DRV_EV_DATA_WIDTH != 64);
 671         drv_ev_reg.u32[0] = event->u32[0];
 672         drv_ev_reg.u32[1] = event->u32[1];
 673         drv_ev_reg.u32[2] = 0;
 674         drv_ev_reg.u32[3] = 0;
 675         EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel);
 676         efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV);
 677 }
 678
 679 /* Handle a transmit completion event
 680  *
 681  * The NIC batches TX completion events; the message we receive is of
 682  * the form "complete all TX events up to this index".
 683  */
 684 static void
 685 efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 686 {
 687         unsigned int tx_ev_desc_ptr;
 688         unsigned int tx_ev_q_label;
 689         struct efx_tx_queue *tx_queue;
 690         struct efx_nic *efx = channel->efx;
 691
 692         if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 693                 /* Transmit completion */
 694                 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
 695                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 696                 tx_queue = &efx->tx_queue[tx_ev_q_label];
 697                 channel->irq_mod_score +=
 698                         (tx_ev_desc_ptr - tx_queue->read_count) &
 699                         EFX_TXQ_MASK;
 700                 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 701         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 702                 /* Rewrite the FIFO write pointer */
 703                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 704                 tx_queue = &efx->tx_queue[tx_ev_q_label];
 705
 706                 if (efx_dev_registered(efx))
 707                         netif_tx_lock(efx->net_dev);
 708                 efx_notify_tx_desc(tx_queue);
 709                 if (efx_dev_registered(efx))
 710                         netif_tx_unlock(efx->net_dev);
 711         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
 712                    EFX_WORKAROUND_10727(efx)) {
 713                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 714         } else {
 715                 EFX_ERR(efx, "channel %d unexpected TX event "
 716                         EFX_QWORD_FMT"\n", channel->channel,
 717                         EFX_QWORD_VAL(*event));
 718         }
 719 }
 720
 721 /* Detect errors included in the rx_evt_pkt_ok bit. */
 722 static void efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
 723                                  const efx_qword_t *event,
 724                                  bool *rx_ev_pkt_ok,
 725                                  bool *discard)
 726 {
 727         struct efx_nic *efx = rx_queue->efx;
 728         bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
 729         bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
 730         bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
 731         bool rx_ev_other_err, rx_ev_pause_frm;
 732         bool rx_ev_hdr_type, rx_ev_mcast_pkt;
 733         unsigned rx_ev_pkt_type;
 734
 735         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 736         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 737         rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
 738         rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
 739         rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
 740                                                  FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
 741         rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
 742                                                   FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
 743         rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
 744                                                    FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
 745         rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
 746         rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
 747         rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
 748                           0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
 749         rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
 750
 751         /* Every error apart from tobe_disc and pause_frm */
 752         rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
 753                            rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
 754                            rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
 755
 756         /* Count errors that are not in MAC stats.  Ignore expected
 757          * checksum errors during self-test. */
 758         if (rx_ev_frm_trunc)
 759                 ++rx_queue->channel->n_rx_frm_trunc;
 760         else if (rx_ev_tobe_disc)
 761                 ++rx_queue->channel->n_rx_tobe_disc;
 762         else if (!efx->loopback_selftest) {
 763                 if (rx_ev_ip_hdr_chksum_err)
 764                         ++rx_queue->channel->n_rx_ip_hdr_chksum_err;
 765                 else if (rx_ev_tcp_udp_chksum_err)
 766                         ++rx_queue->channel->n_rx_tcp_udp_chksum_err;
 767         }
 768
 769         /* The frame must be discarded if any of these are true. */
 770         *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
 771                     rx_ev_tobe_disc | rx_ev_pause_frm);
 772
 773         /* TOBE_DISC is expected on unicast mismatches; don't print out an
 774          * error message.  FRM_TRUNC indicates RXDP dropped the packet due
 775          * to a FIFO overflow.
 776          */
 777 #ifdef EFX_ENABLE_DEBUG
 778         if (rx_ev_other_err) {
 779                 EFX_INFO_RL(efx, " RX queue %d unexpected RX event "
 780                             EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
 781                             rx_queue->queue, EFX_QWORD_VAL(*event),
 782                             rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
 783                             rx_ev_ip_hdr_chksum_err ?
 784                             " [IP_HDR_CHKSUM_ERR]" : "",
 785                             rx_ev_tcp_udp_chksum_err ?
 786                             " [TCP_UDP_CHKSUM_ERR]" : "",
 787                             rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
 788                             rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
 789                             rx_ev_drib_nib ? " [DRIB_NIB]" : "",
 790                             rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
 791                             rx_ev_pause_frm ? " [PAUSE]" : "");
 792         }
 793 #endif
 794 }
 795
 796 /* Handle receive events that are not in-order. */
 797 static void
 798 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 799 {
 800         struct efx_nic *efx = rx_queue->efx;
 801         unsigned expected, dropped;
 802
 803         expected = rx_queue->removed_count & EFX_RXQ_MASK;
 804         dropped = (index - expected) & EFX_RXQ_MASK;
 805         EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n",
 806                 dropped, index, expected);
 807
 808         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
 809                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 810 }
 811
 812 /* Handle a packet received event
 813  *
 814  * The NIC gives a "discard" flag if it's a unicast packet with the
 815  * wrong destination address
 816  * Also "is multicast" and "matches multicast filter" flags can be used to
 817  * discard non-matching multicast packets.
 818  */
 819 static void
 820 efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
 821 {
 822         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
 823         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
 824         unsigned expected_ptr;
 825         bool rx_ev_pkt_ok, discard = false, checksummed;
 826         struct efx_rx_queue *rx_queue;
 827         struct efx_nic *efx = channel->efx;
 828
 829         /* Basic packet information */
 830         rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
 831         rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
 832         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 833         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
 834         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
 835         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
 836                 channel->channel);
 837
 838         rx_queue = &efx->rx_queue[channel->channel];
 839
 840         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
 841         expected_ptr = rx_queue->removed_count & EFX_RXQ_MASK;
 842         if (unlikely(rx_ev_desc_ptr != expected_ptr))
 843                 efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
 844
 845         if (likely(rx_ev_pkt_ok)) {
 846                 /* If packet is marked as OK and packet type is TCP/IP or
 847                  * UDP/IP, then we can rely on the hardware checksum.
 848                  */
 849                 checksummed =
 850                         likely(efx->rx_checksum_enabled) &&
 851                         (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
 852                          rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP);
 853         } else {
 854                 efx_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, &discard);
 855                 checksummed = false;
 856         }
 857
 858         /* Detect multicast packets that didn't match the filter */
 859         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 860         if (rx_ev_mcast_pkt) {
 861                 unsigned int rx_ev_mcast_hash_match =
 862                         EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
 863
 864                 if (unlikely(!rx_ev_mcast_hash_match)) {
 865                         ++channel->n_rx_mcast_mismatch;
 866                         discard = true;
 867                 }
 868         }
 869
 870         channel->irq_mod_score += 2;
 871
 872         /* Handle received packet */
 873         efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
 874                       checksummed, discard);
 875 }
 876
 877 /* Global events are basically PHY events */
 878 static void
 879 efx_handle_global_event(struct efx_channel *channel, efx_qword_t *event)
 880 {
 881         struct efx_nic *efx = channel->efx;
 882         bool handled = false;
 883
 884         if (EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_G_PHY0_INTR) ||
 885             EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XG_PHY0_INTR) ||
 886             EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XFP_PHY0_INTR)) {
 887                 /* Ignored */
 888                 handled = true;
 889         }
 890
 891         if ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) &&
 892             EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_XG_MGT_INTR)) {
 893                 efx->xmac_poll_required = true;
 894                 handled = true;
 895         }
 896
 897         if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1 ?
 898             EFX_QWORD_FIELD(*event, FSF_AA_GLB_EV_RX_RECOVERY) :
 899             EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_RX_RECOVERY)) {
 900                 EFX_ERR(efx, "channel %d seen global RX_RESET "
 901                         "event. Resetting.\n", channel->channel);
 902
 903                 atomic_inc(&efx->rx_reset);
 904                 efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ?
 905                                    RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 906                 handled = true;
 907         }
 908
 909         if (!handled)
 910                 EFX_ERR(efx, "channel %d unknown global event "
 911                         EFX_QWORD_FMT "\n", channel->channel,
 912                         EFX_QWORD_VAL(*event));
 913 }
 914
 915 static void
 916 efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
 917 {
 918         struct efx_nic *efx = channel->efx;
 919         unsigned int ev_sub_code;
 920         unsigned int ev_sub_data;
 921
 922         ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
 923         ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
 924
 925         switch (ev_sub_code) {
 926         case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
 927                 EFX_TRACE(efx, "channel %d TXQ %d flushed\n",
 928                           channel->channel, ev_sub_data);
 929                 break;
 930         case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
 931                 EFX_TRACE(efx, "channel %d RXQ %d flushed\n",
 932                           channel->channel, ev_sub_data);
 933                 break;
 934         case FSE_AZ_EVQ_INIT_DONE_EV:
 935                 EFX_LOG(efx, "channel %d EVQ %d initialised\n",
 936                         channel->channel, ev_sub_data);
 937                 break;
 938         case FSE_AZ_SRM_UPD_DONE_EV:
 939                 EFX_TRACE(efx, "channel %d SRAM update done\n",
 940                           channel->channel);
 941                 break;
 942         case FSE_AZ_WAKE_UP_EV:
 943                 EFX_TRACE(efx, "channel %d RXQ %d wakeup event\n",
 944                           channel->channel, ev_sub_data);
 945                 break;
 946         case FSE_AZ_TIMER_EV:
 947                 EFX_TRACE(efx, "channel %d RX queue %d timer expired\n",
 948                           channel->channel, ev_sub_data);
 949                 break;
 950         case FSE_AA_RX_RECOVER_EV:
 951                 EFX_ERR(efx, "channel %d seen DRIVER RX_RESET event. "
 952                         "Resetting.\n", channel->channel);
 953                 atomic_inc(&efx->rx_reset);
 954                 efx_schedule_reset(efx,
 955                                    EFX_WORKAROUND_6555(efx) ?
 956                                    RESET_TYPE_RX_RECOVERY :
 957                                    RESET_TYPE_DISABLE);
 958                 break;
 959         case FSE_BZ_RX_DSC_ERROR_EV:
 960                 EFX_ERR(efx, "RX DMA Q %d reports descriptor fetch error."
 961                         " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 962                 efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
 963                 break;
 964         case FSE_BZ_TX_DSC_ERROR_EV:
 965                 EFX_ERR(efx, "TX DMA Q %d reports descriptor fetch error."
 966                         " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 967                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 968                 break;
 969         default:
 970                 EFX_TRACE(efx, "channel %d unknown driver event code %d "
 971                           "data %04x\n", channel->channel, ev_sub_code,
 972                           ev_sub_data);
 973                 break;
 974         }
 975 }
 976
 977 int efx_nic_process_eventq(struct efx_channel *channel, int rx_quota)
 978 {
 979         unsigned int read_ptr;
 980         efx_qword_t event, *p_event;
 981         int ev_code;
 982         int rx_packets = 0;
 983
 984         read_ptr = channel->eventq_read_ptr;
 985
 986         do {
 987                 p_event = efx_event(channel, read_ptr);
 988                 event = *p_event;
 989
 990                 if (!efx_event_present(&event))
 991                         /* End of events */
 992                         break;
 993
 994                 EFX_TRACE(channel->efx, "channel %d event is "EFX_QWORD_FMT"\n",
 995                           channel->channel, EFX_QWORD_VAL(event));
 996
 997                 /* Clear this event by marking it all ones */
 998                 EFX_SET_QWORD(*p_event);
 999
1000                 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1001
1002                 switch (ev_code) {
1003                 case FSE_AZ_EV_CODE_RX_EV:
1004                         efx_handle_rx_event(channel, &event);
1005                         ++rx_packets;
1006                         break;
1007                 case FSE_AZ_EV_CODE_TX_EV:
1008                         efx_handle_tx_event(channel, &event);
1009                         break;
1010                 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1011                         channel->eventq_magic = EFX_QWORD_FIELD(
1012                                 event, FSF_AZ_DRV_GEN_EV_MAGIC);
1013                         EFX_LOG(channel->efx, "channel %d received generated "
1014                                 "event "EFX_QWORD_FMT"\n", channel->channel,
1015                                 EFX_QWORD_VAL(event));
1016                         break;
1017                 case FSE_AZ_EV_CODE_GLOBAL_EV:
1018                         efx_handle_global_event(channel, &event);
1019                         break;
1020                 case FSE_AZ_EV_CODE_DRIVER_EV:
1021                         efx_handle_driver_event(channel, &event);
1022                         break;
1023                 default:
1024                         EFX_ERR(channel->efx, "channel %d unknown event type %d"
1025                                 " (data " EFX_QWORD_FMT ")\n", channel->channel,
1026                                 ev_code, EFX_QWORD_VAL(event));
1027                 }
1028
1029                 /* Increment read pointer */
1030                 read_ptr = (read_ptr + 1) & EFX_EVQ_MASK;
1031
1032         } while (rx_packets < rx_quota);
1033
1034         channel->eventq_read_ptr = read_ptr;
1035         return rx_packets;
1036 }
1037
1038 static void falcon_push_irq_moderation(struct efx_channel *channel)
1039 {
1040         efx_dword_t timer_cmd;
1041         struct efx_nic *efx = channel->efx;
1042
1043         /* Set timer register */
1044         if (channel->irq_moderation) {
1045                 EFX_POPULATE_DWORD_2(timer_cmd,
1046                                      FRF_AB_TC_TIMER_MODE,
1047                                      FFE_BB_TIMER_MODE_INT_HLDOFF,
1048                                      FRF_AB_TC_TIMER_VAL,
1049                                      channel->irq_moderation - 1);
1050         } else {
1051                 EFX_POPULATE_DWORD_2(timer_cmd,
1052                                      FRF_AB_TC_TIMER_MODE,
1053                                      FFE_BB_TIMER_MODE_DIS,
1054                                      FRF_AB_TC_TIMER_VAL, 0);
1055         }
1056         BUILD_BUG_ON(FR_AA_TIMER_COMMAND_KER != FR_BZ_TIMER_COMMAND_P0);
1057         efx_writed_page_locked(efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0,
1058                                channel->channel);
1059
1060 }
1061
1062 /* Allocate buffer table entries for event queue */
1063 int efx_nic_probe_eventq(struct efx_channel *channel)
1064 {
1065         struct efx_nic *efx = channel->efx;
1066         BUILD_BUG_ON(EFX_EVQ_SIZE < 512 || EFX_EVQ_SIZE > 32768 ||
1067                      EFX_EVQ_SIZE & EFX_EVQ_MASK);
1068         return efx_alloc_special_buffer(efx, &channel->eventq,
1069                                         EFX_EVQ_SIZE * sizeof(efx_qword_t));
1070 }
1071
1072 void efx_nic_init_eventq(struct efx_channel *channel)
1073 {
1074         efx_oword_t evq_ptr;
1075         struct efx_nic *efx = channel->efx;
1076
1077         EFX_LOG(efx, "channel %d event queue in special buffers %d-%d\n",
1078                 channel->channel, channel->eventq.index,
1079                 channel->eventq.index + channel->eventq.entries - 1);
1080
1081         /* Pin event queue buffer */
1082         efx_init_special_buffer(efx, &channel->eventq);
1083
1084         /* Fill event queue with all ones (i.e. empty events) */
1085         memset(channel->eventq.addr, 0xff, channel->eventq.len);
1086
1087         /* Push event queue to card */
1088         EFX_POPULATE_OWORD_3(evq_ptr,
1089                              FRF_AZ_EVQ_EN, 1,
1090                              FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1091                              FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1092         efx_writeo_table(efx, &evq_ptr, efx->type->evq_ptr_tbl_base,
1093                          channel->channel);
1094
1095         efx->type->push_irq_moderation(channel);
1096 }
1097
1098 void efx_nic_fini_eventq(struct efx_channel *channel)
1099 {
1100         efx_oword_t eventq_ptr;
1101         struct efx_nic *efx = channel->efx;
1102
1103         /* Remove event queue from card */
1104         EFX_ZERO_OWORD(eventq_ptr);
1105         efx_writeo_table(efx, &eventq_ptr, efx->type->evq_ptr_tbl_base,
1106                          channel->channel);
1107
1108         /* Unpin event queue */
1109         efx_fini_special_buffer(efx, &channel->eventq);
1110 }
1111
1112 /* Free buffers backing event queue */
1113 void efx_nic_remove_eventq(struct efx_channel *channel)
1114 {
1115         efx_free_special_buffer(channel->efx, &channel->eventq);
1116 }
1117
1118
1119 /* Generates a test event on the event queue.  A subsequent call to
1120  * process_eventq() should pick up the event and place the value of
1121  * "magic" into channel->eventq_magic;
1122  */
1123 void efx_nic_generate_test_event(struct efx_channel *channel, unsigned int magic)
1124 {
1125         efx_qword_t test_event;
1126
1127         EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
1128                              FSE_AZ_EV_CODE_DRV_GEN_EV,
1129                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
1130         efx_generate_event(channel, &test_event);
1131 }
1132
1133 /**************************************************************************
1134  *
1135  * Flush handling
1136  *
1137  **************************************************************************/
1138
1139
1140 static void efx_poll_flush_events(struct efx_nic *efx)
1141 {
1142         struct efx_channel *channel = &efx->channel[0];
1143         struct efx_tx_queue *tx_queue;
1144         struct efx_rx_queue *rx_queue;
1145         unsigned int read_ptr = channel->eventq_read_ptr;
1146         unsigned int end_ptr = (read_ptr - 1) & EFX_EVQ_MASK;
1147
1148         do {
1149                 efx_qword_t *event = efx_event(channel, read_ptr);
1150                 int ev_code, ev_sub_code, ev_queue;
1151                 bool ev_failed;
1152
1153                 if (!efx_event_present(event))
1154                         break;
1155
1156                 ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE);
1157                 ev_sub_code = EFX_QWORD_FIELD(*event,
1158                                               FSF_AZ_DRIVER_EV_SUBCODE);
1159                 if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1160                     ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) {
1161                         ev_queue = EFX_QWORD_FIELD(*event,
1162                                                    FSF_AZ_DRIVER_EV_SUBDATA);
1163                         if (ev_queue < EFX_TX_QUEUE_COUNT) {
1164                                 tx_queue = efx->tx_queue + ev_queue;
1165                                 tx_queue->flushed = FLUSH_DONE;
1166                         }
1167                 } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1168                            ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) {
1169                         ev_queue = EFX_QWORD_FIELD(
1170                                 *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1171                         ev_failed = EFX_QWORD_FIELD(
1172                                 *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1173                         if (ev_queue < efx->n_rx_queues) {
1174                                 rx_queue = efx->rx_queue + ev_queue;
1175                                 rx_queue->flushed =
1176                                         ev_failed ? FLUSH_FAILED : FLUSH_DONE;
1177                         }
1178                 }
1179
1180                 /* We're about to destroy the queue anyway, so
1181                  * it's ok to throw away every non-flush event */
1182                 EFX_SET_QWORD(*event);
1183
1184                 read_ptr = (read_ptr + 1) & EFX_EVQ_MASK;
1185         } while (read_ptr != end_ptr);
1186
1187         channel->eventq_read_ptr = read_ptr;
1188 }
1189
1190 static void falcon_deconfigure_mac_wrapper(struct efx_nic *efx);
1191
1192 static void falcon_prepare_flush(struct efx_nic *efx)
1193 {
1194         falcon_deconfigure_mac_wrapper(efx);
1195
1196         /* Wait for the tx and rx fifo's to get to the next packet boundary
1197          * (~1ms without back-pressure), then to drain the remainder of the
1198          * fifo's at data path speeds (negligible), with a healthy margin. */
1199         msleep(10);
1200 }
1201
1202 /* Handle tx and rx flushes at the same time, since they run in
1203  * parallel in the hardware and there's no reason for us to
1204  * serialise them */
1205 int efx_nic_flush_queues(struct efx_nic *efx)
1206 {
1207         struct efx_rx_queue *rx_queue;
1208         struct efx_tx_queue *tx_queue;
1209         int i, tx_pending, rx_pending;
1210
1211         /* If necessary prepare the hardware for flushing */
1212         efx->type->prepare_flush(efx);
1213
1214         /* Flush all tx queues in parallel */
1215         efx_for_each_tx_queue(tx_queue, efx)
1216                 efx_flush_tx_queue(tx_queue);
1217
1218         /* The hardware supports four concurrent rx flushes, each of which may
1219          * need to be retried if there is an outstanding descriptor fetch */
1220         for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) {
1221                 rx_pending = tx_pending = 0;
1222                 efx_for_each_rx_queue(rx_queue, efx) {
1223                         if (rx_queue->flushed == FLUSH_PENDING)
1224                                 ++rx_pending;
1225                 }
1226                 efx_for_each_rx_queue(rx_queue, efx) {
1227                         if (rx_pending == EFX_RX_FLUSH_COUNT)
1228                                 break;
1229                         if (rx_queue->flushed == FLUSH_FAILED ||
1230                             rx_queue->flushed == FLUSH_NONE) {
1231                                 efx_flush_rx_queue(rx_queue);
1232                                 ++rx_pending;
1233                         }
1234                 }
1235                 efx_for_each_tx_queue(tx_queue, efx) {
1236                         if (tx_queue->flushed != FLUSH_DONE)
1237                                 ++tx_pending;
1238                 }
1239
1240                 if (rx_pending == 0 && tx_pending == 0)
1241                         return 0;
1242
1243                 msleep(EFX_FLUSH_INTERVAL);
1244                 efx_poll_flush_events(efx);
1245         }
1246
1247         /* Mark the queues as all flushed. We're going to return failure
1248          * leading to a reset, or fake up success anyway */
1249         efx_for_each_tx_queue(tx_queue, efx) {
1250                 if (tx_queue->flushed != FLUSH_DONE)
1251                         EFX_ERR(efx, "tx queue %d flush command timed out\n",
1252                                 tx_queue->queue);
1253                 tx_queue->flushed = FLUSH_DONE;
1254         }
1255         efx_for_each_rx_queue(rx_queue, efx) {
1256                 if (rx_queue->flushed != FLUSH_DONE)
1257                         EFX_ERR(efx, "rx queue %d flush command timed out\n",
1258                                 rx_queue->queue);
1259                 rx_queue->flushed = FLUSH_DONE;
1260         }
1261
1262         if (EFX_WORKAROUND_7803(efx))
1263                 return 0;
1264
1265         return -ETIMEDOUT;
1266 }
1267
1268 /**************************************************************************
1269  *
1270  * Hardware interrupts
1271  * The hardware interrupt handler does very little work; all the event
1272  * queue processing is carried out by per-channel tasklets.
1273  *
1274  **************************************************************************/
1275
1276 /* Enable/disable/generate interrupts */
1277 static inline void efx_nic_interrupts(struct efx_nic *efx,
1278                                       bool enabled, bool force)
1279 {
1280         efx_oword_t int_en_reg_ker;
1281
1282         EFX_POPULATE_OWORD_2(int_en_reg_ker,
1283                              FRF_AZ_KER_INT_KER, force,
1284                              FRF_AZ_DRV_INT_EN_KER, enabled);
1285         efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1286 }
1287
1288 void efx_nic_enable_interrupts(struct efx_nic *efx)
1289 {
1290         struct efx_channel *channel;
1291
1292         EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1293         wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1294
1295         /* Enable interrupts */
1296         efx_nic_interrupts(efx, true, false);
1297
1298         /* Force processing of all the channels to get the EVQ RPTRs up to
1299            date */
1300         efx_for_each_channel(channel, efx)
1301                 efx_schedule_channel(channel);
1302 }
1303
1304 void efx_nic_disable_interrupts(struct efx_nic *efx)
1305 {
1306         /* Disable interrupts */
1307         efx_nic_interrupts(efx, false, false);
1308 }
1309
1310 /* Generate a test interrupt
1311  * Interrupt must already have been enabled, otherwise nasty things
1312  * may happen.
1313  */
1314 void efx_nic_generate_interrupt(struct efx_nic *efx)
1315 {
1316         efx_nic_interrupts(efx, true, true);
1317 }
1318
1319 /* Acknowledge a legacy interrupt from Falcon
1320  *
1321  * This acknowledges a legacy (not MSI) interrupt via INT_ACK_KER_REG.
1322  *
1323  * Due to SFC bug 3706 (silicon revision <=A1) reads can be duplicated in the
1324  * BIU. Interrupt acknowledge is read sensitive so must write instead
1325  * (then read to ensure the BIU collector is flushed)
1326  *
1327  * NB most hardware supports MSI interrupts
1328  */
1329 inline void falcon_irq_ack_a1(struct efx_nic *efx)
1330 {
1331         efx_dword_t reg;
1332
1333         EFX_POPULATE_DWORD_1(reg, FRF_AA_INT_ACK_KER_FIELD, 0xb7eb7e);
1334         efx_writed(efx, &reg, FR_AA_INT_ACK_KER);
1335         efx_readd(efx, &reg, FR_AA_WORK_AROUND_BROKEN_PCI_READS);
1336 }
1337
1338 /* Process a fatal interrupt
1339  * Disable bus mastering ASAP and schedule a reset
1340  */
1341 irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
1342 {
1343         struct falcon_nic_data *nic_data = efx->nic_data;
1344         efx_oword_t *int_ker = efx->irq_status.addr;
1345         efx_oword_t fatal_intr;
1346         int error, mem_perr;
1347
1348         efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1349         error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1350
1351         EFX_ERR(efx, "SYSTEM ERROR " EFX_OWORD_FMT " status "
1352                 EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1353                 EFX_OWORD_VAL(fatal_intr),
1354                 error ? "disabling bus mastering" : "no recognised error");
1355         if (error == 0)
1356                 goto out;
1357
1358         /* If this is a memory parity error dump which blocks are offending */
1359         mem_perr = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER);
1360         if (mem_perr) {
1361                 efx_oword_t reg;
1362                 efx_reado(efx, &reg, FR_AZ_MEM_STAT);
1363                 EFX_ERR(efx, "SYSTEM ERROR: memory parity error "
1364                         EFX_OWORD_FMT "\n", EFX_OWORD_VAL(reg));
1365         }
1366
1367         /* Disable both devices */
1368         pci_clear_master(efx->pci_dev);
1369         if (efx_nic_is_dual_func(efx))
1370                 pci_clear_master(nic_data->pci_dev2);
1371         efx_nic_disable_interrupts(efx);
1372
1373         /* Count errors and reset or disable the NIC accordingly */
1374         if (efx->int_error_count == 0 ||
1375             time_after(jiffies, efx->int_error_expire)) {
1376                 efx->int_error_count = 0;
1377                 efx->int_error_expire =
1378                         jiffies + EFX_INT_ERROR_EXPIRE * HZ;
1379         }
1380         if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
1381                 EFX_ERR(efx, "SYSTEM ERROR - reset scheduled\n");
1382                 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1383         } else {
1384                 EFX_ERR(efx, "SYSTEM ERROR - max number of errors seen."
1385                         "NIC will be disabled\n");
1386                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1387         }
1388 out:
1389         return IRQ_HANDLED;
1390 }
1391
1392 /* Handle a legacy interrupt
1393  * Acknowledges the interrupt and schedule event queue processing.
1394  */
1395 static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
1396 {
1397         struct efx_nic *efx = dev_id;
1398         efx_oword_t *int_ker = efx->irq_status.addr;
1399         irqreturn_t result = IRQ_NONE;
1400         struct efx_channel *channel;
1401         efx_dword_t reg;
1402         u32 queues;
1403         int syserr;
1404
1405         /* Read the ISR which also ACKs the interrupts */
1406         efx_readd(efx, &reg, FR_BZ_INT_ISR0);
1407         queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1408
1409         /* Check to see if we have a serious error condition */
1410         syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1411         if (unlikely(syserr))
1412                 return efx_nic_fatal_interrupt(efx);
1413
1414         /* Schedule processing of any interrupting queues */
1415         efx_for_each_channel(channel, efx) {
1416                 if ((queues & 1) ||
1417                     efx_event_present(
1418                             efx_event(channel, channel->eventq_read_ptr))) {
1419                         efx_schedule_channel(channel);
1420                         result = IRQ_HANDLED;
1421                 }
1422                 queues >>= 1;
1423         }
1424
1425         if (result == IRQ_HANDLED) {
1426                 efx->last_irq_cpu = raw_smp_processor_id();
1427                 EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1428                           irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1429         }
1430
1431         return result;
1432 }
1433
1434
1435 irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id)
1436 {
1437         struct efx_nic *efx = dev_id;
1438         efx_oword_t *int_ker = efx->irq_status.addr;
1439         struct efx_channel *channel;
1440         int syserr;
1441         int queues;
1442
1443         /* Check to see if this is our interrupt.  If it isn't, we
1444          * exit without having touched the hardware.
1445          */
1446         if (unlikely(EFX_OWORD_IS_ZERO(*int_ker))) {
1447                 EFX_TRACE(efx, "IRQ %d on CPU %d not for me\n", irq,
1448                           raw_smp_processor_id());
1449                 return IRQ_NONE;
1450         }
1451         efx->last_irq_cpu = raw_smp_processor_id();
1452         EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1453                   irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1454
1455         /* Check to see if we have a serious error condition */
1456         syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1457         if (unlikely(syserr))
1458                 return efx_nic_fatal_interrupt(efx);
1459
1460         /* Determine interrupting queues, clear interrupt status
1461          * register and acknowledge the device interrupt.
1462          */
1463         BUILD_BUG_ON(FSF_AZ_NET_IVEC_INT_Q_WIDTH > EFX_MAX_CHANNELS);
1464         queues = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_INT_Q);
1465         EFX_ZERO_OWORD(*int_ker);
1466         wmb(); /* Ensure the vector is cleared before interrupt ack */
1467         falcon_irq_ack_a1(efx);
1468
1469         /* Schedule processing of any interrupting queues */
1470         channel = &efx->channel[0];
1471         while (queues) {
1472                 if (queues & 0x01)
1473                         efx_schedule_channel(channel);
1474                 channel++;
1475                 queues >>= 1;
1476         }
1477
1478         return IRQ_HANDLED;
1479 }
1480
1481 /* Handle an MSI interrupt
1482  *
1483  * Handle an MSI hardware interrupt.  This routine schedules event
1484  * queue processing.  No interrupt acknowledgement cycle is necessary.
1485  * Also, we never need to check that the interrupt is for us, since
1486  * MSI interrupts cannot be shared.
1487  */
1488 static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
1489 {
1490         struct efx_channel *channel = dev_id;
1491         struct efx_nic *efx = channel->efx;
1492         efx_oword_t *int_ker = efx->irq_status.addr;
1493         int syserr;
1494
1495         efx->last_irq_cpu = raw_smp_processor_id();
1496         EFX_TRACE(efx, "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1497                   irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1498
1499         /* Check to see if we have a serious error condition */
1500         syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1501         if (unlikely(syserr))
1502                 return efx_nic_fatal_interrupt(efx);
1503
1504         /* Schedule processing of the channel */
1505         efx_schedule_channel(channel);
1506
1507         return IRQ_HANDLED;
1508 }
1509
1510
1511 /* Setup RSS indirection table.
1512  * This maps from the hash value of the packet to RXQ
1513  */
1514 static void efx_setup_rss_indir_table(struct efx_nic *efx)
1515 {
1516         int i = 0;
1517         unsigned long offset;
1518         efx_dword_t dword;
1519
1520         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1521                 return;
1522
1523         for (offset = FR_BZ_RX_INDIRECTION_TBL;
1524              offset < FR_BZ_RX_INDIRECTION_TBL + 0x800;
1525              offset += 0x10) {
1526                 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1527                                      i % efx->n_rx_queues);
1528                 efx_writed(efx, &dword, offset);
1529                 i++;
1530         }
1531 }
1532
1533 /* Hook interrupt handler(s)
1534  * Try MSI and then legacy interrupts.
1535  */
1536 int efx_nic_init_interrupt(struct efx_nic *efx)
1537 {
1538         struct efx_channel *channel;
1539         int rc;
1540
1541         if (!EFX_INT_MODE_USE_MSI(efx)) {
1542                 irq_handler_t handler;
1543                 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1544                         handler = efx_legacy_interrupt;
1545                 else
1546                         handler = falcon_legacy_interrupt_a1;
1547
1548                 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1549                                  efx->name, efx);
1550                 if (rc) {
1551                         EFX_ERR(efx, "failed to hook legacy IRQ %d\n",
1552                                 efx->pci_dev->irq);
1553                         goto fail1;
1554                 }
1555                 return 0;
1556         }
1557
1558         /* Hook MSI or MSI-X interrupt */
1559         efx_for_each_channel(channel, efx) {
1560                 rc = request_irq(channel->irq, efx_msi_interrupt,
1561                                  IRQF_PROBE_SHARED, /* Not shared */
1562                                  channel->name, channel);
1563                 if (rc) {
1564                         EFX_ERR(efx, "failed to hook IRQ %d\n", channel->irq);
1565                         goto fail2;
1566                 }
1567         }
1568
1569         return 0;
1570
1571  fail2:
1572         efx_for_each_channel(channel, efx)
1573                 free_irq(channel->irq, channel);
1574  fail1:
1575         return rc;
1576 }
1577
1578 void efx_nic_fini_interrupt(struct efx_nic *efx)
1579 {
1580         struct efx_channel *channel;
1581         efx_oword_t reg;
1582
1583         /* Disable MSI/MSI-X interrupts */
1584         efx_for_each_channel(channel, efx) {
1585                 if (channel->irq)
1586                         free_irq(channel->irq, channel);
1587         }
1588
1589         /* ACK legacy interrupt */
1590         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1591                 efx_reado(efx, &reg, FR_BZ_INT_ISR0);
1592         else
1593                 falcon_irq_ack_a1(efx);
1594
1595         /* Disable legacy interrupt */
1596         if (efx->legacy_irq)
1597                 free_irq(efx->legacy_irq, efx);
1598 }
1599
1600 /**************************************************************************
1601  *
1602  * EEPROM/flash
1603  *
1604  **************************************************************************
1605  */
1606
1607 #define FALCON_SPI_MAX_LEN sizeof(efx_oword_t)
1608
1609 static int falcon_spi_poll(struct efx_nic *efx)
1610 {
1611         efx_oword_t reg;
1612         efx_reado(efx, &reg, FR_AB_EE_SPI_HCMD);
1613         return EFX_OWORD_FIELD(reg, FRF_AB_EE_SPI_HCMD_CMD_EN) ? -EBUSY : 0;
1614 }
1615
1616 /* Wait for SPI command completion */
1617 static int falcon_spi_wait(struct efx_nic *efx)
1618 {
1619         /* Most commands will finish quickly, so we start polling at
1620          * very short intervals.  Sometimes the command may have to
1621          * wait for VPD or expansion ROM access outside of our
1622          * control, so we allow up to 100 ms. */
1623         unsigned long timeout = jiffies + 1 + DIV_ROUND_UP(HZ, 10);
1624         int i;
1625
1626         for (i = 0; i < 10; i++) {
1627                 if (!falcon_spi_poll(efx))
1628                         return 0;
1629                 udelay(10);
1630         }
1631
1632         for (;;) {
1633                 if (!falcon_spi_poll(efx))
1634                         return 0;
1635                 if (time_after_eq(jiffies, timeout)) {
1636                         EFX_ERR(efx, "timed out waiting for SPI\n");
1637                         return -ETIMEDOUT;
1638                 }
1639                 schedule_timeout_uninterruptible(1);
1640         }
1641 }
1642
1643 int falcon_spi_cmd(const struct efx_spi_device *spi,
1644                    unsigned int command, int address,
1645                    const void *in, void *out, size_t len)
1646 {
1647         struct efx_nic *efx = spi->efx;
1648         bool addressed = (address >= 0);
1649         bool reading = (out != NULL);
1650         efx_oword_t reg;
1651         int rc;
1652
1653         /* Input validation */
1654         if (len > FALCON_SPI_MAX_LEN)
1655                 return -EINVAL;
1656         BUG_ON(!mutex_is_locked(&efx->spi_lock));
1657
1658         /* Check that previous command is not still running */
1659         rc = falcon_spi_poll(efx);
1660         if (rc)
1661                 return rc;
1662
1663         /* Program address register, if we have an address */
1664         if (addressed) {
1665                 EFX_POPULATE_OWORD_1(reg, FRF_AB_EE_SPI_HADR_ADR, address);
1666                 efx_writeo(efx, &reg, FR_AB_EE_SPI_HADR);
1667         }
1668
1669         /* Program data register, if we have data */
1670         if (in != NULL) {
1671                 memcpy(&reg, in, len);
1672                 efx_writeo(efx, &reg, FR_AB_EE_SPI_HDATA);
1673         }
1674
1675         /* Issue read/write command */
1676         EFX_POPULATE_OWORD_7(reg,
1677                              FRF_AB_EE_SPI_HCMD_CMD_EN, 1,
1678                              FRF_AB_EE_SPI_HCMD_SF_SEL, spi->device_id,
1679                              FRF_AB_EE_SPI_HCMD_DABCNT, len,
1680                              FRF_AB_EE_SPI_HCMD_READ, reading,
1681                              FRF_AB_EE_SPI_HCMD_DUBCNT, 0,
1682                              FRF_AB_EE_SPI_HCMD_ADBCNT,
1683                              (addressed ? spi->addr_len : 0),
1684                              FRF_AB_EE_SPI_HCMD_ENC, command);
1685         efx_writeo(efx, &reg, FR_AB_EE_SPI_HCMD);
1686
1687         /* Wait for read/write to complete */
1688         rc = falcon_spi_wait(efx);
1689         if (rc)
1690                 return rc;
1691
1692         /* Read data */
1693         if (out != NULL) {
1694                 efx_reado(efx, &reg, FR_AB_EE_SPI_HDATA);
1695                 memcpy(out, &reg, len);
1696         }
1697
1698         return 0;
1699 }
1700
1701 static size_t
1702 falcon_spi_write_limit(const struct efx_spi_device *spi, size_t start)
1703 {
1704         return min(FALCON_SPI_MAX_LEN,
1705                    (spi->block_size - (start & (spi->block_size - 1))));
1706 }
1707
1708 static inline u8
1709 efx_spi_munge_command(const struct efx_spi_device *spi,
1710                       const u8 command, const unsigned int address)
1711 {
1712         return command | (((address >> 8) & spi->munge_address) << 3);
1713 }
1714
1715 /* Wait up to 10 ms for buffered write completion */
1716 int falcon_spi_wait_write(const struct efx_spi_device *spi)
1717 {
1718         struct efx_nic *efx = spi->efx;
1719         unsigned long timeout = jiffies + 1 + DIV_ROUND_UP(HZ, 100);
1720         u8 status;
1721         int rc;
1722
1723         for (;;) {
1724                 rc = falcon_spi_cmd(spi, SPI_RDSR, -1, NULL,
1725                                     &status, sizeof(status));
1726                 if (rc)
1727                         return rc;
1728                 if (!(status & SPI_STATUS_NRDY))
1729                         return 0;
1730                 if (time_after_eq(jiffies, timeout)) {
1731                         EFX_ERR(efx, "SPI write timeout on device %d"
1732                                 " last status=0x%02x\n",
1733                                 spi->device_id, status);
1734                         return -ETIMEDOUT;
1735                 }
1736                 schedule_timeout_uninterruptible(1);
1737         }
1738 }
1739
1740 int falcon_spi_read(const struct efx_spi_device *spi, loff_t start,
1741                     size_t len, size_t *retlen, u8 *buffer)
1742 {
1743         size_t block_len, pos = 0;
1744         unsigned int command;
1745         int rc = 0;
1746
1747         while (pos < len) {
1748                 block_len = min(len - pos, FALCON_SPI_MAX_LEN);
1749
1750                 command = efx_spi_munge_command(spi, SPI_READ, start + pos);
1751                 rc = falcon_spi_cmd(spi, command, start + pos, NULL,
1752                                     buffer + pos, block_len);
1753                 if (rc)
1754                         break;
1755                 pos += block_len;
1756
1757                 /* Avoid locking up the system */
1758                 cond_resched();
1759                 if (signal_pending(current)) {
1760                         rc = -EINTR;
1761                         break;
1762                 }
1763         }
1764
1765         if (retlen)
1766                 *retlen = pos;
1767         return rc;
1768 }
1769
1770 int falcon_spi_write(const struct efx_spi_device *spi, loff_t start,
1771                      size_t len, size_t *retlen, const u8 *buffer)
1772 {
1773         u8 verify_buffer[FALCON_SPI_MAX_LEN];
1774         size_t block_len, pos = 0;
1775         unsigned int command;
1776         int rc = 0;
1777
1778         while (pos < len) {
1779                 rc = falcon_spi_cmd(spi, SPI_WREN, -1, NULL, NULL, 0);
1780                 if (rc)
1781                         break;
1782
1783                 block_len = min(len - pos,
1784                                 falcon_spi_write_limit(spi, start + pos));
1785                 command = efx_spi_munge_command(spi, SPI_WRITE, start + pos);
1786                 rc = falcon_spi_cmd(spi, command, start + pos,
1787                                     buffer + pos, NULL, block_len);
1788                 if (rc)
1789                         break;
1790
1791                 rc = falcon_spi_wait_write(spi);
1792                 if (rc)
1793                         break;
1794
1795                 command = efx_spi_munge_command(spi, SPI_READ, start + pos);
1796                 rc = falcon_spi_cmd(spi, command, start + pos,
1797                                     NULL, verify_buffer, block_len);
1798                 if (memcmp(verify_buffer, buffer + pos, block_len)) {
1799                         rc = -EIO;
1800                         break;
1801                 }
1802
1803                 pos += block_len;
1804
1805                 /* Avoid locking up the system */
1806                 cond_resched();
1807                 if (signal_pending(current)) {
1808                         rc = -EINTR;
1809                         break;
1810                 }
1811         }
1812
1813         if (retlen)
1814                 *retlen = pos;
1815         return rc;
1816 }
1817
1818 /**************************************************************************
1819  *
1820  * MAC wrapper
1821  *
1822  **************************************************************************
1823  */
1824
1825 static void falcon_push_multicast_hash(struct efx_nic *efx)
1826 {
1827         union efx_multicast_hash *mc_hash = &efx->multicast_hash;
1828
1829         WARN_ON(!mutex_is_locked(&efx->mac_lock));
1830
1831         efx_writeo(efx, &mc_hash->oword[0], FR_AB_MAC_MC_HASH_REG0);
1832         efx_writeo(efx, &mc_hash->oword[1], FR_AB_MAC_MC_HASH_REG1);
1833 }
1834
1835 static void falcon_reset_macs(struct efx_nic *efx)
1836 {
1837         struct falcon_nic_data *nic_data = efx->nic_data;
1838         efx_oword_t reg, mac_ctrl;
1839         int count;
1840
1841         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
1842                 /* It's not safe to use GLB_CTL_REG to reset the
1843                  * macs, so instead use the internal MAC resets
1844                  */
1845                 if (!EFX_IS10G(efx)) {
1846                         EFX_POPULATE_OWORD_1(reg, FRF_AB_GM_SW_RST, 1);
1847                         efx_writeo(efx, &reg, FR_AB_GM_CFG1);
1848                         udelay(1000);
1849
1850                         EFX_POPULATE_OWORD_1(reg, FRF_AB_GM_SW_RST, 0);
1851                         efx_writeo(efx, &reg, FR_AB_GM_CFG1);
1852                         udelay(1000);
1853                         return;
1854                 } else {
1855                         EFX_POPULATE_OWORD_1(reg, FRF_AB_XM_CORE_RST, 1);
1856                         efx_writeo(efx, &reg, FR_AB_XM_GLB_CFG);
1857
1858                         for (count = 0; count < 10000; count++) {
1859                                 efx_reado(efx, &reg, FR_AB_XM_GLB_CFG);
1860                                 if (EFX_OWORD_FIELD(reg, FRF_AB_XM_CORE_RST) ==
1861                                     0)
1862                                         return;
1863                                 udelay(10);
1864                         }
1865
1866                         EFX_ERR(efx, "timed out waiting for XMAC core reset\n");
1867                 }
1868         }
1869
1870         /* Mac stats will fail whist the TX fifo is draining */
1871         WARN_ON(nic_data->stats_disable_count == 0);
1872
1873         efx_reado(efx, &mac_ctrl, FR_AB_MAC_CTRL);
1874         EFX_SET_OWORD_FIELD(mac_ctrl, FRF_BB_TXFIFO_DRAIN_EN, 1);
1875         efx_writeo(efx, &mac_ctrl, FR_AB_MAC_CTRL);
1876
1877         efx_reado(efx, &reg, FR_AB_GLB_CTL);
1878         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_XGTX, 1);
1879         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_XGRX, 1);
1880         EFX_SET_OWORD_FIELD(reg, FRF_AB_RST_EM, 1);
1881         efx_writeo(efx, &reg, FR_AB_GLB_CTL);
1882
1883         count = 0;
1884         while (1) {
1885                 efx_reado(efx, &reg, FR_AB_GLB_CTL);
1886                 if (!EFX_OWORD_FIELD(reg, FRF_AB_RST_XGTX) &&
1887                     !EFX_OWORD_FIELD(reg, FRF_AB_RST_XGRX) &&
1888                     !EFX_OWORD_FIELD(reg, FRF_AB_RST_EM)) {
1889                         EFX_LOG(efx, "Completed MAC reset after %d loops\n",
1890                                 count);
1891                         break;
1892                 }
1893                 if (count > 20) {
1894                         EFX_ERR(efx, "MAC reset failed\n");
1895                         break;
1896                 }
1897                 count++;
1898                 udelay(10);
1899         }
1900
1901         /* Ensure the correct MAC is selected before statistics
1902          * are re-enabled by the caller */
1903         efx_writeo(efx, &mac_ctrl, FR_AB_MAC_CTRL);
1904 }
1905
1906 void falcon_drain_tx_fifo(struct efx_nic *efx)
1907 {
1908         efx_oword_t reg;
1909
1910         if ((efx_nic_rev(efx) < EFX_REV_FALCON_B0) ||
1911             (efx->loopback_mode != LOOPBACK_NONE))
1912                 return;
1913
1914         efx_reado(efx, &reg, FR_AB_MAC_CTRL);
1915         /* There is no point in draining more than once */
1916         if (EFX_OWORD_FIELD(reg, FRF_BB_TXFIFO_DRAIN_EN))
1917                 return;
1918
1919         falcon_reset_macs(efx);
1920 }
1921
1922 static void falcon_deconfigure_mac_wrapper(struct efx_nic *efx)
1923 {
1924         efx_oword_t reg;
1925
1926         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1927                 return;
1928
1929         /* Isolate the MAC -> RX */
1930         efx_reado(efx, &reg, FR_AZ_RX_CFG);
1931         EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 0);
1932         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
1933
1934         /* Isolate TX -> MAC */
1935         falcon_drain_tx_fifo(efx);
1936 }
1937
1938 void falcon_reconfigure_mac_wrapper(struct efx_nic *efx)
1939 {
1940         struct efx_link_state *link_state = &efx->link_state;
1941         efx_oword_t reg;
1942         int link_speed;
1943
1944         switch (link_state->speed) {
1945         case 10000: link_speed = 3; break;
1946         case 1000:  link_speed = 2; break;
1947         case 100:   link_speed = 1; break;
1948         default:    link_speed = 0; break;
1949         }
1950         /* MAC_LINK_STATUS controls MAC backpressure but doesn't work
1951          * as advertised.  Disable to ensure packets are not
1952          * indefinitely held and TX queue can be flushed at any point
1953          * while the link is down. */
1954         EFX_POPULATE_OWORD_5(reg,
1955                              FRF_AB_MAC_XOFF_VAL, 0xffff /* max pause time */,
1956                              FRF_AB_MAC_BCAD_ACPT, 1,
1957                              FRF_AB_MAC_UC_PROM, efx->promiscuous,
1958                              FRF_AB_MAC_LINK_STATUS, 1, /* always set */
1959                              FRF_AB_MAC_SPEED, link_speed);
1960         /* On B0, MAC backpressure can be disabled and packets get
1961          * discarded. */
1962         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1963                 EFX_SET_OWORD_FIELD(reg, FRF_BB_TXFIFO_DRAIN_EN,
1964                                     !link_state->up);
1965         }
1966
1967         efx_writeo(efx, &reg, FR_AB_MAC_CTRL);
1968
1969         /* Restore the multicast hash registers. */
1970         falcon_push_multicast_hash(efx);
1971
1972         efx_reado(efx, &reg, FR_AZ_RX_CFG);
1973         /* Enable XOFF signal from RX FIFO (we enabled it during NIC
1974          * initialisation but it may read back as 0) */
1975         EFX_SET_OWORD_FIELD(reg, FRF_AZ_RX_XOFF_MAC_EN, 1);
1976         /* Unisolate the MAC -> RX */
1977         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1978                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 1);
1979         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
1980 }
1981
1982 static void falcon_stats_request(struct efx_nic *efx)
1983 {
1984         struct falcon_nic_data *nic_data = efx->nic_data;
1985         efx_oword_t reg;
1986
1987         WARN_ON(nic_data->stats_pending);
1988         WARN_ON(nic_data->stats_disable_count);
1989
1990         if (nic_data->stats_dma_done == NULL)
1991                 return; /* no mac selected */
1992
1993         *nic_data->stats_dma_done = FALCON_STATS_NOT_DONE;
1994         nic_data->stats_pending = true;
1995         wmb(); /* ensure done flag is clear */
1996
1997         /* Initiate DMA transfer of stats */
1998         EFX_POPULATE_OWORD_2(reg,
1999                              FRF_AB_MAC_STAT_DMA_CMD, 1,
2000                              FRF_AB_MAC_STAT_DMA_ADR,
2001                              efx->stats_buffer.dma_addr);
2002         efx_writeo(efx, &reg, FR_AB_MAC_STAT_DMA);
2003
2004         mod_timer(&nic_data->stats_timer, round_jiffies_up(jiffies + HZ / 2));
2005 }
2006
2007 static void falcon_stats_complete(struct efx_nic *efx)
2008 {
2009         struct falcon_nic_data *nic_data = efx->nic_data;
2010
2011         if (!nic_data->stats_pending)
2012                 return;
2013
2014         nic_data->stats_pending = 0;
2015         if (*nic_data->stats_dma_done == FALCON_STATS_DONE) {
2016                 rmb(); /* read the done flag before the stats */
2017                 efx->mac_op->update_stats(efx);
2018         } else {
2019                 EFX_ERR(efx, "timed out waiting for statistics\n");
2020         }
2021 }
2022
2023 static void falcon_stats_timer_func(unsigned long context)
2024 {
2025         struct efx_nic *efx = (struct efx_nic *)context;
2026         struct falcon_nic_data *nic_data = efx->nic_data;
2027
2028         spin_lock(&efx->stats_lock);
2029
2030         falcon_stats_complete(efx);
2031         if (nic_data->stats_disable_count == 0)
2032                 falcon_stats_request(efx);
2033
2034         spin_unlock(&efx->stats_lock);
2035 }
2036
2037 static void falcon_switch_mac(struct efx_nic *efx);
2038
2039 static bool falcon_loopback_link_poll(struct efx_nic *efx)
2040 {
2041         struct efx_link_state old_state = efx->link_state;
2042
2043         WARN_ON(!mutex_is_locked(&efx->mac_lock));
2044         WARN_ON(!LOOPBACK_INTERNAL(efx));
2045
2046         efx->link_state.fd = true;
2047         efx->link_state.fc = efx->wanted_fc;
2048         efx->link_state.up = true;
2049
2050         if (efx->loopback_mode == LOOPBACK_GMAC)
2051                 efx->link_state.speed = 1000;
2052         else
2053                 efx->link_state.speed = 10000;
2054
2055         return !efx_link_state_equal(&efx->link_state, &old_state);
2056 }
2057
2058 static int falcon_reconfigure_port(struct efx_nic *efx)
2059 {
2060         int rc;
2061
2062         WARN_ON(efx_nic_rev(efx) > EFX_REV_FALCON_B0);
2063
2064         /* Poll the PHY link state *before* reconfiguring it. This means we
2065          * will pick up the correct speed (in loopback) to select the correct
2066          * MAC.
2067          */
2068         if (LOOPBACK_INTERNAL(efx))
2069                 falcon_loopback_link_poll(efx);
2070         else
2071                 efx->phy_op->poll(efx);
2072
2073         falcon_stop_nic_stats(efx);
2074         falcon_deconfigure_mac_wrapper(efx);
2075
2076         falcon_switch_mac(efx);
2077
2078         efx->phy_op->reconfigure(efx);
2079         rc = efx->mac_op->reconfigure(efx);
2080         BUG_ON(rc);
2081
2082         falcon_start_nic_stats(efx);
2083
2084         /* Synchronise efx->link_state with the kernel */
2085         efx_link_status_changed(efx);
2086
2087         return 0;
2088 }
2089
2090 /**************************************************************************
2091  *
2092  * PHY access via GMII
2093  *
2094  **************************************************************************
2095  */
2096
2097 /* Wait for GMII access to complete */
2098 static int falcon_gmii_wait(struct efx_nic *efx)
2099 {
2100         efx_oword_t md_stat;
2101         int count;
2102
2103         /* wait upto 50ms - taken max from datasheet */
2104         for (count = 0; count < 5000; count++) {
2105                 efx_reado(efx, &md_stat, FR_AB_MD_STAT);
2106                 if (EFX_OWORD_FIELD(md_stat, FRF_AB_MD_BSY) == 0) {
2107                         if (EFX_OWORD_FIELD(md_stat, FRF_AB_MD_LNFL) != 0 ||
2108                             EFX_OWORD_FIELD(md_stat, FRF_AB_MD_BSERR) != 0) {
2109                                 EFX_ERR(efx, "error from GMII access "
2110                                         EFX_OWORD_FMT"\n",
2111                                         EFX_OWORD_VAL(md_stat));
2112                                 return -EIO;
2113                         }
2114                         return 0;
2115                 }
2116                 udelay(10);
2117         }
2118         EFX_ERR(efx, "timed out waiting for GMII\n");
2119         return -ETIMEDOUT;
2120 }
2121
2122 /* Write an MDIO register of a PHY connected to Falcon. */
2123 static int falcon_mdio_write(struct net_device *net_dev,
2124                              int prtad, int devad, u16 addr, u16 value)
2125 {
2126         struct efx_nic *efx = netdev_priv(net_dev);
2127         efx_oword_t reg;
2128         int rc;
2129
2130         EFX_REGDUMP(efx, "writing MDIO %d register %d.%d with 0x%04x\n",
2131                     prtad, devad, addr, value);
2132
2133         mutex_lock(&efx->mdio_lock);
2134
2135         /* Check MDIO not currently being accessed */
2136         rc = falcon_gmii_wait(efx);
2137         if (rc)
2138                 goto out;
2139
2140         /* Write the address/ID register */
2141         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_PHY_ADR, addr);
2142         efx_writeo(efx, &reg, FR_AB_MD_PHY_ADR);
2143
2144         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_PRT_ADR, prtad,
2145                              FRF_AB_MD_DEV_ADR, devad);
2146         efx_writeo(efx, &reg, FR_AB_MD_ID);
2147
2148         /* Write data */
2149         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_TXD, value);
2150         efx_writeo(efx, &reg, FR_AB_MD_TXD);
2151
2152         EFX_POPULATE_OWORD_2(reg,
2153                              FRF_AB_MD_WRC, 1,
2154                              FRF_AB_MD_GC, 0);
2155         efx_writeo(efx, &reg, FR_AB_MD_CS);
2156
2157         /* Wait for data to be written */
2158         rc = falcon_gmii_wait(efx);
2159         if (rc) {
2160                 /* Abort the write operation */
2161                 EFX_POPULATE_OWORD_2(reg,
2162                                      FRF_AB_MD_WRC, 0,
2163                                      FRF_AB_MD_GC, 1);
2164                 efx_writeo(efx, &reg, FR_AB_MD_CS);
2165                 udelay(10);
2166         }
2167
2168 out:
2169         mutex_unlock(&efx->mdio_lock);
2170         return rc;
2171 }
2172
2173 /* Read an MDIO register of a PHY connected to Falcon. */
2174 static int falcon_mdio_read(struct net_device *net_dev,
2175                             int prtad, int devad, u16 addr)
2176 {
2177         struct efx_nic *efx = netdev_priv(net_dev);
2178         efx_oword_t reg;
2179         int rc;
2180
2181         mutex_lock(&efx->mdio_lock);
2182
2183         /* Check MDIO not currently being accessed */
2184         rc = falcon_gmii_wait(efx);
2185         if (rc)
2186                 goto out;
2187
2188         EFX_POPULATE_OWORD_1(reg, FRF_AB_MD_PHY_ADR, addr);
2189         efx_writeo(efx, &reg, FR_AB_MD_PHY_ADR);
2190
2191         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_PRT_ADR, prtad,
2192                              FRF_AB_MD_DEV_ADR, devad);
2193         efx_writeo(efx, &reg, FR_AB_MD_ID);
2194
2195         /* Request data to be read */
2196         EFX_POPULATE_OWORD_2(reg, FRF_AB_MD_RDC, 1, FRF_AB_MD_GC, 0);
2197         efx_writeo(efx, &reg, FR_AB_MD_CS);
2198
2199         /* Wait for data to become available */
2200         rc = falcon_gmii_wait(efx);
2201         if (rc == 0) {
2202                 efx_reado(efx, &reg, FR_AB_MD_RXD);
2203                 rc = EFX_OWORD_FIELD(reg, FRF_AB_MD_RXD);
2204                 EFX_REGDUMP(efx, "read from MDIO %d register %d.%d, got %04x\n",
2205                             prtad, devad, addr, rc);
2206         } else {
2207                 /* Abort the read operation */
2208                 EFX_POPULATE_OWORD_2(reg,
2209                                      FRF_AB_MD_RIC, 0,
2210                                      FRF_AB_MD_GC, 1);
2211                 efx_writeo(efx, &reg, FR_AB_MD_CS);
2212
2213                 EFX_LOG(efx, "read from MDIO %d register %d.%d, got error %d\n",
2214                         prtad, devad, addr, rc);
2215         }
2216
2217 out:
2218         mutex_unlock(&efx->mdio_lock);
2219         return rc;
2220 }
2221
2222 static void falcon_clock_mac(struct efx_nic *efx)
2223 {
2224         unsigned strap_val;
2225         efx_oword_t nic_stat;
2226
2227         /* Configure the NIC generated MAC clock correctly */
2228         efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2229         strap_val = EFX_IS10G(efx) ? 5 : 3;
2230         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
2231                 EFX_SET_OWORD_FIELD(nic_stat, FRF_BB_EE_STRAP_EN, 1);
2232                 EFX_SET_OWORD_FIELD(nic_stat, FRF_BB_EE_STRAP, strap_val);
2233                 efx_writeo(efx, &nic_stat, FR_AB_NIC_STAT);
2234         } else {
2235                 /* Falcon A1 does not support 1G/10G speed switching
2236                  * and must not be used with a PHY that does. */
2237                 BUG_ON(EFX_OWORD_FIELD(nic_stat, FRF_AB_STRAP_PINS) !=
2238                        strap_val);
2239         }
2240 }
2241
2242 static void falcon_switch_mac(struct efx_nic *efx)
2243 {
2244         struct efx_mac_operations *old_mac_op = efx->mac_op;
2245         struct falcon_nic_data *nic_data = efx->nic_data;
2246         unsigned int stats_done_offset;
2247
2248         WARN_ON(!mutex_is_locked(&efx->mac_lock));
2249         WARN_ON(nic_data->stats_disable_count == 0);
2250
2251         efx->mac_op = (EFX_IS10G(efx) ?
2252                        &falcon_xmac_operations : &falcon_gmac_operations);
2253
2254         if (EFX_IS10G(efx))
2255                 stats_done_offset = XgDmaDone_offset;
2256         else
2257                 stats_done_offset = GDmaDone_offset;
2258         nic_data->stats_dma_done = efx->stats_buffer.addr + stats_done_offset;
2259
2260         if (old_mac_op == efx->mac_op)
2261                 return;
2262
2263         falcon_clock_mac(efx);
2264
2265         EFX_LOG(efx, "selected %cMAC\n", EFX_IS10G(efx) ? 'X' : 'G');
2266         /* Not all macs support a mac-level link state */
2267         efx->xmac_poll_required = false;
2268         falcon_reset_macs(efx);
2269 }
2270
2271 /* This call is responsible for hooking in the MAC and PHY operations */
2272 static int falcon_probe_port(struct efx_nic *efx)
2273 {
2274         int rc;
2275
2276         switch (efx->phy_type) {
2277         case PHY_TYPE_SFX7101:
2278                 efx->phy_op = &falcon_sfx7101_phy_ops;
2279                 break;
2280         case PHY_TYPE_SFT9001A:
2281         case PHY_TYPE_SFT9001B:
2282                 efx->phy_op = &falcon_sft9001_phy_ops;
2283                 break;
2284         case PHY_TYPE_QT2022C2:
2285         case PHY_TYPE_QT2025C:
2286                 efx->phy_op = &falcon_qt202x_phy_ops;
2287                 break;
2288         default:
2289                 EFX_ERR(efx, "Unknown PHY type %d\n",
2290                         efx->phy_type);
2291                 return -ENODEV;
2292         }
2293
2294         /* Fill out MDIO structure and loopback modes */
2295         efx->mdio.mdio_read = falcon_mdio_read;
2296         efx->mdio.mdio_write = falcon_mdio_write;
2297         rc = efx->phy_op->probe(efx);
2298         if (rc != 0)
2299                 return rc;
2300
2301         /* Initial assumption */
2302         efx->link_state.speed = 10000;
2303         efx->link_state.fd = true;
2304
2305         /* Hardware flow ctrl. FalconA RX FIFO too small for pause generation */
2306         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
2307                 efx->wanted_fc = EFX_FC_RX | EFX_FC_TX;
2308         else
2309                 efx->wanted_fc = EFX_FC_RX;
2310
2311         /* Allocate buffer for stats */
2312         rc = efx_nic_alloc_buffer(efx, &efx->stats_buffer,
2313                                   FALCON_MAC_STATS_SIZE);
2314         if (rc)
2315                 return rc;
2316         EFX_LOG(efx, "stats buffer at %llx (virt %p phys %llx)\n",
2317                 (u64)efx->stats_buffer.dma_addr,
2318                 efx->stats_buffer.addr,
2319                 (u64)virt_to_phys(efx->stats_buffer.addr));
2320
2321         return 0;
2322 }
2323
2324 static void falcon_remove_port(struct efx_nic *efx)
2325 {
2326         efx_nic_free_buffer(efx, &efx->stats_buffer);
2327 }
2328
2329 /**************************************************************************
2330  *
2331  * Falcon test code
2332  *
2333  **************************************************************************/
2334
2335 static int
2336 falcon_read_nvram(struct efx_nic *efx, struct falcon_nvconfig *nvconfig_out)
2337 {
2338         struct falcon_nvconfig *nvconfig;
2339         struct efx_spi_device *spi;
2340         void *region;
2341         int rc, magic_num, struct_ver;
2342         __le16 *word, *limit;
2343         u32 csum;
2344
2345         spi = efx->spi_flash ? efx->spi_flash : efx->spi_eeprom;
2346         if (!spi)
2347                 return -EINVAL;
2348
2349         region = kmalloc(FALCON_NVCONFIG_END, GFP_KERNEL);
2350         if (!region)
2351                 return -ENOMEM;
2352         nvconfig = region + FALCON_NVCONFIG_OFFSET;
2353
2354         mutex_lock(&efx->spi_lock);
2355         rc = falcon_spi_read(spi, 0, FALCON_NVCONFIG_END, NULL, region);
2356         mutex_unlock(&efx->spi_lock);
2357         if (rc) {
2358                 EFX_ERR(efx, "Failed to read %s\n",
2359                         efx->spi_flash ? "flash" : "EEPROM");
2360                 rc = -EIO;
2361                 goto out;
2362         }
2363
2364         magic_num = le16_to_cpu(nvconfig->board_magic_num);
2365         struct_ver = le16_to_cpu(nvconfig->board_struct_ver);
2366
2367         rc = -EINVAL;
2368         if (magic_num != FALCON_NVCONFIG_BOARD_MAGIC_NUM) {
2369                 EFX_ERR(efx, "NVRAM bad magic 0x%x\n", magic_num);
2370                 goto out;
2371         }
2372         if (struct_ver < 2) {
2373                 EFX_ERR(efx, "NVRAM has ancient version 0x%x\n", struct_ver);
2374                 goto out;
2375         } else if (struct_ver < 4) {
2376                 word = &nvconfig->board_magic_num;
2377                 limit = (__le16 *) (nvconfig + 1);
2378         } else {
2379                 word = region;
2380                 limit = region + FALCON_NVCONFIG_END;
2381         }
2382         for (csum = 0; word < limit; ++word)
2383                 csum += le16_to_cpu(*word);
2384
2385         if (~csum & 0xffff) {
2386                 EFX_ERR(efx, "NVRAM has incorrect checksum\n");
2387                 goto out;
2388         }
2389
2390         rc = 0;
2391         if (nvconfig_out)
2392                 memcpy(nvconfig_out, nvconfig, sizeof(*nvconfig));
2393
2394  out:
2395         kfree(region);
2396         return rc;
2397 }
2398
2399 static int falcon_test_nvram(struct efx_nic *efx)
2400 {
2401         return falcon_read_nvram(efx, NULL);
2402 }
2403
2404 static const struct efx_nic_register_test falcon_b0_register_tests[] = {
2405         { FR_AZ_ADR_REGION,
2406           EFX_OWORD32(0x0001FFFF, 0x0001FFFF, 0x0001FFFF, 0x0001FFFF) },
2407         { FR_AZ_RX_CFG,
2408           EFX_OWORD32(0xFFFFFFFE, 0x00017FFF, 0x00000000, 0x00000000) },
2409         { FR_AZ_TX_CFG,
2410           EFX_OWORD32(0x7FFF0037, 0x00000000, 0x00000000, 0x00000000) },
2411         { FR_AZ_TX_RESERVED,
2412           EFX_OWORD32(0xFFFEFE80, 0x1FFFFFFF, 0x020000FE, 0x007FFFFF) },
2413         { FR_AB_MAC_CTRL,
2414           EFX_OWORD32(0xFFFF0000, 0x00000000, 0x00000000, 0x00000000) },
2415         { FR_AZ_SRM_TX_DC_CFG,
2416           EFX_OWORD32(0x001FFFFF, 0x00000000, 0x00000000, 0x00000000) },
2417         { FR_AZ_RX_DC_CFG,
2418           EFX_OWORD32(0x0000000F, 0x00000000, 0x00000000, 0x00000000) },
2419         { FR_AZ_RX_DC_PF_WM,
2420           EFX_OWORD32(0x000003FF, 0x00000000, 0x00000000, 0x00000000) },
2421         { FR_BZ_DP_CTRL,
2422           EFX_OWORD32(0x00000FFF, 0x00000000, 0x00000000, 0x00000000) },
2423         { FR_AB_GM_CFG2,
2424           EFX_OWORD32(0x00007337, 0x00000000, 0x00000000, 0x00000000) },
2425         { FR_AB_GMF_CFG0,
2426           EFX_OWORD32(0x00001F1F, 0x00000000, 0x00000000, 0x00000000) },
2427         { FR_AB_XM_GLB_CFG,
2428           EFX_OWORD32(0x00000C68, 0x00000000, 0x00000000, 0x00000000) },
2429         { FR_AB_XM_TX_CFG,
2430           EFX_OWORD32(0x00080164, 0x00000000, 0x00000000, 0x00000000) },
2431         { FR_AB_XM_RX_CFG,
2432           EFX_OWORD32(0x07100A0C, 0x00000000, 0x00000000, 0x00000000) },
2433         { FR_AB_XM_RX_PARAM,
2434           EFX_OWORD32(0x00001FF8, 0x00000000, 0x00000000, 0x00000000) },
2435         { FR_AB_XM_FC,
2436           EFX_OWORD32(0xFFFF0001, 0x00000000, 0x00000000, 0x00000000) },
2437         { FR_AB_XM_ADR_LO,
2438           EFX_OWORD32(0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000) },
2439         { FR_AB_XX_SD_CTL,
2440           EFX_OWORD32(0x0003FF0F, 0x00000000, 0x00000000, 0x00000000) },
2441 };
2442
2443 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
2444                                      const efx_oword_t *mask)
2445 {
2446         return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
2447                 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
2448 }
2449
2450 int efx_nic_test_registers(struct efx_nic *efx,
2451                            const struct efx_nic_register_test *regs,
2452                            size_t n_regs)
2453 {
2454         unsigned address = 0, i, j;
2455         efx_oword_t mask, imask, original, reg, buf;
2456
2457         /* Falcon should be in loopback to isolate the XMAC from the PHY */
2458         WARN_ON(!LOOPBACK_INTERNAL(efx));
2459
2460         for (i = 0; i < n_regs; ++i) {
2461                 address = regs[i].address;
2462                 mask = imask = regs[i].mask;
2463                 EFX_INVERT_OWORD(imask);
2464
2465                 efx_reado(efx, &original, address);
2466
2467                 /* bit sweep on and off */
2468                 for (j = 0; j < 128; j++) {
2469                         if (!EFX_EXTRACT_OWORD32(mask, j, j))
2470                                 continue;
2471
2472                         /* Test this testable bit can be set in isolation */
2473                         EFX_AND_OWORD(reg, original, mask);
2474                         EFX_SET_OWORD32(reg, j, j, 1);
2475
2476                         efx_writeo(efx, &reg, address);
2477                         efx_reado(efx, &buf, address);
2478
2479                         if (efx_masked_compare_oword(&reg, &buf, &mask))
2480                                 goto fail;
2481
2482                         /* Test this testable bit can be cleared in isolation */
2483                         EFX_OR_OWORD(reg, original, mask);
2484                         EFX_SET_OWORD32(reg, j, j, 0);
2485
2486                         efx_writeo(efx, &reg, address);
2487                         efx_reado(efx, &buf, address);
2488
2489                         if (efx_masked_compare_oword(&reg, &buf, &mask))
2490                                 goto fail;
2491                 }
2492
2493                 efx_writeo(efx, &original, address);
2494         }
2495
2496         return 0;
2497
2498 fail:
2499         EFX_ERR(efx, "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
2500                 " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
2501                 EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
2502         return -EIO;
2503 }
2504
2505 static int falcon_b0_test_registers(struct efx_nic *efx)
2506 {
2507         return efx_nic_test_registers(efx, falcon_b0_register_tests,
2508                                       ARRAY_SIZE(falcon_b0_register_tests));
2509 }
2510
2511 /**************************************************************************
2512  *
2513  * Device reset
2514  *
2515  **************************************************************************
2516  */
2517
2518 /* Resets NIC to known state.  This routine must be called in process
2519  * context and is allowed to sleep. */
2520 static int falcon_reset_hw(struct efx_nic *efx, enum reset_type method)
2521 {
2522         struct falcon_nic_data *nic_data = efx->nic_data;
2523         efx_oword_t glb_ctl_reg_ker;
2524         int rc;
2525
2526         EFX_LOG(efx, "performing %s hardware reset\n", RESET_TYPE(method));
2527
2528         /* Initiate device reset */
2529         if (method == RESET_TYPE_WORLD) {
2530                 rc = pci_save_state(efx->pci_dev);
2531                 if (rc) {
2532                         EFX_ERR(efx, "failed to backup PCI state of primary "
2533                                 "function prior to hardware reset\n");
2534                         goto fail1;
2535                 }
2536                 if (efx_nic_is_dual_func(efx)) {
2537                         rc = pci_save_state(nic_data->pci_dev2);
2538                         if (rc) {
2539                                 EFX_ERR(efx, "failed to backup PCI state of "
2540                                         "secondary function prior to "
2541                                         "hardware reset\n");
2542                                 goto fail2;
2543                         }
2544                 }
2545
2546                 EFX_POPULATE_OWORD_2(glb_ctl_reg_ker,
2547                                      FRF_AB_EXT_PHY_RST_DUR,
2548                                      FFE_AB_EXT_PHY_RST_DUR_10240US,
2549                                      FRF_AB_SWRST, 1);
2550         } else {
2551                 EFX_POPULATE_OWORD_7(glb_ctl_reg_ker,
2552                                      /* exclude PHY from "invisible" reset */
2553                                      FRF_AB_EXT_PHY_RST_CTL,
2554                                      method == RESET_TYPE_INVISIBLE,
2555                                      /* exclude EEPROM/flash and PCIe */
2556                                      FRF_AB_PCIE_CORE_RST_CTL, 1,
2557                                      FRF_AB_PCIE_NSTKY_RST_CTL, 1,
2558                                      FRF_AB_PCIE_SD_RST_CTL, 1,
2559                                      FRF_AB_EE_RST_CTL, 1,
2560                                      FRF_AB_EXT_PHY_RST_DUR,
2561                                      FFE_AB_EXT_PHY_RST_DUR_10240US,
2562                                      FRF_AB_SWRST, 1);
2563         }
2564         efx_writeo(efx, &glb_ctl_reg_ker, FR_AB_GLB_CTL);
2565
2566         EFX_LOG(efx, "waiting for hardware reset\n");
2567         schedule_timeout_uninterruptible(HZ / 20);
2568
2569         /* Restore PCI configuration if needed */
2570         if (method == RESET_TYPE_WORLD) {
2571                 if (efx_nic_is_dual_func(efx)) {
2572                         rc = pci_restore_state(nic_data->pci_dev2);
2573                         if (rc) {
2574                                 EFX_ERR(efx, "failed to restore PCI config for "
2575                                         "the secondary function\n");
2576                                 goto fail3;
2577                         }
2578                 }
2579                 rc = pci_restore_state(efx->pci_dev);
2580                 if (rc) {
2581                         EFX_ERR(efx, "failed to restore PCI config for the "
2582                                 "primary function\n");
2583                         goto fail4;
2584                 }
2585                 EFX_LOG(efx, "successfully restored PCI config\n");
2586         }
2587
2588         /* Assert that reset complete */
2589         efx_reado(efx, &glb_ctl_reg_ker, FR_AB_GLB_CTL);
2590         if (EFX_OWORD_FIELD(glb_ctl_reg_ker, FRF_AB_SWRST) != 0) {
2591                 rc = -ETIMEDOUT;
2592                 EFX_ERR(efx, "timed out waiting for hardware reset\n");
2593                 goto fail5;
2594         }
2595         EFX_LOG(efx, "hardware reset complete\n");
2596
2597         return 0;
2598
2599         /* pci_save_state() and pci_restore_state() MUST be called in pairs */
2600 fail2:
2601 fail3:
2602         pci_restore_state(efx->pci_dev);
2603 fail1:
2604 fail4:
2605 fail5:
2606         return rc;
2607 }
2608
2609 static void falcon_monitor(struct efx_nic *efx)
2610 {
2611         bool link_changed;
2612         int rc;
2613
2614         BUG_ON(!mutex_is_locked(&efx->mac_lock));
2615
2616         rc = falcon_board(efx)->type->monitor(efx);
2617         if (rc) {
2618                 EFX_ERR(efx, "Board sensor %s; shutting down PHY\n",
2619                         (rc == -ERANGE) ? "reported fault" : "failed");
2620                 efx->phy_mode |= PHY_MODE_LOW_POWER;
2621                 rc = __efx_reconfigure_port(efx);
2622                 WARN_ON(rc);
2623         }
2624
2625         if (LOOPBACK_INTERNAL(efx))
2626                 link_changed = falcon_loopback_link_poll(efx);
2627         else
2628                 link_changed = efx->phy_op->poll(efx);
2629
2630         if (link_changed) {
2631                 falcon_stop_nic_stats(efx);
2632                 falcon_deconfigure_mac_wrapper(efx);
2633
2634                 falcon_switch_mac(efx);
2635                 rc = efx->mac_op->reconfigure(efx);
2636                 BUG_ON(rc);
2637
2638                 falcon_start_nic_stats(efx);
2639
2640                 efx_link_status_changed(efx);
2641         }
2642
2643         if (EFX_IS10G(efx))
2644                 falcon_poll_xmac(efx);
2645 }
2646
2647 /* Zeroes out the SRAM contents.  This routine must be called in
2648  * process context and is allowed to sleep.
2649  */
2650 static int falcon_reset_sram(struct efx_nic *efx)
2651 {
2652         efx_oword_t srm_cfg_reg_ker, gpio_cfg_reg_ker;
2653         int count;
2654
2655         /* Set the SRAM wake/sleep GPIO appropriately. */
2656         efx_reado(efx, &gpio_cfg_reg_ker, FR_AB_GPIO_CTL);
2657         EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, FRF_AB_GPIO1_OEN, 1);
2658         EFX_SET_OWORD_FIELD(gpio_cfg_reg_ker, FRF_AB_GPIO1_OUT, 1);
2659         efx_writeo(efx, &gpio_cfg_reg_ker, FR_AB_GPIO_CTL);
2660
2661         /* Initiate SRAM reset */
2662         EFX_POPULATE_OWORD_2(srm_cfg_reg_ker,
2663                              FRF_AZ_SRM_INIT_EN, 1,
2664                              FRF_AZ_SRM_NB_SZ, 0);
2665         efx_writeo(efx, &srm_cfg_reg_ker, FR_AZ_SRM_CFG);
2666
2667         /* Wait for SRAM reset to complete */
2668         count = 0;
2669         do {
2670                 EFX_LOG(efx, "waiting for SRAM reset (attempt %d)...\n", count);
2671
2672                 /* SRAM reset is slow; expect around 16ms */
2673                 schedule_timeout_uninterruptible(HZ / 50);
2674
2675                 /* Check for reset complete */
2676                 efx_reado(efx, &srm_cfg_reg_ker, FR_AZ_SRM_CFG);
2677                 if (!EFX_OWORD_FIELD(srm_cfg_reg_ker, FRF_AZ_SRM_INIT_EN)) {
2678                         EFX_LOG(efx, "SRAM reset complete\n");
2679
2680                         return 0;
2681                 }
2682         } while (++count < 20); /* wait upto 0.4 sec */
2683
2684         EFX_ERR(efx, "timed out waiting for SRAM reset\n");
2685         return -ETIMEDOUT;
2686 }
2687
2688 static int falcon_spi_device_init(struct efx_nic *efx,
2689                                   struct efx_spi_device **spi_device_ret,
2690                                   unsigned int device_id, u32 device_type)
2691 {
2692         struct efx_spi_device *spi_device;
2693
2694         if (device_type != 0) {
2695                 spi_device = kzalloc(sizeof(*spi_device), GFP_KERNEL);
2696                 if (!spi_device)
2697                         return -ENOMEM;
2698                 spi_device->device_id = device_id;
2699                 spi_device->size =
2700                         1 << SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_SIZE);
2701                 spi_device->addr_len =
2702                         SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_ADDR_LEN);
2703                 spi_device->munge_address = (spi_device->size == 1 << 9 &&
2704                                              spi_device->addr_len == 1);
2705                 spi_device->erase_command =
2706                         SPI_DEV_TYPE_FIELD(device_type, SPI_DEV_TYPE_ERASE_CMD);
2707                 spi_device->erase_size =
2708                         1 << SPI_DEV_TYPE_FIELD(device_type,
2709                                                 SPI_DEV_TYPE_ERASE_SIZE);
2710                 spi_device->block_size =
2711                         1 << SPI_DEV_TYPE_FIELD(device_type,
2712                                                 SPI_DEV_TYPE_BLOCK_SIZE);
2713
2714                 spi_device->efx = efx;
2715         } else {
2716                 spi_device = NULL;
2717         }
2718
2719         kfree(*spi_device_ret);
2720         *spi_device_ret = spi_device;
2721         return 0;
2722 }
2723
2724
2725 static void falcon_remove_spi_devices(struct efx_nic *efx)
2726 {
2727         kfree(efx->spi_eeprom);
2728         efx->spi_eeprom = NULL;
2729         kfree(efx->spi_flash);
2730         efx->spi_flash = NULL;
2731 }
2732
2733 /* Extract non-volatile configuration */
2734 static int falcon_probe_nvconfig(struct efx_nic *efx)
2735 {
2736         struct falcon_nvconfig *nvconfig;
2737         int board_rev;
2738         int rc;
2739
2740         nvconfig = kmalloc(sizeof(*nvconfig), GFP_KERNEL);
2741         if (!nvconfig)
2742                 return -ENOMEM;
2743
2744         rc = falcon_read_nvram(efx, nvconfig);
2745         if (rc == -EINVAL) {
2746                 EFX_ERR(efx, "NVRAM is invalid therefore using defaults\n");
2747                 efx->phy_type = PHY_TYPE_NONE;
2748                 efx->mdio.prtad = MDIO_PRTAD_NONE;
2749                 board_rev = 0;
2750                 rc = 0;
2751         } else if (rc) {
2752                 goto fail1;
2753         } else {
2754                 struct falcon_nvconfig_board_v2 *v2 = &nvconfig->board_v2;
2755                 struct falcon_nvconfig_board_v3 *v3 = &nvconfig->board_v3;
2756
2757                 efx->phy_type = v2->port0_phy_type;
2758                 efx->mdio.prtad = v2->port0_phy_addr;
2759                 board_rev = le16_to_cpu(v2->board_revision);
2760
2761                 if (le16_to_cpu(nvconfig->board_struct_ver) >= 3) {
2762                         rc = falcon_spi_device_init(
2763                                 efx, &efx->spi_flash, FFE_AB_SPI_DEVICE_FLASH,
2764                                 le32_to_cpu(v3->spi_device_type
2765                                             [FFE_AB_SPI_DEVICE_FLASH]));
2766                         if (rc)
2767                                 goto fail2;
2768                         rc = falcon_spi_device_init(
2769                                 efx, &efx->spi_eeprom, FFE_AB_SPI_DEVICE_EEPROM,
2770                                 le32_to_cpu(v3->spi_device_type
2771                                             [FFE_AB_SPI_DEVICE_EEPROM]));
2772                         if (rc)
2773                                 goto fail2;
2774                 }
2775         }
2776
2777         /* Read the MAC addresses */
2778         memcpy(efx->mac_address, nvconfig->mac_address[0], ETH_ALEN);
2779
2780         EFX_LOG(efx, "PHY is %d phy_id %d\n", efx->phy_type, efx->mdio.prtad);
2781
2782         falcon_probe_board(efx, board_rev);
2783
2784         kfree(nvconfig);
2785         return 0;
2786
2787  fail2:
2788         falcon_remove_spi_devices(efx);
2789  fail1:
2790         kfree(nvconfig);
2791         return rc;
2792 }
2793
2794 u32 efx_nic_fpga_ver(struct efx_nic *efx)
2795 {
2796         efx_oword_t altera_build;
2797
2798         efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
2799         return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
2800 }
2801
2802 /* Probe all SPI devices on the NIC */
2803 static void falcon_probe_spi_devices(struct efx_nic *efx)
2804 {
2805         efx_oword_t nic_stat, gpio_ctl, ee_vpd_cfg;
2806         int boot_dev;
2807
2808         efx_reado(efx, &gpio_ctl, FR_AB_GPIO_CTL);
2809         efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2810         efx_reado(efx, &ee_vpd_cfg, FR_AB_EE_VPD_CFG0);
2811
2812         if (EFX_OWORD_FIELD(gpio_ctl, FRF_AB_GPIO3_PWRUP_VALUE)) {
2813                 boot_dev = (EFX_OWORD_FIELD(nic_stat, FRF_AB_SF_PRST) ?
2814                             FFE_AB_SPI_DEVICE_FLASH : FFE_AB_SPI_DEVICE_EEPROM);
2815                 EFX_LOG(efx, "Booted from %s\n",
2816                         boot_dev == FFE_AB_SPI_DEVICE_FLASH ? "flash" : "EEPROM");
2817         } else {
2818                 /* Disable VPD and set clock dividers to safe
2819                  * values for initial programming. */
2820                 boot_dev = -1;
2821                 EFX_LOG(efx, "Booted from internal ASIC settings;"
2822                         " setting SPI config\n");
2823                 EFX_POPULATE_OWORD_3(ee_vpd_cfg, FRF_AB_EE_VPD_EN, 0,
2824                                      /* 125 MHz / 7 ~= 20 MHz */
2825                                      FRF_AB_EE_SF_CLOCK_DIV, 7,
2826                                      /* 125 MHz / 63 ~= 2 MHz */
2827                                      FRF_AB_EE_EE_CLOCK_DIV, 63);
2828                 efx_writeo(efx, &ee_vpd_cfg, FR_AB_EE_VPD_CFG0);
2829         }
2830
2831         if (boot_dev == FFE_AB_SPI_DEVICE_FLASH)
2832                 falcon_spi_device_init(efx, &efx->spi_flash,
2833                                        FFE_AB_SPI_DEVICE_FLASH,
2834                                        default_flash_type);
2835         if (boot_dev == FFE_AB_SPI_DEVICE_EEPROM)
2836                 falcon_spi_device_init(efx, &efx->spi_eeprom,
2837                                        FFE_AB_SPI_DEVICE_EEPROM,
2838                                        large_eeprom_type);
2839 }
2840
2841 static int falcon_probe_nic(struct efx_nic *efx)
2842 {
2843         struct falcon_nic_data *nic_data;
2844         struct falcon_board *board;
2845         int rc;
2846
2847         /* Allocate storage for hardware specific data */
2848         nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);
2849         if (!nic_data)
2850                 return -ENOMEM;
2851         efx->nic_data = nic_data;
2852
2853         rc = -ENODEV;
2854
2855         if (efx_nic_fpga_ver(efx) != 0) {
2856                 EFX_ERR(efx, "Falcon FPGA not supported\n");
2857                 goto fail1;
2858         }
2859
2860         if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1) {
2861                 efx_oword_t nic_stat;
2862                 struct pci_dev *dev;
2863                 u8 pci_rev = efx->pci_dev->revision;
2864
2865                 if ((pci_rev == 0xff) || (pci_rev == 0)) {
2866                         EFX_ERR(efx, "Falcon rev A0 not supported\n");
2867                         goto fail1;
2868                 }
2869                 efx_reado(efx, &nic_stat, FR_AB_NIC_STAT);
2870                 if (EFX_OWORD_FIELD(nic_stat, FRF_AB_STRAP_10G) == 0) {
2871                         EFX_ERR(efx, "Falcon rev A1 1G not supported\n");
2872                         goto fail1;
2873                 }
2874                 if (EFX_OWORD_FIELD(nic_stat, FRF_AA_STRAP_PCIE) == 0) {
2875                         EFX_ERR(efx, "Falcon rev A1 PCI-X not supported\n");
2876                         goto fail1;
2877                 }
2878
2879                 dev = pci_dev_get(efx->pci_dev);
2880                 while ((dev = pci_get_device(EFX_VENDID_SFC, FALCON_A_S_DEVID,
2881                                              dev))) {
2882                         if (dev->bus == efx->pci_dev->bus &&
2883                             dev->devfn == efx->pci_dev->devfn + 1) {
2884                                 nic_data->pci_dev2 = dev;
2885                                 break;
2886                         }
2887                 }
2888                 if (!nic_data->pci_dev2) {
2889                         EFX_ERR(efx, "failed to find secondary function\n");
2890                         rc = -ENODEV;
2891                         goto fail2;
2892                 }
2893         }
2894
2895         /* Now we can reset the NIC */
2896         rc = falcon_reset_hw(efx, RESET_TYPE_ALL);
2897         if (rc) {
2898                 EFX_ERR(efx, "failed to reset NIC\n");
2899                 goto fail3;
2900         }
2901
2902         /* Allocate memory for INT_KER */
2903         rc = efx_nic_alloc_buffer(efx, &efx->irq_status, sizeof(efx_oword_t));
2904         if (rc)
2905                 goto fail4;
2906         BUG_ON(efx->irq_status.dma_addr & 0x0f);
2907
2908         EFX_LOG(efx, "INT_KER at %llx (virt %p phys %llx)\n",
2909                 (u64)efx->irq_status.dma_addr,
2910                 efx->irq_status.addr, (u64)virt_to_phys(efx->irq_status.addr));
2911
2912         falcon_probe_spi_devices(efx);
2913
2914         /* Read in the non-volatile configuration */
2915         rc = falcon_probe_nvconfig(efx);
2916         if (rc)
2917                 goto fail5;
2918
2919         /* Initialise I2C adapter */
2920         board = falcon_board(efx);
2921         board->i2c_adap.owner = THIS_MODULE;
2922         board->i2c_data = falcon_i2c_bit_operations;
2923         board->i2c_data.data = efx;
2924         board->i2c_adap.algo_data = &board->i2c_data;
2925         board->i2c_adap.dev.parent = &efx->pci_dev->dev;
2926         strlcpy(board->i2c_adap.name, "SFC4000 GPIO",
2927                 sizeof(board->i2c_adap.name));
2928         rc = i2c_bit_add_bus(&board->i2c_adap);
2929         if (rc)
2930                 goto fail5;
2931
2932         rc = falcon_board(efx)->type->init(efx);
2933         if (rc) {
2934                 EFX_ERR(efx, "failed to initialise board\n");
2935                 goto fail6;
2936         }
2937
2938         nic_data->stats_disable_count = 1;
2939         setup_timer(&nic_data->stats_timer, &falcon_stats_timer_func,
2940                     (unsigned long)efx);
2941
2942         return 0;
2943
2944  fail6:
2945         BUG_ON(i2c_del_adapter(&board->i2c_adap));
2946         memset(&board->i2c_adap, 0, sizeof(board->i2c_adap));
2947  fail5:
2948         falcon_remove_spi_devices(efx);
2949         efx_nic_free_buffer(efx, &efx->irq_status);
2950  fail4:
2951  fail3:
2952         if (nic_data->pci_dev2) {
2953                 pci_dev_put(nic_data->pci_dev2);
2954                 nic_data->pci_dev2 = NULL;
2955         }
2956  fail2:
2957  fail1:
2958         kfree(efx->nic_data);
2959         return rc;
2960 }
2961
2962 static void falcon_init_rx_cfg(struct efx_nic *efx)
2963 {
2964         /* Prior to Siena the RX DMA engine will split each frame at
2965          * intervals of RX_USR_BUF_SIZE (32-byte units). We set it to
2966          * be so large that that never happens. */
2967         const unsigned huge_buf_size = (3 * 4096) >> 5;
2968         /* RX control FIFO thresholds (32 entries) */
2969         const unsigned ctrl_xon_thr = 20;
2970         const unsigned ctrl_xoff_thr = 25;
2971         /* RX data FIFO thresholds (256-byte units; size varies) */
2972         int data_xon_thr = efx_nic_rx_xon_thresh >> 8;
2973         int data_xoff_thr = efx_nic_rx_xoff_thresh >> 8;
2974         efx_oword_t reg;
2975
2976         efx_reado(efx, &reg, FR_AZ_RX_CFG);
2977         if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1) {
2978                 /* Data FIFO size is 5.5K */
2979                 if (data_xon_thr < 0)
2980                         data_xon_thr = 512 >> 8;
2981                 if (data_xoff_thr < 0)
2982                         data_xoff_thr = 2048 >> 8;
2983                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_DESC_PUSH_EN, 0);
2984                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_USR_BUF_SIZE,
2985                                     huge_buf_size);
2986                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_MAC_TH, data_xon_thr);
2987                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_MAC_TH, data_xoff_thr);
2988                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_TX_TH, ctrl_xon_thr);
2989                 EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_TX_TH, ctrl_xoff_thr);
2990         } else {
2991                 /* Data FIFO size is 80K; register fields moved */
2992                 if (data_xon_thr < 0)
2993                         data_xon_thr = 27648 >> 8; /* ~3*max MTU */
2994                 if (data_xoff_thr < 0)
2995                         data_xoff_thr = 54272 >> 8; /* ~80Kb - 3*max MTU */
2996                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_DESC_PUSH_EN, 0);
2997                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_USR_BUF_SIZE,
2998                                     huge_buf_size);
2999                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_MAC_TH, data_xon_thr);
3000                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_MAC_TH, data_xoff_thr);
3001                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_TX_TH, ctrl_xon_thr);
3002                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_TX_TH, ctrl_xoff_thr);
3003                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 1);
3004         }
3005         /* Always enable XOFF signal from RX FIFO.  We enable
3006          * or disable transmission of pause frames at the MAC. */
3007         EFX_SET_OWORD_FIELD(reg, FRF_AZ_RX_XOFF_MAC_EN, 1);
3008         efx_writeo(efx, &reg, FR_AZ_RX_CFG);
3009 }
3010
3011 void efx_nic_init_common(struct efx_nic *efx)
3012 {
3013         efx_oword_t temp;
3014
3015         /* Set positions of descriptor caches in SRAM. */
3016         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR,
3017                              efx->type->tx_dc_base / 8);
3018         efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
3019         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR,
3020                              efx->type->rx_dc_base / 8);
3021         efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
3022
3023         /* Set TX descriptor cache size. */
3024         BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
3025         EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
3026         efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
3027
3028         /* Set RX descriptor cache size.  Set low watermark to size-8, as
3029          * this allows most efficient prefetching.
3030          */
3031         BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
3032         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
3033         efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
3034         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
3035         efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
3036
3037         /* Program INT_KER address */
3038         EFX_POPULATE_OWORD_2(temp,
3039                              FRF_AZ_NORM_INT_VEC_DIS_KER,
3040                              EFX_INT_MODE_USE_MSI(efx),
3041                              FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
3042         efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
3043
3044         /* Enable all the genuinely fatal interrupts.  (They are still
3045          * masked by the overall interrupt mask, controlled by
3046          * falcon_interrupts()).
3047          *
3048          * Note: All other fatal interrupts are enabled
3049          */
3050         EFX_POPULATE_OWORD_3(temp,
3051                              FRF_AZ_ILL_ADR_INT_KER_EN, 1,
3052                              FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
3053                              FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
3054         EFX_INVERT_OWORD(temp);
3055         efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
3056
3057         efx_setup_rss_indir_table(efx);
3058
3059         /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
3060          * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
3061          */
3062         efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
3063         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
3064         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
3065         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
3066         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 0);
3067         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
3068         /* Enable SW_EV to inherit in char driver - assume harmless here */
3069         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
3070         /* Prefetch threshold 2 => fetch when descriptor cache half empty */
3071         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
3072         /* Squash TX of packets of 16 bytes or less */
3073         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
3074                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
3075         efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
3076 }
3077
3078 /* This call performs hardware-specific global initialisation, such as
3079  * defining the descriptor cache sizes and number of RSS channels.
3080  * It does not set up any buffers, descriptor rings or event queues.
3081  */
3082 static int falcon_init_nic(struct efx_nic *efx)
3083 {
3084         efx_oword_t temp;
3085         int rc;
3086
3087         /* Use on-chip SRAM */
3088         efx_reado(efx, &temp, FR_AB_NIC_STAT);
3089         EFX_SET_OWORD_FIELD(temp, FRF_AB_ONCHIP_SRAM, 1);
3090         efx_writeo(efx, &temp, FR_AB_NIC_STAT);
3091
3092         /* Set the source of the GMAC clock */
3093         if (efx_nic_rev(efx) == EFX_REV_FALCON_B0) {
3094                 efx_reado(efx, &temp, FR_AB_GPIO_CTL);
3095                 EFX_SET_OWORD_FIELD(temp, FRF_AB_USE_NIC_CLK, true);
3096                 efx_writeo(efx, &temp, FR_AB_GPIO_CTL);
3097         }
3098
3099         /* Select the correct MAC */
3100         falcon_clock_mac(efx);
3101
3102         rc = falcon_reset_sram(efx);
3103         if (rc)
3104                 return rc;
3105
3106         /* Clear the parity enables on the TX data fifos as
3107          * they produce false parity errors because of timing issues
3108          */
3109         if (EFX_WORKAROUND_5129(efx)) {
3110                 efx_reado(efx, &temp, FR_AZ_CSR_SPARE);
3111                 EFX_SET_OWORD_FIELD(temp, FRF_AB_MEM_PERR_EN_TX_DATA, 0);
3112                 efx_writeo(efx, &temp, FR_AZ_CSR_SPARE);
3113         }
3114
3115         if (EFX_WORKAROUND_7244(efx)) {
3116                 efx_reado(efx, &temp, FR_BZ_RX_FILTER_CTL);
3117                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_UDP_FULL_SRCH_LIMIT, 8);
3118                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_UDP_WILD_SRCH_LIMIT, 8);
3119                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TCP_FULL_SRCH_LIMIT, 8);
3120                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TCP_WILD_SRCH_LIMIT, 8);
3121                 efx_writeo(efx, &temp, FR_BZ_RX_FILTER_CTL);
3122         }
3123
3124         /* XXX This is documented only for Falcon A0/A1 */
3125         /* Setup RX.  Wait for descriptor is broken and must
3126          * be disabled.  RXDP recovery shouldn't be needed, but is.
3127          */
3128         efx_reado(efx, &temp, FR_AA_RX_SELF_RST);
3129         EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_NODESC_WAIT_DIS, 1);
3130         EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_SELF_RST_EN, 1);
3131         if (EFX_WORKAROUND_5583(efx))
3132                 EFX_SET_OWORD_FIELD(temp, FRF_AA_RX_ISCSI_DIS, 1);
3133         efx_writeo(efx, &temp, FR_AA_RX_SELF_RST);
3134
3135         /* Do not enable TX_NO_EOP_DISC_EN, since it limits packets to 16
3136          * descriptors (which is bad).
3137          */
3138         efx_reado(efx, &temp, FR_AZ_TX_CFG);
3139         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
3140         efx_writeo(efx, &temp, FR_AZ_TX_CFG);
3141
3142         falcon_init_rx_cfg(efx);
3143
3144         /* Set destination of both TX and RX Flush events */
3145         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
3146                 EFX_POPULATE_OWORD_1(temp, FRF_BZ_FLS_EVQ_ID, 0);
3147                 efx_writeo(efx, &temp, FR_BZ_DP_CTRL);
3148         }
3149
3150         efx_nic_init_common(efx);
3151
3152         return 0;
3153 }
3154
3155 static void falcon_remove_nic(struct efx_nic *efx)
3156 {
3157         struct falcon_nic_data *nic_data = efx->nic_data;
3158         struct falcon_board *board = falcon_board(efx);
3159         int rc;
3160
3161         board->type->fini(efx);
3162
3163         /* Remove I2C adapter and clear it in preparation for a retry */
3164         rc = i2c_del_adapter(&board->i2c_adap);
3165         BUG_ON(rc);
3166         memset(&board->i2c_adap, 0, sizeof(board->i2c_adap));
3167
3168         falcon_remove_spi_devices(efx);
3169         efx_nic_free_buffer(efx, &efx->irq_status);
3170
3171         falcon_reset_hw(efx, RESET_TYPE_ALL);
3172
3173         /* Release the second function after the reset */
3174         if (nic_data->pci_dev2) {
3175                 pci_dev_put(nic_data->pci_dev2);
3176                 nic_data->pci_dev2 = NULL;
3177         }
3178
3179         /* Tear down the private nic state */
3180         kfree(efx->nic_data);
3181         efx->nic_data = NULL;
3182 }
3183
3184 static void falcon_update_nic_stats(struct efx_nic *efx)
3185 {
3186         struct falcon_nic_data *nic_data = efx->nic_data;
3187         efx_oword_t cnt;
3188
3189         if (nic_data->stats_disable_count)
3190                 return;
3191
3192         efx_reado(efx, &cnt, FR_AZ_RX_NODESC_DROP);
3193         efx->n_rx_nodesc_drop_cnt +=
3194                 EFX_OWORD_FIELD(cnt, FRF_AB_RX_NODESC_DROP_CNT);
3195
3196         if (nic_data->stats_pending &&
3197             *nic_data->stats_dma_done == FALCON_STATS_DONE) {
3198                 nic_data->stats_pending = false;
3199                 rmb(); /* read the done flag before the stats */
3200                 efx->mac_op->update_stats(efx);
3201         }
3202 }
3203
3204 void falcon_start_nic_stats(struct efx_nic *efx)
3205 {
3206         struct falcon_nic_data *nic_data = efx->nic_data;
3207
3208         spin_lock_bh(&efx->stats_lock);
3209         if (--nic_data->stats_disable_count == 0)
3210                 falcon_stats_request(efx);
3211         spin_unlock_bh(&efx->stats_lock);
3212 }
3213
3214 void falcon_stop_nic_stats(struct efx_nic *efx)
3215 {
3216         struct falcon_nic_data *nic_data = efx->nic_data;
3217         int i;
3218
3219         might_sleep();
3220
3221         spin_lock_bh(&efx->stats_lock);
3222         ++nic_data->stats_disable_count;
3223         spin_unlock_bh(&efx->stats_lock);
3224
3225         del_timer_sync(&nic_data->stats_timer);
3226
3227         /* Wait enough time for the most recent transfer to
3228          * complete. */
3229         for (i = 0; i < 4 && nic_data->stats_pending; i++) {
3230                 if (*nic_data->stats_dma_done == FALCON_STATS_DONE)
3231                         break;
3232                 msleep(1);
3233         }
3234
3235         spin_lock_bh(&efx->stats_lock);
3236         falcon_stats_complete(efx);
3237         spin_unlock_bh(&efx->stats_lock);
3238 }
3239
3240 static void falcon_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
3241 {
3242         falcon_board(efx)->type->set_id_led(efx, mode);
3243 }
3244
3245 /**************************************************************************
3246  *
3247  * Wake on LAN
3248  *
3249  **************************************************************************
3250  */
3251
3252 static void falcon_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol)
3253 {
3254         wol->supported = 0;
3255         wol->wolopts = 0;
3256         memset(&wol->sopass, 0, sizeof(wol->sopass));
3257 }
3258
3259 static int falcon_set_wol(struct efx_nic *efx, u32 type)
3260 {
3261         if (type != 0)
3262                 return -EINVAL;
3263         return 0;
3264 }
3265
3266 /**************************************************************************
3267  *
3268  * Revision-dependent attributes used by efx.c
3269  *
3270  **************************************************************************
3271  */
3272
3273 struct efx_nic_type falcon_a1_nic_type = {
3274         .probe = falcon_probe_nic,
3275         .remove = falcon_remove_nic,
3276         .init = falcon_init_nic,
3277         .fini = efx_port_dummy_op_void,
3278         .monitor = falcon_monitor,
3279         .reset = falcon_reset_hw,
3280         .probe_port = falcon_probe_port,
3281         .remove_port = falcon_remove_port,
3282         .prepare_flush = falcon_prepare_flush,
3283         .update_stats = falcon_update_nic_stats,
3284         .start_stats = falcon_start_nic_stats,
3285         .stop_stats = falcon_stop_nic_stats,
3286         .set_id_led = falcon_set_id_led,
3287         .push_irq_moderation = falcon_push_irq_moderation,
3288         .push_multicast_hash = falcon_push_multicast_hash,
3289         .reconfigure_port = falcon_reconfigure_port,
3290         .get_wol = falcon_get_wol,
3291         .set_wol = falcon_set_wol,
3292         .resume_wol = efx_port_dummy_op_void,
3293         .test_nvram = falcon_test_nvram,
3294         .default_mac_ops = &falcon_xmac_operations,
3295
3296         .revision = EFX_REV_FALCON_A1,
3297         .mem_map_size = 0x20000,
3298         .txd_ptr_tbl_base = FR_AA_TX_DESC_PTR_TBL_KER,
3299         .rxd_ptr_tbl_base = FR_AA_RX_DESC_PTR_TBL_KER,
3300         .buf_tbl_base = FR_AA_BUF_FULL_TBL_KER,
3301         .evq_ptr_tbl_base = FR_AA_EVQ_PTR_TBL_KER,
3302         .evq_rptr_tbl_base = FR_AA_EVQ_RPTR_KER,
3303         .max_dma_mask = DMA_BIT_MASK(FSF_AZ_TX_KER_BUF_ADDR_WIDTH),
3304         .rx_buffer_padding = 0x24,
3305         .max_interrupt_mode = EFX_INT_MODE_MSI,
3306         .phys_addr_channels = 4,
3307         .tx_dc_base = 0x130000,
3308         .rx_dc_base = 0x100000,
3309         .reset_world_flags = ETH_RESET_IRQ,
3310 };
3311
3312 struct efx_nic_type falcon_b0_nic_type = {
3313         .probe = falcon_probe_nic,
3314         .remove = falcon_remove_nic,
3315         .init = falcon_init_nic,
3316         .fini = efx_port_dummy_op_void,
3317         .monitor = falcon_monitor,
3318         .reset = falcon_reset_hw,
3319         .probe_port = falcon_probe_port,
3320         .remove_port = falcon_remove_port,
3321         .prepare_flush = falcon_prepare_flush,
3322         .update_stats = falcon_update_nic_stats,
3323         .start_stats = falcon_start_nic_stats,
3324         .stop_stats = falcon_stop_nic_stats,
3325         .set_id_led = falcon_set_id_led,
3326         .push_irq_moderation = falcon_push_irq_moderation,
3327         .push_multicast_hash = falcon_push_multicast_hash,
3328         .reconfigure_port = falcon_reconfigure_port,
3329         .get_wol = falcon_get_wol,
3330         .set_wol = falcon_set_wol,
3331         .resume_wol = efx_port_dummy_op_void,
3332         .test_registers = falcon_b0_test_registers,
3333         .test_nvram = falcon_test_nvram,
3334         .default_mac_ops = &falcon_xmac_operations,
3335
3336         .revision = EFX_REV_FALCON_B0,
3337         /* Map everything up to and including the RSS indirection
3338          * table.  Don't map MSI-X table, MSI-X PBA since Linux
3339          * requires that they not be mapped.  */
3340         .mem_map_size = (FR_BZ_RX_INDIRECTION_TBL +
3341                          FR_BZ_RX_INDIRECTION_TBL_STEP *
3342                          FR_BZ_RX_INDIRECTION_TBL_ROWS),
3343         .txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL,
3344         .rxd_ptr_tbl_base = FR_BZ_RX_DESC_PTR_TBL,
3345         .buf_tbl_base = FR_BZ_BUF_FULL_TBL,
3346         .evq_ptr_tbl_base = FR_BZ_EVQ_PTR_TBL,
3347         .evq_rptr_tbl_base = FR_BZ_EVQ_RPTR,
3348         .max_dma_mask = DMA_BIT_MASK(FSF_AZ_TX_KER_BUF_ADDR_WIDTH),
3349         .rx_buffer_padding = 0,
3350         .max_interrupt_mode = EFX_INT_MODE_MSIX,
3351         .phys_addr_channels = 32, /* Hardware limit is 64, but the legacy
3352                                    * interrupt handler only supports 32
3353                                    * channels */
3354         .tx_dc_base = 0x130000,
3355         .rx_dc_base = 0x100000,
3356         .reset_world_flags = ETH_RESET_IRQ,
3357 };
3358