]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/net/bnx2x/bnx2x_cmn.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-davem' of git://git.kernel.org/pub/scm/linux/kernel/git/linville...
[net-next-2.6.git] / drivers / net / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2010 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18
19#include <linux/etherdevice.h>
20#include <linux/ip.h>
21#include <linux/ipv6.h>
7f3e01fe 22#include <net/ip6_checksum.h>
6891dd25 23#include <linux/firmware.h>
9f6c9258
DK		 24#include "bnx2x_cmn.h"
25
26#ifdef BCM_VLAN
27#include <linux/if_vlan.h>
28#endif
29
30static int bnx2x_poll(struct napi_struct *napi, int budget);
31
32/* free skb in the packet ring at pos idx
33 * return idx of last bd freed
34 */
35static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
36 u16 idx)
37{
38 struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
39 struct eth_tx_start_bd *tx_start_bd;
40 struct eth_tx_bd *tx_data_bd;
41 struct sk_buff *skb = tx_buf->skb;
42 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
43 int nbd;
44
45 /* prefetch skb end pointer to speedup dev_kfree_skb() */
46 prefetch(&skb->end);
47
48 DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n",
49 idx, tx_buf, skb);
50
51 /* unmap first bd */
52 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
53 tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
54 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
55 BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
56
57 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
58#ifdef BNX2X_STOP_ON_ERROR
59 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
60 BNX2X_ERR("BAD nbd!\n");
61 bnx2x_panic();
62 }
63#endif
64 new_cons = nbd + tx_buf->first_bd;
65
66 /* Get the next bd */
67 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
68
69 /* Skip a parse bd... */
70 --nbd;
71 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
72
73 /* ...and the TSO split header bd since they have no mapping */
74 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
75 --nbd;
76 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
77 }
78
79 /* now free frags */
80 while (nbd > 0) {
81
82 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
83 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
84 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
85 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
86 if (--nbd)
87 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
88 }
89
90 /* release skb */
91 WARN_ON(!skb);
92 dev_kfree_skb(skb);
93 tx_buf->first_bd = 0;
94 tx_buf->skb = NULL;
95
96 return new_cons;
97}
98
99int bnx2x_tx_int(struct bnx2x_fastpath *fp)
100{
101 struct bnx2x *bp = fp->bp;
102 struct netdev_queue *txq;
103 u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
104
105#ifdef BNX2X_STOP_ON_ERROR
106 if (unlikely(bp->panic))
107 return -1;
108#endif
109
110 txq = netdev_get_tx_queue(bp->dev, fp->index);
111 hw_cons = le16_to_cpu(*fp->tx_cons_sb);
112 sw_cons = fp->tx_pkt_cons;
113
114 while (sw_cons != hw_cons) {
115 u16 pkt_cons;
116
117 pkt_cons = TX_BD(sw_cons);
118
119 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
120
121 DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n",
122 hw_cons, sw_cons, pkt_cons);
123
124/* if (NEXT_TX_IDX(sw_cons) != hw_cons) {
125 rmb();
126 prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
127 }
128*/
129 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
130 sw_cons++;
131 }
132
133 fp->tx_pkt_cons = sw_cons;
134 fp->tx_bd_cons = bd_cons;
135
136 /* Need to make the tx_bd_cons update visible to start_xmit()
137 * before checking for netif_tx_queue_stopped(). Without the
138 * memory barrier, there is a small possibility that
139 * start_xmit() will miss it and cause the queue to be stopped
140 * forever.
141 */
142 smp_mb();
143
144 /* TBD need a thresh? */
145 if (unlikely(netif_tx_queue_stopped(txq))) {
146 /* Taking tx_lock() is needed to prevent reenabling the queue
147 * while it's empty. This could have happen if rx_action() gets
148 * suspended in bnx2x_tx_int() after the condition before
149 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
150 *
151 * stops the queue->sees fresh tx_bd_cons->releases the queue->
152 * sends some packets consuming the whole queue again->
153 * stops the queue
154 */
155
156 __netif_tx_lock(txq, smp_processor_id());
157
158 if ((netif_tx_queue_stopped(txq)) &&
159 (bp->state == BNX2X_STATE_OPEN) &&
160 (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
161 netif_tx_wake_queue(txq);
162
163 __netif_tx_unlock(txq);
164 }
165 return 0;
166}
167
168static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
169 u16 idx)
170{
171 u16 last_max = fp->last_max_sge;
172
173 if (SUB_S16(idx, last_max) > 0)
174 fp->last_max_sge = idx;
175}
176
177static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
178 struct eth_fast_path_rx_cqe *fp_cqe)
179{
180 struct bnx2x *bp = fp->bp;
181 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
182 le16_to_cpu(fp_cqe->len_on_bd)) >>
183 SGE_PAGE_SHIFT;
184 u16 last_max, last_elem, first_elem;
185 u16 delta = 0;
186 u16 i;
187
188 if (!sge_len)
189 return;
190
191 /* First mark all used pages */
192 for (i = 0; i < sge_len; i++)
193 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
194
195 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
196 sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
197
198 /* Here we assume that the last SGE index is the biggest */
199 prefetch((void *)(fp->sge_mask));
200 bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
201
202 last_max = RX_SGE(fp->last_max_sge);
203 last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
204 first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
205
206 /* If ring is not full */
207 if (last_elem + 1 != first_elem)
208 last_elem++;
209
210 /* Now update the prod */
211 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
212 if (likely(fp->sge_mask[i]))
213 break;
214
215 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
216 delta += RX_SGE_MASK_ELEM_SZ;
217 }
218
219 if (delta > 0) {
220 fp->rx_sge_prod += delta;
221 /* clear page-end entries */
222 bnx2x_clear_sge_mask_next_elems(fp);
223 }
224
225 DP(NETIF_MSG_RX_STATUS,
226 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
227 fp->last_max_sge, fp->rx_sge_prod);
228}
229
230static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
231 struct sk_buff *skb, u16 cons, u16 prod)
232{
233 struct bnx2x *bp = fp->bp;
234 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
235 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
236 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
237 dma_addr_t mapping;
238
239 /* move empty skb from pool to prod and map it */
240 prod_rx_buf->skb = fp->tpa_pool[queue].skb;
241 mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
242 bp->rx_buf_size, DMA_FROM_DEVICE);
243 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
244
245 /* move partial skb from cons to pool (don't unmap yet) */
246 fp->tpa_pool[queue] = *cons_rx_buf;
247
248 /* mark bin state as start - print error if current state != stop */
249 if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
250 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
251
252 fp->tpa_state[queue] = BNX2X_TPA_START;
253
254 /* point prod_bd to new skb */
255 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
256 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
257
258#ifdef BNX2X_STOP_ON_ERROR
259 fp->tpa_queue_used |= (1 << queue);
260#ifdef _ASM_GENERIC_INT_L64_H
261 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
262#else
263 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
264#endif
265 fp->tpa_queue_used);
266#endif
267}
268
269static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
270 struct sk_buff *skb,
271 struct eth_fast_path_rx_cqe *fp_cqe,
272 u16 cqe_idx)
273{
274 struct sw_rx_page *rx_pg, old_rx_pg;
275 u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
276 u32 i, frag_len, frag_size, pages;
277 int err;
278 int j;
279
280 frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
281 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
282
283 /* This is needed in order to enable forwarding support */
284 if (frag_size)
285 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
286 max(frag_size, (u32)len_on_bd));
287
288#ifdef BNX2X_STOP_ON_ERROR
289 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
290 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
291 pages, cqe_idx);
292 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n",
293 fp_cqe->pkt_len, len_on_bd);
294 bnx2x_panic();
295 return -EINVAL;
296 }
297#endif
298
299 /* Run through the SGL and compose the fragmented skb */
300 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
301 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
302
303 /* FW gives the indices of the SGE as if the ring is an array
304 (meaning that "next" element will consume 2 indices) */
305 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
306 rx_pg = &fp->rx_page_ring[sge_idx];
307 old_rx_pg = *rx_pg;
308
309 /* If we fail to allocate a substitute page, we simply stop
310 where we are and drop the whole packet */
311 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
312 if (unlikely(err)) {
313 fp->eth_q_stats.rx_skb_alloc_failed++;
314 return err;
315 }
316
317 /* Unmap the page as we r going to pass it to the stack */
318 dma_unmap_page(&bp->pdev->dev,
319 dma_unmap_addr(&old_rx_pg, mapping),
320 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
321
322 /* Add one frag and update the appropriate fields in the skb */
323 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
324
325 skb->data_len += frag_len;
326 skb->truesize += frag_len;
327 skb->len += frag_len;
328
329 frag_size -= frag_len;
330 }
331
332 return 0;
333}
334
335static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
336 u16 queue, int pad, int len, union eth_rx_cqe *cqe,
337 u16 cqe_idx)
338{
339 struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
340 struct sk_buff *skb = rx_buf->skb;
341 /* alloc new skb */
342 struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
343
344 /* Unmap skb in the pool anyway, as we are going to change
345 pool entry status to BNX2X_TPA_STOP even if new skb allocation
346 fails. */
347 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
348 bp->rx_buf_size, DMA_FROM_DEVICE);
349
350 if (likely(new_skb)) {
351 /* fix ip xsum and give it to the stack */
352 /* (no need to map the new skb) */
353#ifdef BCM_VLAN
354 int is_vlan_cqe =
355 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
356 PARSING_FLAGS_VLAN);
357 int is_not_hwaccel_vlan_cqe =
358 (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
359#endif
360
361 prefetch(skb);
362 prefetch(((char *)(skb)) + 128);
363
364#ifdef BNX2X_STOP_ON_ERROR
365 if (pad + len > bp->rx_buf_size) {
366 BNX2X_ERR("skb_put is about to fail... "
367 "pad %d len %d rx_buf_size %d\n",
368 pad, len, bp->rx_buf_size);
369 bnx2x_panic();
370 return;
371 }
372#endif
373
374 skb_reserve(skb, pad);
375 skb_put(skb, len);
376
377 skb->protocol = eth_type_trans(skb, bp->dev);
378 skb->ip_summed = CHECKSUM_UNNECESSARY;
379
380 {
381 struct iphdr *iph;
382
383 iph = (struct iphdr *)skb->data;
384#ifdef BCM_VLAN
385 /* If there is no Rx VLAN offloading -
386 take VLAN tag into an account */
387 if (unlikely(is_not_hwaccel_vlan_cqe))
388 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
389#endif
390 iph->check = 0;
391 iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
392 }
393
394 if (!bnx2x_fill_frag_skb(bp, fp, skb,
395 &cqe->fast_path_cqe, cqe_idx)) {
396#ifdef BCM_VLAN
397 if ((bp->vlgrp != NULL) && is_vlan_cqe &&
398 (!is_not_hwaccel_vlan_cqe))
399 vlan_gro_receive(&fp->napi, bp->vlgrp,
400 le16_to_cpu(cqe->fast_path_cqe.
401 vlan_tag), skb);
402 else
403#endif
404 napi_gro_receive(&fp->napi, skb);
405 } else {
406 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
407 " - dropping packet!\n");
408 dev_kfree_skb(skb);
409 }
410
411
412 /* put new skb in bin */
413 fp->tpa_pool[queue].skb = new_skb;
414
415 } else {
416 /* else drop the packet and keep the buffer in the bin */
417 DP(NETIF_MSG_RX_STATUS,
418 "Failed to allocate new skb - dropping packet!\n");
419 fp->eth_q_stats.rx_skb_alloc_failed++;
420 }
421
422 fp->tpa_state[queue] = BNX2X_TPA_STOP;
423}
424
425/* Set Toeplitz hash value in the skb using the value from the
426 * CQE (calculated by HW).
427 */
428static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
429 struct sk_buff *skb)
430{
431 /* Set Toeplitz hash from CQE */
432 if ((bp->dev->features & NETIF_F_RXHASH) &&
433 (cqe->fast_path_cqe.status_flags &
434 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
435 skb->rxhash =
436 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
437}
438
439int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
440{
441 struct bnx2x *bp = fp->bp;
442 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
443 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
444 int rx_pkt = 0;
445
446#ifdef BNX2X_STOP_ON_ERROR
447 if (unlikely(bp->panic))
448 return 0;
449#endif
450
451 /* CQ "next element" is of the size of the regular element,
452 that's why it's ok here */
453 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
454 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
455 hw_comp_cons++;
456
457 bd_cons = fp->rx_bd_cons;
458 bd_prod = fp->rx_bd_prod;
459 bd_prod_fw = bd_prod;
460 sw_comp_cons = fp->rx_comp_cons;
461 sw_comp_prod = fp->rx_comp_prod;
462
463 /* Memory barrier necessary as speculative reads of the rx
464 * buffer can be ahead of the index in the status block
465 */
466 rmb();
467
468 DP(NETIF_MSG_RX_STATUS,
469 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
470 fp->index, hw_comp_cons, sw_comp_cons);
471
472 while (sw_comp_cons != hw_comp_cons) {
473 struct sw_rx_bd *rx_buf = NULL;
474 struct sk_buff *skb;
475 union eth_rx_cqe *cqe;
476 u8 cqe_fp_flags;
477 u16 len, pad;
478
479 comp_ring_cons = RCQ_BD(sw_comp_cons);
480 bd_prod = RX_BD(bd_prod);
481 bd_cons = RX_BD(bd_cons);
482
483 /* Prefetch the page containing the BD descriptor
484 at producer's index. It will be needed when new skb is
485 allocated */
486 prefetch((void *)(PAGE_ALIGN((unsigned long)
487 (&fp->rx_desc_ring[bd_prod])) -
488 PAGE_SIZE + 1));
489
490 cqe = &fp->rx_comp_ring[comp_ring_cons];
491 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
492
493 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
494 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
495 cqe_fp_flags, cqe->fast_path_cqe.status_flags,
496 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
497 le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
498 le16_to_cpu(cqe->fast_path_cqe.pkt_len));
499
500 /* is this a slowpath msg? */
501 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
502 bnx2x_sp_event(fp, cqe);
503 goto next_cqe;
504
505 /* this is an rx packet */
506 } else {
507 rx_buf = &fp->rx_buf_ring[bd_cons];
508 skb = rx_buf->skb;
509 prefetch(skb);
510 len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
511 pad = cqe->fast_path_cqe.placement_offset;
512
513 /* If CQE is marked both TPA_START and TPA_END
514 it is a non-TPA CQE */
515 if ((!fp->disable_tpa) &&
516 (TPA_TYPE(cqe_fp_flags) !=
517 (TPA_TYPE_START | TPA_TYPE_END))) {
518 u16 queue = cqe->fast_path_cqe.queue_index;
519
520 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
521 DP(NETIF_MSG_RX_STATUS,
522 "calling tpa_start on queue %d\n",
523 queue);
524
525 bnx2x_tpa_start(fp, queue, skb,
526 bd_cons, bd_prod);
527
528 /* Set Toeplitz hash for an LRO skb */
529 bnx2x_set_skb_rxhash(bp, cqe, skb);
530
531 goto next_rx;
532 }
533
534 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
535 DP(NETIF_MSG_RX_STATUS,
536 "calling tpa_stop on queue %d\n",
537 queue);
538
539 if (!BNX2X_RX_SUM_FIX(cqe))
540 BNX2X_ERR("STOP on none TCP "
541 "data\n");
542
543 /* This is a size of the linear data
544 on this skb */
545 len = le16_to_cpu(cqe->fast_path_cqe.
546 len_on_bd);
547 bnx2x_tpa_stop(bp, fp, queue, pad,
548 len, cqe, comp_ring_cons);
549#ifdef BNX2X_STOP_ON_ERROR
550 if (bp->panic)
551 return 0;
552#endif
553
554 bnx2x_update_sge_prod(fp,
555 &cqe->fast_path_cqe);
556 goto next_cqe;
557 }
558 }
559
560 dma_sync_single_for_device(&bp->pdev->dev,
561 dma_unmap_addr(rx_buf, mapping),
562 pad + RX_COPY_THRESH,
563 DMA_FROM_DEVICE);
564 prefetch(((char *)(skb)) + 128);
565
566 /* is this an error packet? */
567 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
568 DP(NETIF_MSG_RX_ERR,
569 "ERROR flags %x rx packet %u\n",
570 cqe_fp_flags, sw_comp_cons);
571 fp->eth_q_stats.rx_err_discard_pkt++;
572 goto reuse_rx;
573 }
574
575 /* Since we don't have a jumbo ring
576 * copy small packets if mtu > 1500
577 */
578 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
579 (len <= RX_COPY_THRESH)) {
580 struct sk_buff *new_skb;
581
582 new_skb = netdev_alloc_skb(bp->dev,
583 len + pad);
584 if (new_skb == NULL) {
585 DP(NETIF_MSG_RX_ERR,
586 "ERROR packet dropped "
587 "because of alloc failure\n");
588 fp->eth_q_stats.rx_skb_alloc_failed++;
589 goto reuse_rx;
590 }
591
592 /* aligned copy */
593 skb_copy_from_linear_data_offset(skb, pad,
594 new_skb->data + pad, len);
595 skb_reserve(new_skb, pad);
596 skb_put(new_skb, len);
597
598 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
599
600 skb = new_skb;
601
602 } else
603 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
604 dma_unmap_single(&bp->pdev->dev,
605 dma_unmap_addr(rx_buf, mapping),
606 bp->rx_buf_size,
607 DMA_FROM_DEVICE);
608 skb_reserve(skb, pad);
609 skb_put(skb, len);
610
611 } else {
612 DP(NETIF_MSG_RX_ERR,
613 "ERROR packet dropped because "
614 "of alloc failure\n");
615 fp->eth_q_stats.rx_skb_alloc_failed++;
616reuse_rx:
617 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
618 goto next_rx;
619 }
620
621 skb->protocol = eth_type_trans(skb, bp->dev);
622
623 /* Set Toeplitz hash for a none-LRO skb */
624 bnx2x_set_skb_rxhash(bp, cqe, skb);
625
626 skb->ip_summed = CHECKSUM_NONE;
627 if (bp->rx_csum) {
628 if (likely(BNX2X_RX_CSUM_OK(cqe)))
629 skb->ip_summed = CHECKSUM_UNNECESSARY;
630 else
631 fp->eth_q_stats.hw_csum_err++;
632 }
633 }
634
635 skb_record_rx_queue(skb, fp->index);
636
637#ifdef BCM_VLAN
638 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
639 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
640 PARSING_FLAGS_VLAN))
641 vlan_gro_receive(&fp->napi, bp->vlgrp,
642 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
643 else
644#endif
645 napi_gro_receive(&fp->napi, skb);
646
647
648next_rx:
649 rx_buf->skb = NULL;
650
651 bd_cons = NEXT_RX_IDX(bd_cons);
652 bd_prod = NEXT_RX_IDX(bd_prod);
653 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
654 rx_pkt++;
655next_cqe:
656 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
657 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
658
659 if (rx_pkt == budget)
660 break;
661 } /* while */
662
663 fp->rx_bd_cons = bd_cons;
664 fp->rx_bd_prod = bd_prod_fw;
665 fp->rx_comp_cons = sw_comp_cons;
666 fp->rx_comp_prod = sw_comp_prod;
667
668 /* Update producers */
669 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
670 fp->rx_sge_prod);
671
672 fp->rx_pkt += rx_pkt;
673 fp->rx_calls++;
674
675 return rx_pkt;
676}
677
678static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
679{
680 struct bnx2x_fastpath *fp = fp_cookie;
681 struct bnx2x *bp = fp->bp;
682
683 /* Return here if interrupt is disabled */
684 if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
685 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
686 return IRQ_HANDLED;
687 }
688
689 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
690 fp->index, fp->sb_id);
691 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
692
693#ifdef BNX2X_STOP_ON_ERROR
694 if (unlikely(bp->panic))
695 return IRQ_HANDLED;
696#endif
697
698 /* Handle Rx and Tx according to MSI-X vector */
699 prefetch(fp->rx_cons_sb);
700 prefetch(fp->tx_cons_sb);
701 prefetch(&fp->status_blk->u_status_block.status_block_index);
702 prefetch(&fp->status_blk->c_status_block.status_block_index);
703 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
704
705 return IRQ_HANDLED;
706}
707
708
709/* HW Lock for shared dual port PHYs */
710void bnx2x_acquire_phy_lock(struct bnx2x *bp)
711{
712 mutex_lock(&bp->port.phy_mutex);
713
714 if (bp->port.need_hw_lock)
715 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
716}
717
718void bnx2x_release_phy_lock(struct bnx2x *bp)
719{
720 if (bp->port.need_hw_lock)
721 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
722
723 mutex_unlock(&bp->port.phy_mutex);
724}
725
726void bnx2x_link_report(struct bnx2x *bp)
727{
728 if (bp->flags & MF_FUNC_DIS) {
729 netif_carrier_off(bp->dev);
730 netdev_err(bp->dev, "NIC Link is Down\n");
731 return;
732 }
733
734 if (bp->link_vars.link_up) {
735 u16 line_speed;
736
737 if (bp->state == BNX2X_STATE_OPEN)
738 netif_carrier_on(bp->dev);
739 netdev_info(bp->dev, "NIC Link is Up, ");
740
741 line_speed = bp->link_vars.line_speed;
742 if (IS_E1HMF(bp)) {
743 u16 vn_max_rate;
744
745 vn_max_rate =
746 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
747 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
748 if (vn_max_rate < line_speed)
749 line_speed = vn_max_rate;
750 }
751 pr_cont("%d Mbps ", line_speed);
752
753 if (bp->link_vars.duplex == DUPLEX_FULL)
754 pr_cont("full duplex");
755 else
756 pr_cont("half duplex");
757
758 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
759 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
760 pr_cont(", receive ");
761 if (bp->link_vars.flow_ctrl &
762 BNX2X_FLOW_CTRL_TX)
763 pr_cont("& transmit ");
764 } else {
765 pr_cont(", transmit ");
766 }
767 pr_cont("flow control ON");
768 }
769 pr_cont("\n");
770
771 } else { /* link_down */
772 netif_carrier_off(bp->dev);
773 netdev_err(bp->dev, "NIC Link is Down\n");
774 }
775}
776
777void bnx2x_init_rx_rings(struct bnx2x *bp)
778{
779 int func = BP_FUNC(bp);
780 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
781 ETH_MAX_AGGREGATION_QUEUES_E1H;
782 u16 ring_prod, cqe_ring_prod;
783 int i, j;
784
785 bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
786 DP(NETIF_MSG_IFUP,
787 "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
788
789 if (bp->flags & TPA_ENABLE_FLAG) {
790
791 for_each_queue(bp, j) {
792 struct bnx2x_fastpath *fp = &bp->fp[j];
793
794 for (i = 0; i < max_agg_queues; i++) {
795 fp->tpa_pool[i].skb =
796 netdev_alloc_skb(bp->dev, bp->rx_buf_size);
797 if (!fp->tpa_pool[i].skb) {
798 BNX2X_ERR("Failed to allocate TPA "
799 "skb pool for queue[%d] - "
800 "disabling TPA on this "
801 "queue!\n", j);
802 bnx2x_free_tpa_pool(bp, fp, i);
803 fp->disable_tpa = 1;
804 break;
805 }
806 dma_unmap_addr_set((struct sw_rx_bd *)
807 &bp->fp->tpa_pool[i],
808 mapping, 0);
809 fp->tpa_state[i] = BNX2X_TPA_STOP;
810 }
811 }
812 }
813
814 for_each_queue(bp, j) {
815 struct bnx2x_fastpath *fp = &bp->fp[j];
816
817 fp->rx_bd_cons = 0;
818 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
819 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
820
821 /* "next page" elements initialization */
822 /* SGE ring */
823 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
824 struct eth_rx_sge *sge;
825
826 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
827 sge->addr_hi =
828 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
829 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
830 sge->addr_lo =
831 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
832 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
833 }
834
835 bnx2x_init_sge_ring_bit_mask(fp);
836
837 /* RX BD ring */
838 for (i = 1; i <= NUM_RX_RINGS; i++) {
839 struct eth_rx_bd *rx_bd;
840
841 rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
842 rx_bd->addr_hi =
843 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
844 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
845 rx_bd->addr_lo =
846 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
847 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
848 }
849
850 /* CQ ring */
851 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
852 struct eth_rx_cqe_next_page *nextpg;
853
854 nextpg = (struct eth_rx_cqe_next_page *)
855 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
856 nextpg->addr_hi =
857 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
858 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
859 nextpg->addr_lo =
860 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
861 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
862 }
863
864 /* Allocate SGEs and initialize the ring elements */
865 for (i = 0, ring_prod = 0;
866 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
867
868 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
869 BNX2X_ERR("was only able to allocate "
870 "%d rx sges\n", i);
871 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
872 /* Cleanup already allocated elements */
873 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
874 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
875 fp->disable_tpa = 1;
876 ring_prod = 0;
877 break;
878 }
879 ring_prod = NEXT_SGE_IDX(ring_prod);
880 }
881 fp->rx_sge_prod = ring_prod;
882
883 /* Allocate BDs and initialize BD ring */
884 fp->rx_comp_cons = 0;
885 cqe_ring_prod = ring_prod = 0;
886 for (i = 0; i < bp->rx_ring_size; i++) {
887 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
888 BNX2X_ERR("was only able to allocate "
889 "%d rx skbs on queue[%d]\n", i, j);
890 fp->eth_q_stats.rx_skb_alloc_failed++;
891 break;
892 }
893 ring_prod = NEXT_RX_IDX(ring_prod);
894 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
895 WARN_ON(ring_prod <= i);
896 }
897
898 fp->rx_bd_prod = ring_prod;
899 /* must not have more available CQEs than BDs */
900 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
901 cqe_ring_prod);
902 fp->rx_pkt = fp->rx_calls = 0;
903
904 /* Warning!
905 * this will generate an interrupt (to the TSTORM)
906 * must only be done after chip is initialized
907 */
908 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
909 fp->rx_sge_prod);
910 if (j != 0)
911 continue;
912
913 REG_WR(bp, BAR_USTRORM_INTMEM +
914 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
915 U64_LO(fp->rx_comp_mapping));
916 REG_WR(bp, BAR_USTRORM_INTMEM +
917 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
918 U64_HI(fp->rx_comp_mapping));
919 }
920}
921static void bnx2x_free_tx_skbs(struct bnx2x *bp)
922{
923 int i;
924
925 for_each_queue(bp, i) {
926 struct bnx2x_fastpath *fp = &bp->fp[i];
927
928 u16 bd_cons = fp->tx_bd_cons;
929 u16 sw_prod = fp->tx_pkt_prod;
930 u16 sw_cons = fp->tx_pkt_cons;
931
932 while (sw_cons != sw_prod) {
933 bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
934 sw_cons++;
935 }
936 }
937}
938
939static void bnx2x_free_rx_skbs(struct bnx2x *bp)
940{
941 int i, j;
942
943 for_each_queue(bp, j) {
944 struct bnx2x_fastpath *fp = &bp->fp[j];
945
946 for (i = 0; i < NUM_RX_BD; i++) {
947 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
948 struct sk_buff *skb = rx_buf->skb;
949
950 if (skb == NULL)
951 continue;
952
953 dma_unmap_single(&bp->pdev->dev,
954 dma_unmap_addr(rx_buf, mapping),
955 bp->rx_buf_size, DMA_FROM_DEVICE);
956
957 rx_buf->skb = NULL;
958 dev_kfree_skb(skb);
959 }
960 if (!fp->disable_tpa)
961 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
962 ETH_MAX_AGGREGATION_QUEUES_E1 :
963 ETH_MAX_AGGREGATION_QUEUES_E1H);
964 }
965}
966
967void bnx2x_free_skbs(struct bnx2x *bp)
968{
969 bnx2x_free_tx_skbs(bp);
970 bnx2x_free_rx_skbs(bp);
971}
972
973static void bnx2x_free_msix_irqs(struct bnx2x *bp)
974{
975 int i, offset = 1;
976
977 free_irq(bp->msix_table[0].vector, bp->dev);
978 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
979 bp->msix_table[0].vector);
980
981#ifdef BCM_CNIC
982 offset++;
983#endif
984 for_each_queue(bp, i) {
985 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq "
986 "state %x\n", i, bp->msix_table[i + offset].vector,
987 bnx2x_fp(bp, i, state));
988
989 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
990 }
991}
992
993void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
994{
995 if (bp->flags & USING_MSIX_FLAG) {
996 if (!disable_only)
997 bnx2x_free_msix_irqs(bp);
998 pci_disable_msix(bp->pdev);
999 bp->flags &= ~USING_MSIX_FLAG;
1000
1001 } else if (bp->flags & USING_MSI_FLAG) {
1002 if (!disable_only)
1003 free_irq(bp->pdev->irq, bp->dev);
1004 pci_disable_msi(bp->pdev);
1005 bp->flags &= ~USING_MSI_FLAG;
1006
1007 } else if (!disable_only)
1008 free_irq(bp->pdev->irq, bp->dev);
1009}
1010
1011static int bnx2x_enable_msix(struct bnx2x *bp)
1012{
1013 int i, rc, offset = 1;
1014 int igu_vec = 0;
1015
1016 bp->msix_table[0].entry = igu_vec;
1017 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1018
1019#ifdef BCM_CNIC
1020 igu_vec = BP_L_ID(bp) + offset;
1021 bp->msix_table[1].entry = igu_vec;
1022 DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1023 offset++;
1024#endif
1025 for_each_queue(bp, i) {
1026 igu_vec = BP_L_ID(bp) + offset + i;
1027 bp->msix_table[i + offset].entry = igu_vec;
1028 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1029 "(fastpath #%u)\n", i + offset, igu_vec, i);
1030 }
1031
1032 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1033 BNX2X_NUM_QUEUES(bp) + offset);
1034
1035 /*
1036 * reconfigure number of tx/rx queues according to available
1037 * MSI-X vectors
1038 */
1039 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1040 /* vectors available for FP */
1041 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1042
1043 DP(NETIF_MSG_IFUP,
1044 "Trying to use less MSI-X vectors: %d\n", rc);
1045
1046 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1047
1048 if (rc) {
1049 DP(NETIF_MSG_IFUP,
1050 "MSI-X is not attainable rc %d\n", rc);
1051 return rc;
1052 }
1053
1054 bp->num_queues = min(bp->num_queues, fp_vec);
1055
1056 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1057 bp->num_queues);
1058 } else if (rc) {
1059 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1060 return rc;
1061 }
1062
1063 bp->flags |= USING_MSIX_FLAG;
1064
1065 return 0;
1066}
1067
1068static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1069{
1070 int i, rc, offset = 1;
1071
1072 rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1073 bp->dev->name, bp->dev);
1074 if (rc) {
1075 BNX2X_ERR("request sp irq failed\n");
1076 return -EBUSY;
1077 }
1078
1079#ifdef BCM_CNIC
1080 offset++;
1081#endif
1082 for_each_queue(bp, i) {
1083 struct bnx2x_fastpath *fp = &bp->fp[i];
1084 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1085 bp->dev->name, i);
1086
1087 rc = request_irq(bp->msix_table[i + offset].vector,
1088 bnx2x_msix_fp_int, 0, fp->name, fp);
1089 if (rc) {
1090 BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc);
1091 bnx2x_free_msix_irqs(bp);
1092 return -EBUSY;
1093 }
1094
1095 fp->state = BNX2X_FP_STATE_IRQ;
1096 }
1097
1098 i = BNX2X_NUM_QUEUES(bp);
1099 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1100 " ... fp[%d] %d\n",
1101 bp->msix_table[0].vector,
1102 0, bp->msix_table[offset].vector,
1103 i - 1, bp->msix_table[offset + i - 1].vector);
1104
1105 return 0;
1106}
1107
1108static int bnx2x_enable_msi(struct bnx2x *bp)
1109{
1110 int rc;
1111
1112 rc = pci_enable_msi(bp->pdev);
1113 if (rc) {
1114 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1115 return -1;
1116 }
1117 bp->flags |= USING_MSI_FLAG;
1118
1119 return 0;
1120}
1121
1122static int bnx2x_req_irq(struct bnx2x *bp)
1123{
1124 unsigned long flags;
1125 int rc;
1126
1127 if (bp->flags & USING_MSI_FLAG)
1128 flags = 0;
1129 else
1130 flags = IRQF_SHARED;
1131
1132 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1133 bp->dev->name, bp->dev);
1134 if (!rc)
1135 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1136
1137 return rc;
1138}
1139
1140static void bnx2x_napi_enable(struct bnx2x *bp)
1141{
1142 int i;
1143
1144 for_each_queue(bp, i)
1145 napi_enable(&bnx2x_fp(bp, i, napi));
1146}
1147
1148static void bnx2x_napi_disable(struct bnx2x *bp)
1149{
1150 int i;
1151
1152 for_each_queue(bp, i)
1153 napi_disable(&bnx2x_fp(bp, i, napi));
1154}
1155
1156void bnx2x_netif_start(struct bnx2x *bp)
1157{
1158 int intr_sem;
1159
1160 intr_sem = atomic_dec_and_test(&bp->intr_sem);
1161 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1162
1163 if (intr_sem) {
1164 if (netif_running(bp->dev)) {
1165 bnx2x_napi_enable(bp);
1166 bnx2x_int_enable(bp);
1167 if (bp->state == BNX2X_STATE_OPEN)
1168 netif_tx_wake_all_queues(bp->dev);
1169 }
1170 }
1171}
1172
1173void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1174{
1175 bnx2x_int_disable_sync(bp, disable_hw);
1176 bnx2x_napi_disable(bp);
1177 netif_tx_disable(bp->dev);
1178}
1179static int bnx2x_set_num_queues(struct bnx2x *bp)
1180{
1181 int rc = 0;
1182
1183 switch (bp->int_mode) {
1184 case INT_MODE_INTx:
1185 case INT_MODE_MSI:
1186 bp->num_queues = 1;
1187 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1188 break;
1189 default:
1190 /* Set number of queues according to bp->multi_mode value */
1191 bnx2x_set_num_queues_msix(bp);
1192
1193 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1194 bp->num_queues);
1195
1196 /* if we can't use MSI-X we only need one fp,
1197 * so try to enable MSI-X with the requested number of fp's
1198 * and fallback to MSI or legacy INTx with one fp
1199 */
1200 rc = bnx2x_enable_msix(bp);
1201 if (rc)
1202 /* failed to enable MSI-X */
1203 bp->num_queues = 1;
1204 break;
1205 }
1206 bp->dev->real_num_tx_queues = bp->num_queues;
1207 return rc;
1208}
1209
6891dd25
DK		 1210static void bnx2x_release_firmware(struct bnx2x *bp)
1211{
1212 kfree(bp->init_ops_offsets);
1213 kfree(bp->init_ops);
1214 kfree(bp->init_data);
1215 release_firmware(bp->firmware);
1216}
1217
9f6c9258
DK		 1218/* must be called with rtnl_lock */
1219int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1220{
1221 u32 load_code;
1222 int i, rc;
1223
6891dd25
DK		 1224 /* Set init arrays */
1225 rc = bnx2x_init_firmware(bp);
1226 if (rc) {
1227 BNX2X_ERR("Error loading firmware\n");
1228 return rc;
1229 }
1230
9f6c9258
DK		 1231#ifdef BNX2X_STOP_ON_ERROR
1232 if (unlikely(bp->panic))
1233 return -EPERM;
1234#endif
1235
1236 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1237
1238 rc = bnx2x_set_num_queues(bp);
1239
1240 if (bnx2x_alloc_mem(bp)) {
1241 bnx2x_free_irq(bp, true);
1242 return -ENOMEM;
1243 }
1244
1245 for_each_queue(bp, i)
1246 bnx2x_fp(bp, i, disable_tpa) =
1247 ((bp->flags & TPA_ENABLE_FLAG) == 0);
1248
1249 for_each_queue(bp, i)
1250 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1251 bnx2x_poll, 128);
1252
1253 bnx2x_napi_enable(bp);
1254
1255 if (bp->flags & USING_MSIX_FLAG) {
1256 rc = bnx2x_req_msix_irqs(bp);
1257 if (rc) {
1258 bnx2x_free_irq(bp, true);
1259 goto load_error1;
1260 }
1261 } else {
1262 /* Fall to INTx if failed to enable MSI-X due to lack of
1263 memory (in bnx2x_set_num_queues()) */
1264 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1265 bnx2x_enable_msi(bp);
1266 bnx2x_ack_int(bp);
1267 rc = bnx2x_req_irq(bp);
1268 if (rc) {
1269 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1270 bnx2x_free_irq(bp, true);
1271 goto load_error1;
1272 }
1273 if (bp->flags & USING_MSI_FLAG) {
1274 bp->dev->irq = bp->pdev->irq;
1275 netdev_info(bp->dev, "using MSI IRQ %d\n",
1276 bp->pdev->irq);
1277 }
1278 }
1279
1280 /* Send LOAD_REQUEST command to MCP
1281 Returns the type of LOAD command:
1282 if it is the first port to be initialized
1283 common blocks should be initialized, otherwise - not
1284 */
1285 if (!BP_NOMCP(bp)) {
1286 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ);
1287 if (!load_code) {
1288 BNX2X_ERR("MCP response failure, aborting\n");
1289 rc = -EBUSY;
1290 goto load_error2;
1291 }
1292 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1293 rc = -EBUSY; /* other port in diagnostic mode */
1294 goto load_error2;
1295 }
1296
1297 } else {
1298 int port = BP_PORT(bp);
1299
1300 DP(NETIF_MSG_IFUP, "NO MCP - load counts %d, %d, %d\n",
1301 load_count[0], load_count[1], load_count[2]);
1302 load_count[0]++;
1303 load_count[1 + port]++;
1304 DP(NETIF_MSG_IFUP, "NO MCP - new load counts %d, %d, %d\n",
1305 load_count[0], load_count[1], load_count[2]);
1306 if (load_count[0] == 1)
1307 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1308 else if (load_count[1 + port] == 1)
1309 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1310 else
1311 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1312 }
1313
1314 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1315 (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1316 bp->port.pmf = 1;
1317 else
1318 bp->port.pmf = 0;
1319 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1320
1321 /* Initialize HW */
1322 rc = bnx2x_init_hw(bp, load_code);
1323 if (rc) {
1324 BNX2X_ERR("HW init failed, aborting\n");
1325 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1326 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1327 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1328 goto load_error2;
1329 }
1330
1331 /* Setup NIC internals and enable interrupts */
1332 bnx2x_nic_init(bp, load_code);
1333
1334 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1335 (bp->common.shmem2_base))
1336 SHMEM2_WR(bp, dcc_support,
1337 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1338 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1339
1340 /* Send LOAD_DONE command to MCP */
1341 if (!BP_NOMCP(bp)) {
1342 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1343 if (!load_code) {
1344 BNX2X_ERR("MCP response failure, aborting\n");
1345 rc = -EBUSY;
1346 goto load_error3;
1347 }
1348 }
1349
1350 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1351
1352 rc = bnx2x_setup_leading(bp);
1353 if (rc) {
1354 BNX2X_ERR("Setup leading failed!\n");
1355#ifndef BNX2X_STOP_ON_ERROR
1356 goto load_error3;
1357#else
1358 bp->panic = 1;
1359 return -EBUSY;
1360#endif
1361 }
1362
1363 if (CHIP_IS_E1H(bp))
1364 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1365 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1366 bp->flags |= MF_FUNC_DIS;
1367 }
1368
1369 if (bp->state == BNX2X_STATE_OPEN) {
1370#ifdef BCM_CNIC
1371 /* Enable Timer scan */
1372 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1373#endif
1374 for_each_nondefault_queue(bp, i) {
1375 rc = bnx2x_setup_multi(bp, i);
1376 if (rc)
1377#ifdef BCM_CNIC
1378 goto load_error4;
1379#else
1380 goto load_error3;
1381#endif
1382 }
1383
1384 if (CHIP_IS_E1(bp))
1385 bnx2x_set_eth_mac_addr_e1(bp, 1);
1386 else
1387 bnx2x_set_eth_mac_addr_e1h(bp, 1);
1388#ifdef BCM_CNIC
1389 /* Set iSCSI L2 MAC */
1390 mutex_lock(&bp->cnic_mutex);
1391 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1392 bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1393 bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1394 bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1395 CNIC_SB_ID(bp));
1396 }
1397 mutex_unlock(&bp->cnic_mutex);
1398#endif
1399 }
1400
1401 if (bp->port.pmf)
1402 bnx2x_initial_phy_init(bp, load_mode);
1403
1404 /* Start fast path */
1405 switch (load_mode) {
1406 case LOAD_NORMAL:
1407 if (bp->state == BNX2X_STATE_OPEN) {
1408 /* Tx queue should be only reenabled */
1409 netif_tx_wake_all_queues(bp->dev);
1410 }
1411 /* Initialize the receive filter. */
1412 bnx2x_set_rx_mode(bp->dev);
1413 break;
1414
1415 case LOAD_OPEN:
1416 netif_tx_start_all_queues(bp->dev);
1417 if (bp->state != BNX2X_STATE_OPEN)
1418 netif_tx_disable(bp->dev);
1419 /* Initialize the receive filter. */
1420 bnx2x_set_rx_mode(bp->dev);
1421 break;
1422
1423 case LOAD_DIAG:
1424 /* Initialize the receive filter. */
1425 bnx2x_set_rx_mode(bp->dev);
1426 bp->state = BNX2X_STATE_DIAG;
1427 break;
1428
1429 default:
1430 break;
1431 }
1432
1433 if (!bp->port.pmf)
1434 bnx2x__link_status_update(bp);
1435
1436 /* start the timer */
1437 mod_timer(&bp->timer, jiffies + bp->current_interval);
1438
1439#ifdef BCM_CNIC
1440 bnx2x_setup_cnic_irq_info(bp);
1441 if (bp->state == BNX2X_STATE_OPEN)
1442 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1443#endif
1444 bnx2x_inc_load_cnt(bp);
1445
6891dd25
DK		 1446 bnx2x_release_firmware(bp);
1447
9f6c9258
DK		 1448 return 0;
1449
1450#ifdef BCM_CNIC
1451load_error4:
1452 /* Disable Timer scan */
1453 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1454#endif
1455load_error3:
1456 bnx2x_int_disable_sync(bp, 1);
1457 if (!BP_NOMCP(bp)) {
1458 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1459 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1460 }
1461 bp->port.pmf = 0;
1462 /* Free SKBs, SGEs, TPA pool and driver internals */
1463 bnx2x_free_skbs(bp);
1464 for_each_queue(bp, i)
1465 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1466load_error2:
1467 /* Release IRQs */
1468 bnx2x_free_irq(bp, false);
1469load_error1:
1470 bnx2x_napi_disable(bp);
1471 for_each_queue(bp, i)
1472 netif_napi_del(&bnx2x_fp(bp, i, napi));
1473 bnx2x_free_mem(bp);
1474
6891dd25
DK		 1475 bnx2x_release_firmware(bp);
1476
9f6c9258
DK		 1477 return rc;
1478}
1479
1480/* must be called with rtnl_lock */
1481int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1482{
1483 int i;
1484
1485 if (bp->state == BNX2X_STATE_CLOSED) {
1486 /* Interface has been removed - nothing to recover */
1487 bp->recovery_state = BNX2X_RECOVERY_DONE;
1488 bp->is_leader = 0;
1489 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1490 smp_wmb();
1491
1492 return -EINVAL;
1493 }
1494
1495#ifdef BCM_CNIC
1496 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1497#endif
1498 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1499
1500 /* Set "drop all" */
1501 bp->rx_mode = BNX2X_RX_MODE_NONE;
1502 bnx2x_set_storm_rx_mode(bp);
1503
1504 /* Disable HW interrupts, NAPI and Tx */
1505 bnx2x_netif_stop(bp, 1);
1506 netif_carrier_off(bp->dev);
1507
1508 del_timer_sync(&bp->timer);
1509 SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1510 (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1511 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1512
1513 /* Release IRQs */
1514 bnx2x_free_irq(bp, false);
1515
1516 /* Cleanup the chip if needed */
1517 if (unload_mode != UNLOAD_RECOVERY)
1518 bnx2x_chip_cleanup(bp, unload_mode);
1519
1520 bp->port.pmf = 0;
1521
1522 /* Free SKBs, SGEs, TPA pool and driver internals */
1523 bnx2x_free_skbs(bp);
1524 for_each_queue(bp, i)
1525 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1526 for_each_queue(bp, i)
1527 netif_napi_del(&bnx2x_fp(bp, i, napi));
1528 bnx2x_free_mem(bp);
1529
1530 bp->state = BNX2X_STATE_CLOSED;
1531
1532 /* The last driver must disable a "close the gate" if there is no
1533 * parity attention or "process kill" pending.
1534 */
1535 if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1536 bnx2x_reset_is_done(bp))
1537 bnx2x_disable_close_the_gate(bp);
1538
1539 /* Reset MCP mail box sequence if there is on going recovery */
1540 if (unload_mode == UNLOAD_RECOVERY)
1541 bp->fw_seq = 0;
1542
1543 return 0;
1544}
1545int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1546{
1547 u16 pmcsr;
1548
1549 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1550
1551 switch (state) {
1552 case PCI_D0:
1553 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1554 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1555 PCI_PM_CTRL_PME_STATUS));
1556
1557 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1558 /* delay required during transition out of D3hot */
1559 msleep(20);
1560 break;
1561
1562 case PCI_D3hot:
1563 /* If there are other clients above don't
1564 shut down the power */
1565 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1566 return 0;
1567 /* Don't shut down the power for emulation and FPGA */
1568 if (CHIP_REV_IS_SLOW(bp))
1569 return 0;
1570
1571 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1572 pmcsr |= 3;
1573
1574 if (bp->wol)
1575 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1576
1577 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1578 pmcsr);
1579
1580 /* No more memory access after this point until
1581 * device is brought back to D0.
1582 */
1583 break;
1584
1585 default:
1586 return -EINVAL;
1587 }
1588 return 0;
1589}
1590
1591
1592
1593/*
1594 * net_device service functions
1595 */
1596
1597static int bnx2x_poll(struct napi_struct *napi, int budget)
1598{
1599 int work_done = 0;
1600 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1601 napi);
1602 struct bnx2x *bp = fp->bp;
1603
1604 while (1) {
1605#ifdef BNX2X_STOP_ON_ERROR
1606 if (unlikely(bp->panic)) {
1607 napi_complete(napi);
1608 return 0;
1609 }
1610#endif
1611
1612 if (bnx2x_has_tx_work(fp))
1613 bnx2x_tx_int(fp);
1614
1615 if (bnx2x_has_rx_work(fp)) {
1616 work_done += bnx2x_rx_int(fp, budget - work_done);
1617
1618 /* must not complete if we consumed full budget */
1619 if (work_done >= budget)
1620 break;
1621 }
1622
1623 /* Fall out from the NAPI loop if needed */
1624 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1625 bnx2x_update_fpsb_idx(fp);
1626 /* bnx2x_has_rx_work() reads the status block, thus we need
1627 * to ensure that status block indices have been actually read
1628 * (bnx2x_update_fpsb_idx) prior to this check
1629 * (bnx2x_has_rx_work) so that we won't write the "newer"
1630 * value of the status block to IGU (if there was a DMA right
1631 * after bnx2x_has_rx_work and if there is no rmb, the memory
1632 * reading (bnx2x_update_fpsb_idx) may be postponed to right
1633 * before bnx2x_ack_sb). In this case there will never be
1634 * another interrupt until there is another update of the
1635 * status block, while there is still unhandled work.
1636 */
1637 rmb();
1638
1639 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1640 napi_complete(napi);
1641 /* Re-enable interrupts */
1642 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1643 le16_to_cpu(fp->fp_c_idx),
1644 IGU_INT_NOP, 1);
1645 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1646 le16_to_cpu(fp->fp_u_idx),
1647 IGU_INT_ENABLE, 1);
1648 break;
1649 }
1650 }
1651 }
1652
1653 return work_done;
1654}
1655
1656
1657/* we split the first BD into headers and data BDs
1658 * to ease the pain of our fellow microcode engineers
1659 * we use one mapping for both BDs
1660 * So far this has only been observed to happen
1661 * in Other Operating Systems(TM)
1662 */
1663static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1664 struct bnx2x_fastpath *fp,
1665 struct sw_tx_bd *tx_buf,
1666 struct eth_tx_start_bd **tx_bd, u16 hlen,
1667 u16 bd_prod, int nbd)
1668{
1669 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1670 struct eth_tx_bd *d_tx_bd;
1671 dma_addr_t mapping;
1672 int old_len = le16_to_cpu(h_tx_bd->nbytes);
1673
1674 /* first fix first BD */
1675 h_tx_bd->nbd = cpu_to_le16(nbd);
1676 h_tx_bd->nbytes = cpu_to_le16(hlen);
1677
1678 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1679 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1680 h_tx_bd->addr_lo, h_tx_bd->nbd);
1681
1682 /* now get a new data BD
1683 * (after the pbd) and fill it */
1684 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1685 d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1686
1687 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1688 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1689
1690 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1691 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1692 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1693
1694 /* this marks the BD as one that has no individual mapping */
1695 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1696
1697 DP(NETIF_MSG_TX_QUEUED,
1698 "TSO split data size is %d (%x:%x)\n",
1699 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1700
1701 /* update tx_bd */
1702 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1703
1704 return bd_prod;
1705}
1706
1707static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1708{
1709 if (fix > 0)
1710 csum = (u16) ~csum_fold(csum_sub(csum,
1711 csum_partial(t_header - fix, fix, 0)));
1712
1713 else if (fix < 0)
1714 csum = (u16) ~csum_fold(csum_add(csum,
1715 csum_partial(t_header, -fix, 0)));
1716
1717 return swab16(csum);
1718}
1719
1720static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1721{
1722 u32 rc;
1723
1724 if (skb->ip_summed != CHECKSUM_PARTIAL)
1725 rc = XMIT_PLAIN;
1726
1727 else {
1728 if (skb->protocol == htons(ETH_P_IPV6)) {
1729 rc = XMIT_CSUM_V6;
1730 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1731 rc |= XMIT_CSUM_TCP;
1732
1733 } else {
1734 rc = XMIT_CSUM_V4;
1735 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1736 rc |= XMIT_CSUM_TCP;
1737 }
1738 }
1739
1740 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1741 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1742
1743 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1744 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1745
1746 return rc;
1747}
1748
1749#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1750/* check if packet requires linearization (packet is too fragmented)
1751 no need to check fragmentation if page size > 8K (there will be no
1752 violation to FW restrictions) */
1753static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1754 u32 xmit_type)
1755{
1756 int to_copy = 0;
1757 int hlen = 0;
1758 int first_bd_sz = 0;
1759
1760 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1761 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1762
1763 if (xmit_type & XMIT_GSO) {
1764 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1765 /* Check if LSO packet needs to be copied:
1766 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1767 int wnd_size = MAX_FETCH_BD - 3;
1768 /* Number of windows to check */
1769 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1770 int wnd_idx = 0;
1771 int frag_idx = 0;
1772 u32 wnd_sum = 0;
1773
1774 /* Headers length */
1775 hlen = (int)(skb_transport_header(skb) - skb->data) +
1776 tcp_hdrlen(skb);
1777
1778 /* Amount of data (w/o headers) on linear part of SKB*/
1779 first_bd_sz = skb_headlen(skb) - hlen;
1780
1781 wnd_sum = first_bd_sz;
1782
1783 /* Calculate the first sum - it's special */
1784 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1785 wnd_sum +=
1786 skb_shinfo(skb)->frags[frag_idx].size;
1787
1788 /* If there was data on linear skb data - check it */
1789 if (first_bd_sz > 0) {
1790 if (unlikely(wnd_sum < lso_mss)) {
1791 to_copy = 1;
1792 goto exit_lbl;
1793 }
1794
1795 wnd_sum -= first_bd_sz;
1796 }
1797
1798 /* Others are easier: run through the frag list and
1799 check all windows */
1800 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1801 wnd_sum +=
1802 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1803
1804 if (unlikely(wnd_sum < lso_mss)) {
1805 to_copy = 1;
1806 break;
1807 }
1808 wnd_sum -=
1809 skb_shinfo(skb)->frags[wnd_idx].size;
1810 }
1811 } else {
1812 /* in non-LSO too fragmented packet should always
1813 be linearized */
1814 to_copy = 1;
1815 }
1816 }
1817
1818exit_lbl:
1819 if (unlikely(to_copy))
1820 DP(NETIF_MSG_TX_QUEUED,
1821 "Linearization IS REQUIRED for %s packet. "
1822 "num_frags %d hlen %d first_bd_sz %d\n",
1823 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1824 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1825
1826 return to_copy;
1827}
1828#endif
1829
1830/* called with netif_tx_lock
1831 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1832 * netif_wake_queue()
1833 */
1834netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1835{
1836 struct bnx2x *bp = netdev_priv(dev);
1837 struct bnx2x_fastpath *fp;
1838 struct netdev_queue *txq;
1839 struct sw_tx_bd *tx_buf;
1840 struct eth_tx_start_bd *tx_start_bd;
1841 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1842 struct eth_tx_parse_bd *pbd = NULL;
1843 u16 pkt_prod, bd_prod;
1844 int nbd, fp_index;
1845 dma_addr_t mapping;
1846 u32 xmit_type = bnx2x_xmit_type(bp, skb);
1847 int i;
1848 u8 hlen = 0;
1849 __le16 pkt_size = 0;
1850 struct ethhdr *eth;
1851 u8 mac_type = UNICAST_ADDRESS;
1852
1853#ifdef BNX2X_STOP_ON_ERROR
1854 if (unlikely(bp->panic))
1855 return NETDEV_TX_BUSY;
1856#endif
1857
1858 fp_index = skb_get_queue_mapping(skb);
1859 txq = netdev_get_tx_queue(dev, fp_index);
1860
1861 fp = &bp->fp[fp_index];
1862
1863 if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1864 fp->eth_q_stats.driver_xoff++;
1865 netif_tx_stop_queue(txq);
1866 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1867 return NETDEV_TX_BUSY;
1868 }
1869
1870 DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x protocol %x protocol(%x,%x)"
1871 " gso type %x xmit_type %x\n",
1872 skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1873 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1874
1875 eth = (struct ethhdr *)skb->data;
1876
1877 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1878 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1879 if (is_broadcast_ether_addr(eth->h_dest))
1880 mac_type = BROADCAST_ADDRESS;
1881 else
1882 mac_type = MULTICAST_ADDRESS;
1883 }
1884
1885#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1886 /* First, check if we need to linearize the skb (due to FW
1887 restrictions). No need to check fragmentation if page size > 8K
1888 (there will be no violation to FW restrictions) */
1889 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1890 /* Statistics of linearization */
1891 bp->lin_cnt++;
1892 if (skb_linearize(skb) != 0) {
1893 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1894 "silently dropping this SKB\n");
1895 dev_kfree_skb_any(skb);
1896 return NETDEV_TX_OK;
1897 }
1898 }
1899#endif
1900
1901 /*
1902 Please read carefully. First we use one BD which we mark as start,
1903 then we have a parsing info BD (used for TSO or xsum),
1904 and only then we have the rest of the TSO BDs.
1905 (don't forget to mark the last one as last,
1906 and to unmap only AFTER you write to the BD ...)
1907 And above all, all pdb sizes are in words - NOT DWORDS!
1908 */
1909
1910 pkt_prod = fp->tx_pkt_prod++;
1911 bd_prod = TX_BD(fp->tx_bd_prod);
1912
1913 /* get a tx_buf and first BD */
1914 tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1915 tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1916
1917 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1918 tx_start_bd->general_data = (mac_type <<
1919 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1920 /* header nbd */
1921 tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1922
1923 /* remember the first BD of the packet */
1924 tx_buf->first_bd = fp->tx_bd_prod;
1925 tx_buf->skb = skb;
1926 tx_buf->flags = 0;
1927
1928 DP(NETIF_MSG_TX_QUEUED,
1929 "sending pkt %u @%p next_idx %u bd %u @%p\n",
1930 pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1931
1932#ifdef BCM_VLAN
1933 if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1934 (bp->flags & HW_VLAN_TX_FLAG)) {
1935 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1936 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1937 } else
1938#endif
1939 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1940
1941 /* turn on parsing and get a BD */
1942 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1943 pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1944
1945 memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1946
1947 if (xmit_type & XMIT_CSUM) {
1948 hlen = (skb_network_header(skb) - skb->data) / 2;
1949
1950 /* for now NS flag is not used in Linux */
1951 pbd->global_data =
1952 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1953 ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1954
1955 pbd->ip_hlen = (skb_transport_header(skb) -
1956 skb_network_header(skb)) / 2;
1957
1958 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1959
1960 pbd->total_hlen = cpu_to_le16(hlen);
1961 hlen = hlen*2;
1962
1963 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1964
1965 if (xmit_type & XMIT_CSUM_V4)
1966 tx_start_bd->bd_flags.as_bitfield |=
1967 ETH_TX_BD_FLAGS_IP_CSUM;
1968 else
1969 tx_start_bd->bd_flags.as_bitfield |=
1970 ETH_TX_BD_FLAGS_IPV6;
1971
1972 if (xmit_type & XMIT_CSUM_TCP) {
1973 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1974
1975 } else {
1976 s8 fix = SKB_CS_OFF(skb); /* signed! */
1977
1978 pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1979
1980 DP(NETIF_MSG_TX_QUEUED,
1981 "hlen %d fix %d csum before fix %x\n",
1982 le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1983
1984 /* HW bug: fixup the CSUM */
1985 pbd->tcp_pseudo_csum =
1986 bnx2x_csum_fix(skb_transport_header(skb),
1987 SKB_CS(skb), fix);
1988
1989 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1990 pbd->tcp_pseudo_csum);
1991 }
1992 }
1993
1994 mapping = dma_map_single(&bp->pdev->dev, skb->data,
1995 skb_headlen(skb), DMA_TO_DEVICE);
1996
1997 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1998 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1999 nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2000 tx_start_bd->nbd = cpu_to_le16(nbd);
2001 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2002 pkt_size = tx_start_bd->nbytes;
2003
2004 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
2005 " nbytes %d flags %x vlan %x\n",
2006 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2007 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2008 tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
2009
2010 if (xmit_type & XMIT_GSO) {
2011
2012 DP(NETIF_MSG_TX_QUEUED,
2013 "TSO packet len %d hlen %d total len %d tso size %d\n",
2014 skb->len, hlen, skb_headlen(skb),
2015 skb_shinfo(skb)->gso_size);
2016
2017 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2018
2019 if (unlikely(skb_headlen(skb) > hlen))
2020 bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2021 hlen, bd_prod, ++nbd);
2022
2023 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2024 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2025 pbd->tcp_flags = pbd_tcp_flags(skb);
2026
2027 if (xmit_type & XMIT_GSO_V4) {
2028 pbd->ip_id = swab16(ip_hdr(skb)->id);
2029 pbd->tcp_pseudo_csum =
2030 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2031 ip_hdr(skb)->daddr,
2032 0, IPPROTO_TCP, 0));
2033
2034 } else
2035 pbd->tcp_pseudo_csum =
2036 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2037 &ipv6_hdr(skb)->daddr,
2038 0, IPPROTO_TCP, 0));
2039
2040 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2041 }
2042 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2043
2044 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2045 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2046
2047 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2048 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2049 if (total_pkt_bd == NULL)
2050 total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2051
2052 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2053 frag->page_offset,
2054 frag->size, DMA_TO_DEVICE);
2055
2056 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2057 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2058 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2059 le16_add_cpu(&pkt_size, frag->size);
2060
2061 DP(NETIF_MSG_TX_QUEUED,
2062 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2063 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2064 le16_to_cpu(tx_data_bd->nbytes));
2065 }
2066
2067 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2068
2069 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2070
2071 /* now send a tx doorbell, counting the next BD
2072 * if the packet contains or ends with it
2073 */
2074 if (TX_BD_POFF(bd_prod) < nbd)
2075 nbd++;
2076
2077 if (total_pkt_bd != NULL)
2078 total_pkt_bd->total_pkt_bytes = pkt_size;
2079
2080 if (pbd)
2081 DP(NETIF_MSG_TX_QUEUED,
2082 "PBD @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2083 " tcp_flags %x xsum %x seq %u hlen %u\n",
2084 pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2085 pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2086 pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2087
2088 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2089
2090 /*
2091 * Make sure that the BD data is updated before updating the producer
2092 * since FW might read the BD right after the producer is updated.
2093 * This is only applicable for weak-ordered memory model archs such
2094 * as IA-64. The following barrier is also mandatory since FW will
2095 * assumes packets must have BDs.
2096 */
2097 wmb();
2098
2099 fp->tx_db.data.prod += nbd;
2100 barrier();
2101 DOORBELL(bp, fp->index, fp->tx_db.raw);
2102
2103 mmiowb();
2104
2105 fp->tx_bd_prod += nbd;
2106
2107 if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2108 netif_tx_stop_queue(txq);
2109
2110 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2111 * ordering of set_bit() in netif_tx_stop_queue() and read of
2112 * fp->bd_tx_cons */
2113 smp_mb();
2114
2115 fp->eth_q_stats.driver_xoff++;
2116 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2117 netif_tx_wake_queue(txq);
2118 }
2119 fp->tx_pkt++;
2120
2121 return NETDEV_TX_OK;
2122}
2123/* called with rtnl_lock */
2124int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2125{
2126 struct sockaddr *addr = p;
2127 struct bnx2x *bp = netdev_priv(dev);
2128
2129 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2130 return -EINVAL;
2131
2132 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2133 if (netif_running(dev)) {
2134 if (CHIP_IS_E1(bp))
2135 bnx2x_set_eth_mac_addr_e1(bp, 1);
2136 else
2137 bnx2x_set_eth_mac_addr_e1h(bp, 1);
2138 }
2139
2140 return 0;
2141}
2142
2143/* called with rtnl_lock */
2144int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2145{
2146 struct bnx2x *bp = netdev_priv(dev);
2147 int rc = 0;
2148
2149 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2150 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2151 return -EAGAIN;
2152 }
2153
2154 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2155 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2156 return -EINVAL;
2157
2158 /* This does not race with packet allocation
2159 * because the actual alloc size is
2160 * only updated as part of load
2161 */
2162 dev->mtu = new_mtu;
2163
2164 if (netif_running(dev)) {
2165 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2166 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2167 }
2168
2169 return rc;
2170}
2171
2172void bnx2x_tx_timeout(struct net_device *dev)
2173{
2174 struct bnx2x *bp = netdev_priv(dev);
2175
2176#ifdef BNX2X_STOP_ON_ERROR
2177 if (!bp->panic)
2178 bnx2x_panic();
2179#endif
2180 /* This allows the netif to be shutdown gracefully before resetting */
2181 schedule_delayed_work(&bp->reset_task, 0);
2182}
2183
2184#ifdef BCM_VLAN
2185/* called with rtnl_lock */
2186void bnx2x_vlan_rx_register(struct net_device *dev,
2187 struct vlan_group *vlgrp)
2188{
2189 struct bnx2x *bp = netdev_priv(dev);
2190
2191 bp->vlgrp = vlgrp;
2192
2193 /* Set flags according to the required capabilities */
2194 bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2195
2196 if (dev->features & NETIF_F_HW_VLAN_TX)
2197 bp->flags |= HW_VLAN_TX_FLAG;
2198
2199 if (dev->features & NETIF_F_HW_VLAN_RX)
2200 bp->flags |= HW_VLAN_RX_FLAG;
2201
2202 if (netif_running(dev))
2203 bnx2x_set_client_config(bp);
2204}
2205
2206#endif
2207int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2208{
2209 struct net_device *dev = pci_get_drvdata(pdev);
2210 struct bnx2x *bp;
2211
2212 if (!dev) {
2213 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2214 return -ENODEV;
2215 }
2216 bp = netdev_priv(dev);
2217
2218 rtnl_lock();
2219
2220 pci_save_state(pdev);
2221
2222 if (!netif_running(dev)) {
2223 rtnl_unlock();
2224 return 0;
2225 }
2226
2227 netif_device_detach(dev);
2228
2229 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2230
2231 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2232
2233 rtnl_unlock();
2234
2235 return 0;
2236}
2237
2238int bnx2x_resume(struct pci_dev *pdev)
2239{
2240 struct net_device *dev = pci_get_drvdata(pdev);
2241 struct bnx2x *bp;
2242 int rc;
2243
2244 if (!dev) {
2245 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2246 return -ENODEV;
2247 }
2248 bp = netdev_priv(dev);
2249
2250 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2251 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2252 return -EAGAIN;
2253 }
2254
2255 rtnl_lock();
2256
2257 pci_restore_state(pdev);
2258
2259 if (!netif_running(dev)) {
2260 rtnl_unlock();
2261 return 0;
2262 }
2263
2264 bnx2x_set_power_state(bp, PCI_D0);
2265 netif_device_attach(dev);
2266
2267 rc = bnx2x_nic_load(bp, LOAD_OPEN);
2268
2269 rtnl_unlock();
2270
2271 return rc;
2272}
Clone of davem's net-next-2.6 tree