X-Git-Url: https://bbs.cooldavid.org/git/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_main.c;h=dc78736d30528185a6082687b7fa2b38b228dc7e;hb=33cf09c9586a0dce472ecd2aac13e8140c9ed1a1;hp=f85631263af8573f7f0ea96ac63e961abebe225b;hpb=e596c79050eafea89cf9fe26b7a807643a2a9904;p=net-next-2.6.git diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c index f85631263af..dc78736d305 100644 --- a/drivers/net/ixgbe/ixgbe_main.c +++ b/drivers/net/ixgbe/ixgbe_main.c @@ -600,18 +600,17 @@ static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, } } -void ixgbe_unmap_and_free_tx_resource(struct ixgbe_adapter *adapter, - struct ixgbe_tx_buffer - *tx_buffer_info) +void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring, + struct ixgbe_tx_buffer *tx_buffer_info) { if (tx_buffer_info->dma) { if (tx_buffer_info->mapped_as_page) - dma_unmap_page(&adapter->pdev->dev, + dma_unmap_page(tx_ring->dev, tx_buffer_info->dma, tx_buffer_info->length, DMA_TO_DEVICE); else - dma_unmap_single(&adapter->pdev->dev, + dma_unmap_single(tx_ring->dev, tx_buffer_info->dma, tx_buffer_info->length, DMA_TO_DEVICE); @@ -704,8 +703,8 @@ static inline bool ixgbe_check_tx_hang(struct ixgbe_adapter *adapter, " time_stamp <%lx>\n" " jiffies <%lx>\n", tx_ring->queue_index, - IXGBE_READ_REG(hw, tx_ring->head), - IXGBE_READ_REG(hw, tx_ring->tail), + IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), + IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), tx_ring->next_to_use, eop, tx_ring->tx_buffer_info[eop].time_stamp, jiffies); return true; @@ -734,7 +733,6 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring) { struct ixgbe_adapter *adapter = q_vector->adapter; - struct net_device *netdev = adapter->netdev; union ixgbe_adv_tx_desc *tx_desc, *eop_desc; struct ixgbe_tx_buffer *tx_buffer_info; unsigned int i, eop, count = 0; @@ -749,44 +747,23 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, bool cleaned = false; rmb(); /* read buffer_info after eop_desc */ for ( ; !cleaned; count++) { - struct sk_buff *skb; tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i); tx_buffer_info = &tx_ring->tx_buffer_info[i]; - cleaned = (i == eop); - skb = tx_buffer_info->skb; - - if (cleaned && skb) { - unsigned int segs, bytecount; - unsigned int hlen = skb_headlen(skb); - - /* gso_segs is currently only valid for tcp */ - segs = skb_shinfo(skb)->gso_segs ?: 1; -#ifdef IXGBE_FCOE - /* adjust for FCoE Sequence Offload */ - if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) - && (skb->protocol == htons(ETH_P_FCOE)) && - skb_is_gso(skb)) { - hlen = skb_transport_offset(skb) + - sizeof(struct fc_frame_header) + - sizeof(struct fcoe_crc_eof); - segs = DIV_ROUND_UP(skb->len - hlen, - skb_shinfo(skb)->gso_size); - } -#endif /* IXGBE_FCOE */ - /* multiply data chunks by size of headers */ - bytecount = ((segs - 1) * hlen) + skb->len; - total_packets += segs; - total_bytes += bytecount; - } - - ixgbe_unmap_and_free_tx_resource(adapter, - tx_buffer_info); tx_desc->wb.status = 0; + cleaned = (i == eop); i++; if (i == tx_ring->count) i = 0; + + if (cleaned && tx_buffer_info->skb) { + total_bytes += tx_buffer_info->bytecount; + total_packets += tx_buffer_info->gso_segs; + } + + ixgbe_unmap_and_free_tx_resource(tx_ring, + tx_buffer_info); } eop = tx_ring->tx_buffer_info[i].next_to_watch; @@ -796,16 +773,16 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, tx_ring->next_to_clean = i; #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) - if (unlikely(count && netif_carrier_ok(netdev) && + if (unlikely(count && netif_carrier_ok(tx_ring->netdev) && (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. */ smp_mb(); - if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) && + if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) && !test_bit(__IXGBE_DOWN, &adapter->state)) { - netif_wake_subqueue(netdev, tx_ring->queue_index); - ++tx_ring->restart_queue; + netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index); + ++tx_ring->tx_stats.restart_queue; } } @@ -833,63 +810,98 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, #ifdef CONFIG_IXGBE_DCA static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *rx_ring, + int cpu) { + struct ixgbe_hw *hw = &adapter->hw; u32 rxctrl; - int cpu = get_cpu(); - int q = rx_ring->reg_idx; + u8 reg_idx = rx_ring->reg_idx; - if (rx_ring->cpu != cpu) { - rxctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_DCA_RXCTRL(q)); - if (adapter->hw.mac.type == ixgbe_mac_82598EB) { - rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK; - rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); - } else if (adapter->hw.mac.type == ixgbe_mac_82599EB) { - rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599; - rxctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) << - IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599); - } - rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN; - rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN; - rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN); - rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_WRO_EN | - IXGBE_DCA_RXCTRL_DESC_HSRO_EN); - IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_RXCTRL(q), rxctrl); - rx_ring->cpu = cpu; + rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx)); + switch (hw->mac.type) { + case ixgbe_mac_82598EB: + rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK; + rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); + break; + case ixgbe_mac_82599EB: + rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599; + rxctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) << + IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599); + break; + default: + break; } - put_cpu(); + rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN; + rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN; + rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN); + rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_WRO_EN | + IXGBE_DCA_RXCTRL_DESC_HSRO_EN); + IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); } static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) + struct ixgbe_ring *tx_ring, + int cpu) { + struct ixgbe_hw *hw = &adapter->hw; u32 txctrl; + u8 reg_idx = tx_ring->reg_idx; + + switch (hw->mac.type) { + case ixgbe_mac_82598EB: + txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx)); + txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK; + txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); + txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; + txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; + IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl); + break; + case ixgbe_mac_82599EB: + txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx)); + txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599; + txctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) << + IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599); + txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; + txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; + IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl); + break; + default: + break; + } +} + +static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) +{ + struct ixgbe_adapter *adapter = q_vector->adapter; int cpu = get_cpu(); - int q = tx_ring->reg_idx; - struct ixgbe_hw *hw = &adapter->hw; + long r_idx; + int i; - if (tx_ring->cpu != cpu) { - if (adapter->hw.mac.type == ixgbe_mac_82598EB) { - txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(q)); - txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK; - txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); - txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; - IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(q), txctrl); - } else if (adapter->hw.mac.type == ixgbe_mac_82599EB) { - txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(q)); - txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599; - txctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) << - IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599); - txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; - IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(q), txctrl); - } - tx_ring->cpu = cpu; + if (q_vector->cpu == cpu) + goto out_no_update; + + r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); + for (i = 0; i < q_vector->txr_count; i++) { + ixgbe_update_tx_dca(adapter, adapter->tx_ring[r_idx], cpu); + r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, + r_idx + 1); + } + + r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); + for (i = 0; i < q_vector->rxr_count; i++) { + ixgbe_update_rx_dca(adapter, adapter->rx_ring[r_idx], cpu); + r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, + r_idx + 1); } + + q_vector->cpu = cpu; +out_no_update: put_cpu(); } static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) { + int num_q_vectors; int i; if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) @@ -898,22 +910,25 @@ static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) /* always use CB2 mode, difference is masked in the CB driver */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); - for (i = 0; i < adapter->num_tx_queues; i++) { - adapter->tx_ring[i]->cpu = -1; - ixgbe_update_tx_dca(adapter, adapter->tx_ring[i]); - } - for (i = 0; i < adapter->num_rx_queues; i++) { - adapter->rx_ring[i]->cpu = -1; - ixgbe_update_rx_dca(adapter, adapter->rx_ring[i]); + if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) + num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; + else + num_q_vectors = 1; + + for (i = 0; i < num_q_vectors; i++) { + adapter->q_vector[i]->cpu = -1; + ixgbe_update_dca(adapter->q_vector[i]); } } static int __ixgbe_notify_dca(struct device *dev, void *data) { - struct net_device *netdev = dev_get_drvdata(dev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = dev_get_drvdata(dev); unsigned long event = *(unsigned long *)data; + if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) + return 0; + switch (event) { case DCA_PROVIDER_ADD: /* if we're already enabled, don't do it again */ @@ -1012,8 +1027,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter, skb->ip_summed = CHECKSUM_UNNECESSARY; } -static inline void ixgbe_release_rx_desc(struct ixgbe_hw *hw, - struct ixgbe_ring *rx_ring, u32 val) +static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val) { /* * Force memory writes to complete before letting h/w @@ -1022,72 +1036,81 @@ static inline void ixgbe_release_rx_desc(struct ixgbe_hw *hw, * such as IA-64). */ wmb(); - IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->reg_idx), val); + writel(val, rx_ring->tail); } /** * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split - * @adapter: address of board private structure + * @rx_ring: ring to place buffers on + * @cleaned_count: number of buffers to replace **/ -void ixgbe_alloc_rx_buffers(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring, - int cleaned_count) +void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { - struct net_device *netdev = adapter->netdev; - struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; - unsigned int i; - unsigned int bufsz = rx_ring->rx_buf_len; + struct sk_buff *skb; + u16 i = rx_ring->next_to_use; - i = rx_ring->next_to_use; - bi = &rx_ring->rx_buffer_info[i]; + /* do nothing if no valid netdev defined */ + if (!rx_ring->netdev) + return; while (cleaned_count--) { rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i); + bi = &rx_ring->rx_buffer_info[i]; + skb = bi->skb; - if (!bi->page_dma && - (rx_ring->flags & IXGBE_RING_RX_PS_ENABLED)) { - if (!bi->page) { - bi->page = netdev_alloc_page(netdev); - if (!bi->page) { - adapter->alloc_rx_page_failed++; - goto no_buffers; - } - bi->page_offset = 0; - } else { - /* use a half page if we're re-using */ - bi->page_offset ^= (PAGE_SIZE / 2); - } - - bi->page_dma = dma_map_page(&pdev->dev, bi->page, - bi->page_offset, - (PAGE_SIZE / 2), - DMA_FROM_DEVICE); - } - - if (!bi->skb) { - struct sk_buff *skb = netdev_alloc_skb_ip_align(netdev, - bufsz); - bi->skb = skb; - + if (!skb) { + skb = netdev_alloc_skb_ip_align(rx_ring->netdev, + rx_ring->rx_buf_len); if (!skb) { - adapter->alloc_rx_buff_failed++; + rx_ring->rx_stats.alloc_rx_buff_failed++; goto no_buffers; } /* initialize queue mapping */ skb_record_rx_queue(skb, rx_ring->queue_index); + bi->skb = skb; } if (!bi->dma) { - bi->dma = dma_map_single(&pdev->dev, - bi->skb->data, + bi->dma = dma_map_single(rx_ring->dev, + skb->data, rx_ring->rx_buf_len, DMA_FROM_DEVICE); + if (dma_mapping_error(rx_ring->dev, bi->dma)) { + rx_ring->rx_stats.alloc_rx_buff_failed++; + bi->dma = 0; + goto no_buffers; + } } - /* Refresh the desc even if buffer_addrs didn't change because - * each write-back erases this info. */ + if (rx_ring->flags & IXGBE_RING_RX_PS_ENABLED) { + if (!bi->page) { + bi->page = netdev_alloc_page(rx_ring->netdev); + if (!bi->page) { + rx_ring->rx_stats.alloc_rx_page_failed++; + goto no_buffers; + } + } + + if (!bi->page_dma) { + /* use a half page if we're re-using */ + bi->page_offset ^= PAGE_SIZE / 2; + bi->page_dma = dma_map_page(rx_ring->dev, + bi->page, + bi->page_offset, + PAGE_SIZE / 2, + DMA_FROM_DEVICE); + if (dma_mapping_error(rx_ring->dev, + bi->page_dma)) { + rx_ring->rx_stats.alloc_rx_page_failed++; + bi->page_dma = 0; + goto no_buffers; + } + } + + /* Refresh the desc even if buffer_addrs didn't change + * because each write-back erases this info. */ rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); } else { @@ -1098,16 +1121,12 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_adapter *adapter, i++; if (i == rx_ring->count) i = 0; - bi = &rx_ring->rx_buffer_info[i]; } no_buffers: if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; - if (i-- == 0) - i = (rx_ring->count - 1); - - ixgbe_release_rx_desc(&adapter->hw, rx_ring, i); + ixgbe_release_rx_desc(rx_ring, i); } } @@ -1170,7 +1189,6 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, int *work_done, int work_to_do) { struct ixgbe_adapter *adapter = q_vector->adapter; - struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc, *next_rxd; struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer; struct sk_buff *skb; @@ -1227,7 +1245,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, IXGBE_RSC_CB(skb)->delay_unmap = true; IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma; } else { - dma_unmap_single(&pdev->dev, + dma_unmap_single(rx_ring->dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); @@ -1237,8 +1255,10 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (upper_len) { - dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, - PAGE_SIZE / 2, DMA_FROM_DEVICE); + dma_unmap_page(rx_ring->dev, + rx_buffer_info->page_dma, + PAGE_SIZE / 2, + DMA_FROM_DEVICE); rx_buffer_info->page_dma = 0; skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, rx_buffer_info->page, @@ -1278,10 +1298,10 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, if (staterr & IXGBE_RXD_STAT_EOP) { if (skb->prev) skb = ixgbe_transform_rsc_queue(skb, - &(rx_ring->rsc_count)); + &(rx_ring->rx_stats.rsc_count)); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { if (IXGBE_RSC_CB(skb)->delay_unmap) { - dma_unmap_single(&pdev->dev, + dma_unmap_single(rx_ring->dev, IXGBE_RSC_CB(skb)->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); @@ -1289,11 +1309,11 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, IXGBE_RSC_CB(skb)->delay_unmap = false; } if (rx_ring->flags & IXGBE_RING_RX_PS_ENABLED) - rx_ring->rsc_count += - skb_shinfo(skb)->nr_frags; + rx_ring->rx_stats.rsc_count += + skb_shinfo(skb)->nr_frags; else - rx_ring->rsc_count++; - rx_ring->rsc_flush++; + rx_ring->rx_stats.rsc_count++; + rx_ring->rx_stats.rsc_flush++; } u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets++; @@ -1309,7 +1329,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, skb->next = next_buffer->skb; skb->next->prev = skb; } - rx_ring->non_eop_descs++; + rx_ring->rx_stats.non_eop_descs++; goto next_desc; } @@ -1324,7 +1344,7 @@ static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_bytes += skb->len; total_rx_packets++; - skb->protocol = eth_type_trans(skb, adapter->netdev); + skb->protocol = eth_type_trans(skb, rx_ring->netdev); #ifdef IXGBE_FCOE /* if ddp, not passing to ULD unless for FCP_RSP or error */ if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { @@ -1340,7 +1360,7 @@ next_desc: /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { - ixgbe_alloc_rx_buffers(adapter, rx_ring, cleaned_count); + ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); cleaned_count = 0; } @@ -1355,14 +1375,14 @@ next_desc: cleaned_count = IXGBE_DESC_UNUSED(rx_ring); if (cleaned_count) - ixgbe_alloc_rx_buffers(adapter, rx_ring, cleaned_count); + ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); #ifdef IXGBE_FCOE /* include DDPed FCoE data */ if (ddp_bytes > 0) { unsigned int mss; - mss = adapter->netdev->mtu - sizeof(struct fcoe_hdr) - + mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) - sizeof(struct fc_frame_header) - sizeof(struct fcoe_crc_eof); if (mss > 512) @@ -1846,8 +1866,13 @@ static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data) int r_idx; int i; +#ifdef CONFIG_IXGBE_DCA + if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) + ixgbe_update_dca(q_vector); +#endif + r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); - for (i = 0; i < q_vector->rxr_count; i++) { + for (i = 0; i < q_vector->rxr_count; i++) { rx_ring = adapter->rx_ring[r_idx]; rx_ring->total_bytes = 0; rx_ring->total_packets = 0; @@ -1858,7 +1883,6 @@ static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data) if (!q_vector->rxr_count) return IRQ_HANDLED; - /* disable interrupts on this vector only */ /* EIAM disabled interrupts (on this vector) for us */ napi_schedule(&q_vector->napi); @@ -1917,13 +1941,14 @@ static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget) int work_done = 0; long r_idx; - r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); - rx_ring = adapter->rx_ring[r_idx]; #ifdef CONFIG_IXGBE_DCA if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - ixgbe_update_rx_dca(adapter, rx_ring); + ixgbe_update_dca(q_vector); #endif + r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); + rx_ring = adapter->rx_ring[r_idx]; + ixgbe_clean_rx_irq(q_vector, rx_ring, &work_done, budget); /* If all Rx work done, exit the polling mode */ @@ -1957,13 +1982,14 @@ static int ixgbe_clean_rxtx_many(struct napi_struct *napi, int budget) long r_idx; bool tx_clean_complete = true; +#ifdef CONFIG_IXGBE_DCA + if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) + ixgbe_update_dca(q_vector); +#endif + r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); for (i = 0; i < q_vector->txr_count; i++) { ring = adapter->tx_ring[r_idx]; -#ifdef CONFIG_IXGBE_DCA - if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - ixgbe_update_tx_dca(adapter, ring); -#endif tx_clean_complete &= ixgbe_clean_tx_irq(q_vector, ring); r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, r_idx + 1); @@ -1976,10 +2002,6 @@ static int ixgbe_clean_rxtx_many(struct napi_struct *napi, int budget) r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); for (i = 0; i < q_vector->rxr_count; i++) { ring = adapter->rx_ring[r_idx]; -#ifdef CONFIG_IXGBE_DCA - if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - ixgbe_update_rx_dca(adapter, ring); -#endif ixgbe_clean_rx_irq(q_vector, ring, &work_done, budget); r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, r_idx + 1); @@ -2018,13 +2040,14 @@ static int ixgbe_clean_txonly(struct napi_struct *napi, int budget) int work_done = 0; long r_idx; - r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); - tx_ring = adapter->tx_ring[r_idx]; #ifdef CONFIG_IXGBE_DCA if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - ixgbe_update_tx_dca(adapter, tx_ring); + ixgbe_update_dca(q_vector); #endif + r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); + tx_ring = adapter->tx_ring[r_idx]; + if (!ixgbe_clean_tx_irq(q_vector, tx_ring)) work_done = budget; @@ -2483,8 +2506,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, ring->count * sizeof(union ixgbe_adv_tx_desc)); IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); - ring->head = IXGBE_TDH(reg_idx); - ring->tail = IXGBE_TDT(reg_idx); + ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx); /* configure fetching thresholds */ if (adapter->rx_itr_setting == 0) { @@ -2809,8 +2831,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, ring->count * sizeof(union ixgbe_adv_rx_desc)); IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); - ring->head = IXGBE_RDH(reg_idx); - ring->tail = IXGBE_RDT(reg_idx); + ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx); ixgbe_configure_srrctl(adapter, ring); ixgbe_configure_rscctl(adapter, ring); @@ -2832,7 +2853,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); ixgbe_rx_desc_queue_enable(adapter, ring); - ixgbe_alloc_rx_buffers(adapter, ring, IXGBE_DESC_UNUSED(ring)); + ixgbe_alloc_rx_buffers(ring, IXGBE_DESC_UNUSED(ring)); } static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) @@ -3347,8 +3368,7 @@ static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; - u32 txdctl; - int i, j; + int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { if (hw->mac.type == ixgbe_mac_82598EB) @@ -3359,23 +3379,23 @@ static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) if (hw->mac.type == ixgbe_mac_82598EB) netif_set_gso_max_size(adapter->netdev, 32768); - ixgbe_dcb_calculate_tc_credits(&adapter->dcb_cfg, DCB_TX_CONFIG); - ixgbe_dcb_calculate_tc_credits(&adapter->dcb_cfg, DCB_RX_CONFIG); +#ifdef CONFIG_FCOE + if (adapter->netdev->features & NETIF_F_FCOE_MTU) + max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); +#endif - /* reconfigure the hardware */ - ixgbe_dcb_hw_config(&adapter->hw, &adapter->dcb_cfg); + ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, + DCB_TX_CONFIG); + ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, + DCB_RX_CONFIG); - for (i = 0; i < adapter->num_tx_queues; i++) { - j = adapter->tx_ring[i]->reg_idx; - txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(j)); - /* PThresh workaround for Tx hang with DFP enabled. */ - txdctl |= 32; - IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(j), txdctl); - } /* Enable VLAN tag insert/strip */ adapter->netdev->features |= NETIF_F_HW_VLAN_RX; hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true); + + /* reconfigure the hardware */ + ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg); } #endif @@ -3678,15 +3698,13 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) /** * ixgbe_clean_rx_ring - Free Rx Buffers per Queue - * @adapter: board private structure * @rx_ring: ring to free buffers from **/ -static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) +static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; + struct device *dev = rx_ring->dev; unsigned long size; - unsigned int i; + u16 i; /* ring already cleared, nothing to do */ if (!rx_ring->rx_buffer_info) @@ -3698,7 +3716,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, rx_buffer_info = &rx_ring->rx_buffer_info[i]; if (rx_buffer_info->dma) { - dma_unmap_single(&pdev->dev, rx_buffer_info->dma, + dma_unmap_single(rx_ring->dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); rx_buffer_info->dma = 0; @@ -3709,7 +3727,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, do { struct sk_buff *this = skb; if (IXGBE_RSC_CB(this)->delay_unmap) { - dma_unmap_single(&pdev->dev, + dma_unmap_single(dev, IXGBE_RSC_CB(this)->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); @@ -3723,7 +3741,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, if (!rx_buffer_info->page) continue; if (rx_buffer_info->page_dma) { - dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, + dma_unmap_page(dev, rx_buffer_info->page_dma, PAGE_SIZE / 2, DMA_FROM_DEVICE); rx_buffer_info->page_dma = 0; } @@ -3740,24 +3758,17 @@ static void ixgbe_clean_rx_ring(struct ixgbe_adapter *adapter, rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; - - if (rx_ring->head) - writel(0, adapter->hw.hw_addr + rx_ring->head); - if (rx_ring->tail) - writel(0, adapter->hw.hw_addr + rx_ring->tail); } /** * ixgbe_clean_tx_ring - Free Tx Buffers - * @adapter: board private structure * @tx_ring: ring to be cleaned **/ -static void ixgbe_clean_tx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) +static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) { struct ixgbe_tx_buffer *tx_buffer_info; unsigned long size; - unsigned int i; + u16 i; /* ring already cleared, nothing to do */ if (!tx_ring->tx_buffer_info) @@ -3766,7 +3777,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_adapter *adapter, /* Free all the Tx ring sk_buffs */ for (i = 0; i < tx_ring->count; i++) { tx_buffer_info = &tx_ring->tx_buffer_info[i]; - ixgbe_unmap_and_free_tx_resource(adapter, tx_buffer_info); + ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); } size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; @@ -3777,11 +3788,6 @@ static void ixgbe_clean_tx_ring(struct ixgbe_adapter *adapter, tx_ring->next_to_use = 0; tx_ring->next_to_clean = 0; - - if (tx_ring->head) - writel(0, adapter->hw.hw_addr + tx_ring->head); - if (tx_ring->tail) - writel(0, adapter->hw.hw_addr + tx_ring->tail); } /** @@ -3793,7 +3799,7 @@ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) int i; for (i = 0; i < adapter->num_rx_queues; i++) - ixgbe_clean_rx_ring(adapter, adapter->rx_ring[i]); + ixgbe_clean_rx_ring(adapter->rx_ring[i]); } /** @@ -3805,7 +3811,7 @@ static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) int i; for (i = 0; i < adapter->num_tx_queues; i++) - ixgbe_clean_tx_ring(adapter, adapter->tx_ring[i]); + ixgbe_clean_tx_ring(adapter->tx_ring[i]); } void ixgbe_down(struct ixgbe_adapter *adapter) @@ -3916,10 +3922,8 @@ static int ixgbe_poll(struct napi_struct *napi, int budget) int tx_clean_complete, work_done = 0; #ifdef CONFIG_IXGBE_DCA - if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { - ixgbe_update_tx_dca(adapter, adapter->tx_ring[0]); - ixgbe_update_rx_dca(adapter, adapter->rx_ring[0]); - } + if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) + ixgbe_update_dca(q_vector); #endif tx_clean_complete = ixgbe_clean_tx_irq(q_vector, adapter->tx_ring[0]); @@ -4463,6 +4467,7 @@ static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter) static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter) { int i; + int rx_count; int orig_node = adapter->node; for (i = 0; i < adapter->num_tx_queues; i++) { @@ -4481,6 +4486,8 @@ static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter) goto err_tx_ring_allocation; ring->count = adapter->tx_ring_count; ring->queue_index = i; + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; ring->numa_node = adapter->node; adapter->tx_ring[i] = ring; @@ -4489,6 +4496,7 @@ static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter) /* Restore the adapter's original node */ adapter->node = orig_node; + rx_count = adapter->rx_ring_count; for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; if (orig_node == -1) { @@ -4503,8 +4511,10 @@ static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter) ring = kzalloc(sizeof(struct ixgbe_ring), GFP_KERNEL); if (!ring) goto err_rx_ring_allocation; - ring->count = adapter->rx_ring_count; + ring->count = rx_count; ring->queue_index = i; + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; ring->numa_node = adapter->node; adapter->rx_ring[i] = ring; @@ -4742,6 +4752,11 @@ err_set_interrupt: return err; } +static void ring_free_rcu(struct rcu_head *head) +{ + kfree(container_of(head, struct ixgbe_ring, rcu)); +} + /** * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings * @adapter: board private structure to clear interrupt scheme on @@ -4758,7 +4773,12 @@ void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter) adapter->tx_ring[i] = NULL; } for (i = 0; i < adapter->num_rx_queues; i++) { - kfree(adapter->rx_ring[i]); + struct ixgbe_ring *ring = adapter->rx_ring[i]; + + /* ixgbe_get_stats64() might access this ring, we must wait + * a grace period before freeing it. + */ + call_rcu(&ring->rcu, ring_free_rcu); adapter->rx_ring[i] = NULL; } @@ -4835,6 +4855,7 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) int j; struct tc_configuration *tc; #endif + int max_frame = dev->mtu + ETH_HLEN + ETH_FCS_LEN; /* PCI config space info */ @@ -4911,8 +4932,8 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) #ifdef CONFIG_DCB adapter->last_lfc_mode = hw->fc.current_mode; #endif - hw->fc.high_water = IXGBE_DEFAULT_FCRTH; - hw->fc.low_water = IXGBE_DEFAULT_FCRTL; + hw->fc.high_water = FC_HIGH_WATER(max_frame); + hw->fc.low_water = FC_LOW_WATER(max_frame); hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; hw->fc.send_xon = true; hw->fc.disable_fc_autoneg = false; @@ -4950,15 +4971,13 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) /** * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) - * @adapter: board private structure * @tx_ring: tx descriptor ring (for a specific queue) to setup * * Return 0 on success, negative on failure **/ -int ixgbe_setup_tx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) +int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) { - struct pci_dev *pdev = adapter->pdev; + struct device *dev = tx_ring->dev; int size; size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; @@ -4973,7 +4992,7 @@ int ixgbe_setup_tx_resources(struct ixgbe_adapter *adapter, tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); tx_ring->size = ALIGN(tx_ring->size, 4096); - tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, + tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; @@ -4986,7 +5005,7 @@ int ixgbe_setup_tx_resources(struct ixgbe_adapter *adapter, err: vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; - e_err(probe, "Unable to allocate memory for the Tx descriptor ring\n"); + dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); return -ENOMEM; } @@ -5005,7 +5024,7 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) int i, err = 0; for (i = 0; i < adapter->num_tx_queues; i++) { - err = ixgbe_setup_tx_resources(adapter, adapter->tx_ring[i]); + err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); if (!err) continue; e_err(probe, "Allocation for Tx Queue %u failed\n", i); @@ -5017,48 +5036,41 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) - * @adapter: board private structure * @rx_ring: rx descriptor ring (for a specific queue) to setup * * Returns 0 on success, negative on failure **/ -int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) +int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; + struct device *dev = rx_ring->dev; int size; size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; - rx_ring->rx_buffer_info = vmalloc_node(size, adapter->node); + rx_ring->rx_buffer_info = vmalloc_node(size, rx_ring->numa_node); if (!rx_ring->rx_buffer_info) rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) { - e_err(probe, "vmalloc allocation failed for the Rx " - "descriptor ring\n"); - goto alloc_failed; - } + if (!rx_ring->rx_buffer_info) + goto err; memset(rx_ring->rx_buffer_info, 0, size); /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); - rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) { - e_err(probe, "Memory allocation failed for the Rx " - "descriptor ring\n"); - vfree(rx_ring->rx_buffer_info); - goto alloc_failed; - } + if (!rx_ring->desc) + goto err; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; return 0; - -alloc_failed: +err: + vfree(rx_ring->rx_buffer_info); + rx_ring->rx_buffer_info = NULL; + dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); return -ENOMEM; } @@ -5072,13 +5084,12 @@ alloc_failed: * * Return 0 on success, negative on failure **/ - static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) { int i, err = 0; for (i = 0; i < adapter->num_rx_queues; i++) { - err = ixgbe_setup_rx_resources(adapter, adapter->rx_ring[i]); + err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); if (!err) continue; e_err(probe, "Allocation for Rx Queue %u failed\n", i); @@ -5090,23 +5101,23 @@ static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) /** * ixgbe_free_tx_resources - Free Tx Resources per Queue - * @adapter: board private structure * @tx_ring: Tx descriptor ring for a specific queue * * Free all transmit software resources **/ -void ixgbe_free_tx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) +void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) { - struct pci_dev *pdev = adapter->pdev; - - ixgbe_clean_tx_ring(adapter, tx_ring); + ixgbe_clean_tx_ring(tx_ring); vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; - dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, - tx_ring->dma); + /* if not set, then don't free */ + if (!tx_ring->desc) + return; + + dma_free_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; } @@ -5123,28 +5134,28 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) if (adapter->tx_ring[i]->desc) - ixgbe_free_tx_resources(adapter, adapter->tx_ring[i]); + ixgbe_free_tx_resources(adapter->tx_ring[i]); } /** * ixgbe_free_rx_resources - Free Rx Resources - * @adapter: board private structure * @rx_ring: ring to clean the resources from * * Free all receive software resources **/ -void ixgbe_free_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) +void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; - - ixgbe_clean_rx_ring(adapter, rx_ring); + ixgbe_clean_rx_ring(rx_ring); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; - dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, - rx_ring->dma); + /* if not set, then don't free */ + if (!rx_ring->desc) + return; + + dma_free_coherent(rx_ring->dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; } @@ -5161,7 +5172,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) if (adapter->rx_ring[i]->desc) - ixgbe_free_rx_resources(adapter, adapter->rx_ring[i]); + ixgbe_free_rx_resources(adapter->rx_ring[i]); } /** @@ -5174,6 +5185,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) { struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_hw *hw = &adapter->hw; int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; /* MTU < 68 is an error and causes problems on some kernels */ @@ -5184,6 +5196,9 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) /* must set new MTU before calling down or up */ netdev->mtu = new_mtu; + hw->fc.high_water = FC_HIGH_WATER(max_frame); + hw->fc.low_water = FC_LOW_WATER(max_frame); + if (netif_running(netdev)) ixgbe_reinit_locked(adapter); @@ -5279,8 +5294,8 @@ static int ixgbe_close(struct net_device *netdev) #ifdef CONFIG_PM static int ixgbe_resume(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; u32 err; pci_set_power_state(pdev, PCI_D0); @@ -5311,7 +5326,7 @@ static int ixgbe_resume(struct pci_dev *pdev) IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); if (netif_running(netdev)) { - err = ixgbe_open(adapter->netdev); + err = ixgbe_open(netdev); if (err) return err; } @@ -5324,8 +5339,8 @@ static int ixgbe_resume(struct pci_dev *pdev) static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; u32 ctrl, fctrl; u32 wufc = adapter->wol; @@ -5342,6 +5357,8 @@ static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) ixgbe_free_all_rx_resources(adapter); } + ixgbe_clear_interrupt_scheme(adapter); + #ifdef CONFIG_PM retval = pci_save_state(pdev); if (retval) @@ -5375,8 +5392,6 @@ static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) *enable_wake = !!wufc; - ixgbe_clear_interrupt_scheme(adapter); - ixgbe_release_hw_control(adapter); pci_disable_device(pdev); @@ -5425,10 +5440,12 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; + struct ixgbe_hw_stats *hwstats = &adapter->stats; u64 total_mpc = 0; u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; - u64 non_eop_descs = 0, restart_queue = 0; - struct ixgbe_hw_stats *hwstats = &adapter->stats; + u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; + u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; + u64 bytes = 0, packets = 0; if (test_bit(__IXGBE_DOWN, &adapter->state) || test_bit(__IXGBE_RESETTING, &adapter->state)) @@ -5441,21 +5458,41 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) adapter->hw_rx_no_dma_resources += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); for (i = 0; i < adapter->num_rx_queues; i++) { - rsc_count += adapter->rx_ring[i]->rsc_count; - rsc_flush += adapter->rx_ring[i]->rsc_flush; + rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; + rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; } adapter->rsc_total_count = rsc_count; adapter->rsc_total_flush = rsc_flush; } + for (i = 0; i < adapter->num_rx_queues; i++) { + struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + non_eop_descs += rx_ring->rx_stats.non_eop_descs; + alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; + alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; + bytes += rx_ring->stats.bytes; + packets += rx_ring->stats.packets; + } + adapter->non_eop_descs = non_eop_descs; + adapter->alloc_rx_page_failed = alloc_rx_page_failed; + adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; + netdev->stats.rx_bytes = bytes; + netdev->stats.rx_packets = packets; + + bytes = 0; + packets = 0; /* gather some stats to the adapter struct that are per queue */ - for (i = 0; i < adapter->num_tx_queues; i++) - restart_queue += adapter->tx_ring[i]->restart_queue; + for (i = 0; i < adapter->num_tx_queues; i++) { + struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; + restart_queue += tx_ring->tx_stats.restart_queue; + tx_busy += tx_ring->tx_stats.tx_busy; + bytes += tx_ring->stats.bytes; + packets += tx_ring->stats.packets; + } adapter->restart_queue = restart_queue; - - for (i = 0; i < adapter->num_rx_queues; i++) - non_eop_descs += adapter->rx_ring[i]->non_eop_descs; - adapter->non_eop_descs = non_eop_descs; + adapter->tx_busy = tx_busy; + netdev->stats.tx_bytes = bytes; + netdev->stats.tx_packets = packets; hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); for (i = 0; i < 8; i++) { @@ -5815,7 +5852,7 @@ static void ixgbe_watchdog_task(struct work_struct *work) static int ixgbe_tso(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring, struct sk_buff *skb, - u32 tx_flags, u8 *hdr_len) + u32 tx_flags, u8 *hdr_len, __be16 protocol) { struct ixgbe_adv_tx_context_desc *context_desc; unsigned int i; @@ -5833,7 +5870,7 @@ static int ixgbe_tso(struct ixgbe_adapter *adapter, l4len = tcp_hdrlen(skb); *hdr_len += l4len; - if (skb->protocol == htons(ETH_P_IP)) { + if (protocol == htons(ETH_P_IP)) { struct iphdr *iph = ip_hdr(skb); iph->tot_len = 0; iph->check = 0; @@ -5872,7 +5909,7 @@ static int ixgbe_tso(struct ixgbe_adapter *adapter, type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT); - if (skb->protocol == htons(ETH_P_IP)) + if (protocol == htons(ETH_P_IP)) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); @@ -5898,16 +5935,10 @@ static int ixgbe_tso(struct ixgbe_adapter *adapter, return false; } -static u32 ixgbe_psum(struct ixgbe_adapter *adapter, struct sk_buff *skb) +static u32 ixgbe_psum(struct ixgbe_adapter *adapter, struct sk_buff *skb, + __be16 protocol) { u32 rtn = 0; - __be16 protocol; - - if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) - protocol = ((const struct vlan_ethhdr *)skb->data)-> - h_vlan_encapsulated_proto; - else - protocol = skb->protocol; switch (protocol) { case cpu_to_be16(ETH_P_IP): @@ -5935,7 +5966,7 @@ static u32 ixgbe_psum(struct ixgbe_adapter *adapter, struct sk_buff *skb) default: if (unlikely(net_ratelimit())) e_warn(probe, "partial checksum but proto=%x!\n", - skb->protocol); + protocol); break; } @@ -5944,7 +5975,8 @@ static u32 ixgbe_psum(struct ixgbe_adapter *adapter, struct sk_buff *skb) static bool ixgbe_tx_csum(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring, - struct sk_buff *skb, u32 tx_flags) + struct sk_buff *skb, u32 tx_flags, + __be16 protocol) { struct ixgbe_adv_tx_context_desc *context_desc; unsigned int i; @@ -5973,7 +6005,7 @@ static bool ixgbe_tx_csum(struct ixgbe_adapter *adapter, IXGBE_ADVTXD_DTYP_CTXT); if (skb->ip_summed == CHECKSUM_PARTIAL) - type_tucmd_mlhl |= ixgbe_psum(adapter, skb); + type_tucmd_mlhl |= ixgbe_psum(adapter, skb, protocol); context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); /* use index zero for tx checksum offload */ @@ -5996,15 +6028,17 @@ static bool ixgbe_tx_csum(struct ixgbe_adapter *adapter, static int ixgbe_tx_map(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring, struct sk_buff *skb, u32 tx_flags, - unsigned int first) + unsigned int first, const u8 hdr_len) { - struct pci_dev *pdev = adapter->pdev; + struct device *dev = tx_ring->dev; struct ixgbe_tx_buffer *tx_buffer_info; unsigned int len; unsigned int total = skb->len; unsigned int offset = 0, size, count = 0, i; unsigned int nr_frags = skb_shinfo(skb)->nr_frags; unsigned int f; + unsigned int bytecount = skb->len; + u16 gso_segs = 1; i = tx_ring->next_to_use; @@ -6019,10 +6053,10 @@ static int ixgbe_tx_map(struct ixgbe_adapter *adapter, tx_buffer_info->length = size; tx_buffer_info->mapped_as_page = false; - tx_buffer_info->dma = dma_map_single(&pdev->dev, + tx_buffer_info->dma = dma_map_single(dev, skb->data + offset, size, DMA_TO_DEVICE); - if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) + if (dma_mapping_error(dev, tx_buffer_info->dma)) goto dma_error; tx_buffer_info->time_stamp = jiffies; tx_buffer_info->next_to_watch = i; @@ -6055,12 +6089,12 @@ static int ixgbe_tx_map(struct ixgbe_adapter *adapter, size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD); tx_buffer_info->length = size; - tx_buffer_info->dma = dma_map_page(&adapter->pdev->dev, + tx_buffer_info->dma = dma_map_page(dev, frag->page, offset, size, DMA_TO_DEVICE); tx_buffer_info->mapped_as_page = true; - if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) + if (dma_mapping_error(dev, tx_buffer_info->dma)) goto dma_error; tx_buffer_info->time_stamp = jiffies; tx_buffer_info->next_to_watch = i; @@ -6074,6 +6108,19 @@ static int ixgbe_tx_map(struct ixgbe_adapter *adapter, break; } + if (tx_flags & IXGBE_TX_FLAGS_TSO) + gso_segs = skb_shinfo(skb)->gso_segs; +#ifdef IXGBE_FCOE + /* adjust for FCoE Sequence Offload */ + else if (tx_flags & IXGBE_TX_FLAGS_FSO) + gso_segs = DIV_ROUND_UP(skb->len - hdr_len, + skb_shinfo(skb)->gso_size); +#endif /* IXGBE_FCOE */ + bytecount += (gso_segs - 1) * hdr_len; + + /* multiply data chunks by size of headers */ + tx_ring->tx_buffer_info[i].bytecount = bytecount; + tx_ring->tx_buffer_info[i].gso_segs = gso_segs; tx_ring->tx_buffer_info[i].skb = skb; tx_ring->tx_buffer_info[first].next_to_watch = i; @@ -6095,14 +6142,13 @@ dma_error: i += tx_ring->count; i--; tx_buffer_info = &tx_ring->tx_buffer_info[i]; - ixgbe_unmap_and_free_tx_resource(adapter, tx_buffer_info); + ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); } return 0; } -static void ixgbe_tx_queue(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring, +static void ixgbe_tx_queue(struct ixgbe_ring *tx_ring, int tx_flags, int count, u32 paylen, u8 hdr_len) { union ixgbe_adv_tx_desc *tx_desc = NULL; @@ -6167,11 +6213,11 @@ static void ixgbe_tx_queue(struct ixgbe_adapter *adapter, wmb(); tx_ring->next_to_use = i; - writel(i, adapter->hw.hw_addr + tx_ring->tail); + writel(i, tx_ring->tail); } static void ixgbe_atr(struct ixgbe_adapter *adapter, struct sk_buff *skb, - int queue, u32 tx_flags) + int queue, u32 tx_flags, __be16 protocol) { struct ixgbe_atr_input atr_input; struct tcphdr *th; @@ -6182,7 +6228,7 @@ static void ixgbe_atr(struct ixgbe_adapter *adapter, struct sk_buff *skb, u8 l4type = 0; /* Right now, we support IPv4 only */ - if (skb->protocol != htons(ETH_P_IP)) + if (protocol != htons(ETH_P_IP)) return; /* check if we're UDP or TCP */ if (iph->protocol == IPPROTO_TCP) { @@ -6217,10 +6263,9 @@ static void ixgbe_atr(struct ixgbe_adapter *adapter, struct sk_buff *skb, ixgbe_fdir_add_signature_filter_82599(&adapter->hw, &atr_input, queue); } -static int __ixgbe_maybe_stop_tx(struct net_device *netdev, - struct ixgbe_ring *tx_ring, int size) +static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, int size) { - netif_stop_subqueue(netdev, tx_ring->queue_index); + netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); /* Herbert's original patch had: * smp_mb__after_netif_stop_queue(); * but since that doesn't exist yet, just open code it. */ @@ -6232,27 +6277,29 @@ static int __ixgbe_maybe_stop_tx(struct net_device *netdev, return -EBUSY; /* A reprieve! - use start_queue because it doesn't call schedule */ - netif_start_subqueue(netdev, tx_ring->queue_index); - ++tx_ring->restart_queue; + netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); + ++tx_ring->tx_stats.restart_queue; return 0; } -static int ixgbe_maybe_stop_tx(struct net_device *netdev, - struct ixgbe_ring *tx_ring, int size) +static int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, int size) { if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) return 0; - return __ixgbe_maybe_stop_tx(netdev, tx_ring, size); + return __ixgbe_maybe_stop_tx(tx_ring, size); } static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) { struct ixgbe_adapter *adapter = netdev_priv(dev); int txq = smp_processor_id(); - #ifdef IXGBE_FCOE - if ((skb->protocol == htons(ETH_P_FCOE)) || - (skb->protocol == htons(ETH_P_FIP))) { + __be16 protocol; + + protocol = vlan_get_protocol(skb); + + if ((protocol == htons(ETH_P_FCOE)) || + (protocol == htons(ETH_P_FIP))) { if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1); txq += adapter->ring_feature[RING_F_FCOE].mask; @@ -6284,10 +6331,11 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) return skb_tx_hash(dev, skb); } -netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev, +netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { + struct net_device *netdev = tx_ring->netdev; struct netdev_queue *txq; unsigned int first; unsigned int tx_flags = 0; @@ -6295,6 +6343,9 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev int tso; int count = 0; unsigned int f; + __be16 protocol; + + protocol = vlan_get_protocol(skb); if (vlan_tx_tag_present(skb)) { tx_flags |= vlan_tx_tag_get(skb); @@ -6315,8 +6366,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev /* for FCoE with DCB, we force the priority to what * was specified by the switch */ if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED && - (skb->protocol == htons(ETH_P_FCOE) || - skb->protocol == htons(ETH_P_FIP))) { + (protocol == htons(ETH_P_FCOE) || + protocol == htons(ETH_P_FIP))) { #ifdef CONFIG_IXGBE_DCB if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) { tx_flags &= ~(IXGBE_TX_FLAGS_VLAN_PRIO_MASK @@ -6326,7 +6377,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev } #endif /* flag for FCoE offloads */ - if (skb->protocol == htons(ETH_P_FCOE)) + if (protocol == htons(ETH_P_FCOE)) tx_flags |= IXGBE_TX_FLAGS_FCOE; } #endif @@ -6342,8 +6393,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); - if (ixgbe_maybe_stop_tx(netdev, tx_ring, count)) { - adapter->tx_busy++; + if (ixgbe_maybe_stop_tx(tx_ring, count)) { + tx_ring->tx_stats.tx_busy++; return NETDEV_TX_BUSY; } @@ -6360,9 +6411,10 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev tx_flags |= IXGBE_TX_FLAGS_FSO; #endif /* IXGBE_FCOE */ } else { - if (skb->protocol == htons(ETH_P_IP)) + if (protocol == htons(ETH_P_IP)) tx_flags |= IXGBE_TX_FLAGS_IPV4; - tso = ixgbe_tso(adapter, tx_ring, skb, tx_flags, &hdr_len); + tso = ixgbe_tso(adapter, tx_ring, skb, tx_flags, &hdr_len, + protocol); if (tso < 0) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; @@ -6370,12 +6422,13 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev if (tso) tx_flags |= IXGBE_TX_FLAGS_TSO; - else if (ixgbe_tx_csum(adapter, tx_ring, skb, tx_flags) && + else if (ixgbe_tx_csum(adapter, tx_ring, skb, tx_flags, + protocol) && (skb->ip_summed == CHECKSUM_PARTIAL)) tx_flags |= IXGBE_TX_FLAGS_CSUM; } - count = ixgbe_tx_map(adapter, tx_ring, skb, tx_flags, first); + count = ixgbe_tx_map(adapter, tx_ring, skb, tx_flags, first, hdr_len); if (count) { /* add the ATR filter if ATR is on */ if (tx_ring->atr_sample_rate) { @@ -6384,16 +6437,15 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct net_device *netdev test_bit(__IXGBE_FDIR_INIT_DONE, &tx_ring->reinit_state)) { ixgbe_atr(adapter, skb, tx_ring->queue_index, - tx_flags); + tx_flags, protocol); tx_ring->atr_count = 0; } } txq = netdev_get_tx_queue(netdev, tx_ring->queue_index); txq->tx_bytes += skb->len; txq->tx_packets++; - ixgbe_tx_queue(adapter, tx_ring, tx_flags, count, skb->len, - hdr_len); - ixgbe_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED); + ixgbe_tx_queue(tx_ring, tx_flags, count, skb->len, hdr_len); + ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); } else { dev_kfree_skb_any(skb); @@ -6410,7 +6462,7 @@ static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netd struct ixgbe_ring *tx_ring; tx_ring = adapter->tx_ring[skb->queue_mapping]; - return ixgbe_xmit_frame_ring(skb, netdev, adapter, tx_ring); + return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); } /** @@ -6551,20 +6603,23 @@ static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev, /* accurate rx/tx bytes/packets stats */ dev_txq_stats_fold(netdev, stats); + rcu_read_lock(); for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbe_ring *ring = adapter->rx_ring[i]; + struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]); u64 bytes, packets; unsigned int start; - do { - start = u64_stats_fetch_begin_bh(&ring->syncp); - packets = ring->stats.packets; - bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); - stats->rx_packets += packets; - stats->rx_bytes += bytes; + if (ring) { + do { + start = u64_stats_fetch_begin_bh(&ring->syncp); + packets = ring->stats.packets; + bytes = ring->stats.bytes; + } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); + stats->rx_packets += packets; + stats->rx_bytes += bytes; + } } - + rcu_read_unlock(); /* following stats updated by ixgbe_watchdog_task() */ stats->multicast = netdev->stats.multicast; stats->rx_errors = netdev->stats.rx_errors; @@ -6746,8 +6801,8 @@ static int __devinit ixgbe_probe(struct pci_dev *pdev, SET_NETDEV_DEV(netdev, &pdev->dev); - pci_set_drvdata(pdev, netdev); adapter = netdev_priv(netdev); + pci_set_drvdata(pdev, adapter); adapter->netdev = netdev; adapter->pdev = pdev; @@ -7070,8 +7125,8 @@ err_dma: **/ static void __devexit ixgbe_remove(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; set_bit(__IXGBE_DOWN, &adapter->state); /* clear the module not found bit to make sure the worker won't @@ -7141,8 +7196,8 @@ static void __devexit ixgbe_remove(struct pci_dev *pdev) static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; netif_device_detach(netdev); @@ -7165,8 +7220,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, */ static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); pci_ers_result_t result; int err; @@ -7204,8 +7258,8 @@ static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) */ static void ixgbe_io_resume(struct pci_dev *pdev) { - struct net_device *netdev = pci_get_drvdata(pdev); - struct ixgbe_adapter *adapter = netdev_priv(netdev); + struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; if (netif_running(netdev)) { if (ixgbe_up(adapter)) { @@ -7270,6 +7324,7 @@ static void __exit ixgbe_exit_module(void) dca_unregister_notify(&dca_notifier); #endif pci_unregister_driver(&ixgbe_driver); + rcu_barrier(); /* Wait for completion of call_rcu()'s */ } #ifdef CONFIG_IXGBE_DCA