From 16eb8815c2355b50bff218513367778e6303e9f9 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Fri, 26 Aug 2011 07:43:54 +0000 Subject: [PATCH] igb: Refactor clean_rx_irq to reduce overhead and improve performance This change is meant to be a general cleanup and performance improvement for clean_rx_irq. The previous patch should have updated the allocation so that the rings can be treated as read-only within the clean_rx_irq function. In addition I am re-ordering the operations such that several goals are accomplished including reducing the overhead for packet accounting, reducing the number of items on the stack, and improving overall performance. Signed-off-by: Alexander Duyck Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igb/igb_main.c | 96 +++++++++++------------ 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 9fa2ad01c6b7..dd85df0ed7f2 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -138,7 +138,7 @@ static void igb_setup_dca(struct igb_adapter *); #endif /* CONFIG_IGB_DCA */ static bool igb_clean_tx_irq(struct igb_q_vector *); static int igb_poll(struct napi_struct *, int); -static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int); +static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int); static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); static void igb_tx_timeout(struct net_device *); static void igb_reset_task(struct work_struct *); @@ -5481,28 +5481,27 @@ static int igb_poll(struct napi_struct *napi, int budget) struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi); - int tx_clean_complete = 1, work_done = 0; + bool clean_complete = true; #ifdef CONFIG_IGB_DCA if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) igb_update_dca(q_vector); #endif if (q_vector->tx_ring) - tx_clean_complete = igb_clean_tx_irq(q_vector); + clean_complete = !!igb_clean_tx_irq(q_vector); if (q_vector->rx_ring) - igb_clean_rx_irq_adv(q_vector, &work_done, budget); + clean_complete &= igb_clean_rx_irq_adv(q_vector, budget); - if (!tx_clean_complete) - work_done = budget; + /* If all work not completed, return budget and keep polling */ + if (!clean_complete) + return budget; /* If not enough Rx work done, exit the polling mode */ - if (work_done < budget) { - napi_complete(napi); - igb_ring_irq_enable(q_vector); - } + napi_complete(napi); + igb_ring_irq_enable(q_vector); - return work_done; + return 0; } /** @@ -5751,37 +5750,26 @@ static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc) return hlen; } -static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, - int *work_done, int budget) +static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, int budget) { struct igb_ring *rx_ring = q_vector->rx_ring; - struct net_device *netdev = rx_ring->netdev; - struct device *dev = rx_ring->dev; - union e1000_adv_rx_desc *rx_desc , *next_rxd; - struct igb_buffer *buffer_info , *next_buffer; - struct sk_buff *skb; - bool cleaned = false; - u16 cleaned_count = igb_desc_unused(rx_ring); - int current_node = numa_node_id(); + union e1000_adv_rx_desc *rx_desc; + const int current_node = numa_node_id(); unsigned int total_bytes = 0, total_packets = 0; - unsigned int i; u32 staterr; - u16 length; + u16 cleaned_count = igb_desc_unused(rx_ring); + u16 i = rx_ring->next_to_clean; - i = rx_ring->next_to_clean; - buffer_info = &rx_ring->buffer_info[i]; rx_desc = E1000_RX_DESC_ADV(*rx_ring, i); staterr = le32_to_cpu(rx_desc->wb.upper.status_error); while (staterr & E1000_RXD_STAT_DD) { - if (*work_done >= budget) - break; - (*work_done)++; - rmb(); /* read descriptor and rx_buffer_info after status DD */ + struct igb_buffer *buffer_info = &rx_ring->buffer_info[i]; + struct sk_buff *skb = buffer_info->skb; + union e1000_adv_rx_desc *next_rxd; - skb = buffer_info->skb; - prefetch(skb->data - NET_IP_ALIGN); buffer_info->skb = NULL; + prefetch(skb->data); i++; if (i == rx_ring->count) @@ -5789,42 +5777,48 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, next_rxd = E1000_RX_DESC_ADV(*rx_ring, i); prefetch(next_rxd); - next_buffer = &rx_ring->buffer_info[i]; - length = le16_to_cpu(rx_desc->wb.upper.length); - cleaned = true; - cleaned_count++; + /* + * This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * RXD_STAT_DD bit is set + */ + rmb(); - if (buffer_info->dma) { - dma_unmap_single(dev, buffer_info->dma, + if (!skb_is_nonlinear(skb)) { + __skb_put(skb, igb_get_hlen(rx_desc)); + dma_unmap_single(rx_ring->dev, buffer_info->dma, IGB_RX_HDR_LEN, DMA_FROM_DEVICE); buffer_info->dma = 0; - skb_put(skb, igb_get_hlen(rx_desc)); } - if (length) { - dma_unmap_page(dev, buffer_info->page_dma, - PAGE_SIZE / 2, DMA_FROM_DEVICE); - buffer_info->page_dma = 0; + if (rx_desc->wb.upper.length) { + u16 length = le16_to_cpu(rx_desc->wb.upper.length); skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, buffer_info->page, buffer_info->page_offset, length); + skb->len += length; + skb->data_len += length; + skb->truesize += length; + if ((page_count(buffer_info->page) != 1) || (page_to_nid(buffer_info->page) != current_node)) buffer_info->page = NULL; else get_page(buffer_info->page); - skb->len += length; - skb->data_len += length; - skb->truesize += length; + dma_unmap_page(rx_ring->dev, buffer_info->page_dma, + PAGE_SIZE / 2, DMA_FROM_DEVICE); + buffer_info->page_dma = 0; } if (!(staterr & E1000_RXD_STAT_EOP)) { + struct igb_buffer *next_buffer; + next_buffer = &rx_ring->buffer_info[i]; buffer_info->skb = next_buffer->skb; buffer_info->dma = next_buffer->dma; next_buffer->skb = skb; @@ -5833,7 +5827,7 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, } if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { - dev_kfree_skb_irq(skb); + dev_kfree_skb_any(skb); goto next_desc; } @@ -5844,7 +5838,7 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, igb_rx_checksum_adv(rx_ring, staterr, skb); - skb->protocol = eth_type_trans(skb, netdev); + skb->protocol = eth_type_trans(skb, rx_ring->netdev); if (staterr & E1000_RXD_STAT_VP) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); @@ -5853,7 +5847,12 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, } napi_gro_receive(&q_vector->napi, skb); + budget--; next_desc: + if (!budget) + break; + + cleaned_count++; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IGB_RX_BUFFER_WRITE) { igb_alloc_rx_buffers_adv(rx_ring, cleaned_count); @@ -5862,7 +5861,6 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, /* use prefetched values */ rx_desc = next_rxd; - buffer_info = next_buffer; staterr = le32_to_cpu(rx_desc->wb.upper.status_error); } @@ -5877,7 +5875,7 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector, if (cleaned_count) igb_alloc_rx_buffers_adv(rx_ring, cleaned_count); - return cleaned; + return !!budget; } static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,