cxgb4: Add low latency socket busy_poll support
cxgb_busy_poll, corresponding to ndo_busy_poll, gets called by the socket waiting for data. With busy_poll enabled, improvement is seen in latency numbers as observed by collecting netperf TCP_RR numbers. Below are latency number, with and without busy-poll, in a switched environment for a particular msg size: netperf command: netperf -4 -H <ip> -l 30 -t TCP_RR -- -r1,1 Latency without busy-poll: ~16.25 us Latency with busy-poll : ~08.79 us Based on original work by Kumar Sanghvi <kumaras@chelsio.com> Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Hariprasad Shenai
David S. Miller
parent
3fcf901118
commit
3a336cb171
4 changed files with 174 additions and 3 deletions
|
|
@ -476,6 +476,22 @@ struct sge_rspq { /* state for an SGE response queue */
|
|||
struct adapter *adap;
|
||||
struct net_device *netdev; /* associated net device */
|
||||
rspq_handler_t handler;
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
#define CXGB_POLL_STATE_IDLE 0
|
||||
#define CXGB_POLL_STATE_NAPI BIT(0) /* NAPI owns this poll */
|
||||
#define CXGB_POLL_STATE_POLL BIT(1) /* poll owns this poll */
|
||||
#define CXGB_POLL_STATE_NAPI_YIELD BIT(2) /* NAPI yielded this poll */
|
||||
#define CXGB_POLL_STATE_POLL_YIELD BIT(3) /* poll yielded this poll */
|
||||
#define CXGB_POLL_YIELD (CXGB_POLL_STATE_NAPI_YIELD | \
|
||||
CXGB_POLL_STATE_POLL_YIELD)
|
||||
#define CXGB_POLL_LOCKED (CXGB_POLL_STATE_NAPI | \
|
||||
CXGB_POLL_STATE_POLL)
|
||||
#define CXGB_POLL_USER_PEND (CXGB_POLL_STATE_POLL | \
|
||||
CXGB_POLL_STATE_POLL_YIELD)
|
||||
unsigned int bpoll_state;
|
||||
spinlock_t bpoll_lock; /* lock for busy poll */
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
|
||||
};
|
||||
|
||||
struct sge_eth_stats { /* Ethernet queue statistics */
|
||||
|
|
@ -880,6 +896,102 @@ static inline struct adapter *netdev2adap(const struct net_device *dev)
|
|||
return netdev2pinfo(dev)->adapter;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
static inline void cxgb_busy_poll_init_lock(struct sge_rspq *q)
|
||||
{
|
||||
spin_lock_init(&q->bpoll_lock);
|
||||
q->bpoll_state = CXGB_POLL_STATE_IDLE;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_lock_napi(struct sge_rspq *q)
|
||||
{
|
||||
bool rc = true;
|
||||
|
||||
spin_lock(&q->bpoll_lock);
|
||||
if (q->bpoll_state & CXGB_POLL_LOCKED) {
|
||||
q->bpoll_state |= CXGB_POLL_STATE_NAPI_YIELD;
|
||||
rc = false;
|
||||
} else {
|
||||
q->bpoll_state = CXGB_POLL_STATE_NAPI;
|
||||
}
|
||||
spin_unlock(&q->bpoll_lock);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_unlock_napi(struct sge_rspq *q)
|
||||
{
|
||||
bool rc = false;
|
||||
|
||||
spin_lock(&q->bpoll_lock);
|
||||
if (q->bpoll_state & CXGB_POLL_STATE_POLL_YIELD)
|
||||
rc = true;
|
||||
q->bpoll_state = CXGB_POLL_STATE_IDLE;
|
||||
spin_unlock(&q->bpoll_lock);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_lock_poll(struct sge_rspq *q)
|
||||
{
|
||||
bool rc = true;
|
||||
|
||||
spin_lock_bh(&q->bpoll_lock);
|
||||
if (q->bpoll_state & CXGB_POLL_LOCKED) {
|
||||
q->bpoll_state |= CXGB_POLL_STATE_POLL_YIELD;
|
||||
rc = false;
|
||||
} else {
|
||||
q->bpoll_state |= CXGB_POLL_STATE_POLL;
|
||||
}
|
||||
spin_unlock_bh(&q->bpoll_lock);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_unlock_poll(struct sge_rspq *q)
|
||||
{
|
||||
bool rc = false;
|
||||
|
||||
spin_lock_bh(&q->bpoll_lock);
|
||||
if (q->bpoll_state & CXGB_POLL_STATE_POLL_YIELD)
|
||||
rc = true;
|
||||
q->bpoll_state = CXGB_POLL_STATE_IDLE;
|
||||
spin_unlock_bh(&q->bpoll_lock);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_busy_polling(struct sge_rspq *q)
|
||||
{
|
||||
return q->bpoll_state & CXGB_POLL_USER_PEND;
|
||||
}
|
||||
#else
|
||||
static inline void cxgb_busy_poll_init_lock(struct sge_rspq *q)
|
||||
{
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_lock_napi(struct sge_rspq *q)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_unlock_napi(struct sge_rspq *q)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_lock_poll(struct sge_rspq *q)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_unlock_poll(struct sge_rspq *q)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool cxgb_poll_busy_polling(struct sge_rspq *q)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
|
||||
void t4_os_portmod_changed(const struct adapter *adap, int port_id);
|
||||
void t4_os_link_changed(struct adapter *adap, int port_id, int link_stat);
|
||||
|
||||
|
|
@ -908,6 +1020,7 @@ irqreturn_t t4_sge_intr_msix(int irq, void *cookie);
|
|||
int t4_sge_init(struct adapter *adap);
|
||||
void t4_sge_start(struct adapter *adap);
|
||||
void t4_sge_stop(struct adapter *adap);
|
||||
int cxgb_busy_poll(struct napi_struct *napi);
|
||||
extern int dbfifo_int_thresh;
|
||||
|
||||
#define for_each_port(adapter, iter) \
|
||||
|
|
|
|||
|
|
@ -923,8 +923,14 @@ static void quiesce_rx(struct adapter *adap)
|
|||
for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
|
||||
struct sge_rspq *q = adap->sge.ingr_map[i];
|
||||
|
||||
if (q && q->handler)
|
||||
if (q && q->handler) {
|
||||
napi_disable(&q->napi);
|
||||
local_bh_disable();
|
||||
while (!cxgb_poll_lock_napi(q))
|
||||
mdelay(1);
|
||||
local_bh_enable();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -940,8 +946,10 @@ static void enable_rx(struct adapter *adap)
|
|||
|
||||
if (!q)
|
||||
continue;
|
||||
if (q->handler)
|
||||
if (q->handler) {
|
||||
cxgb_busy_poll_init_lock(q);
|
||||
napi_enable(&q->napi);
|
||||
}
|
||||
/* 0-increment GTS to start the timer and enable interrupts */
|
||||
t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
|
||||
SEINTARM_V(q->intr_params) |
|
||||
|
|
@ -4563,6 +4571,10 @@ static const struct net_device_ops cxgb4_netdev_ops = {
|
|||
#ifdef CONFIG_NET_POLL_CONTROLLER
|
||||
.ndo_poll_controller = cxgb_netpoll,
|
||||
#endif
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
.ndo_busy_poll = cxgb_busy_poll,
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
void t4_fatal_err(struct adapter *adap)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,9 @@
|
|||
#include <linux/export.h>
|
||||
#include <net/ipv6.h>
|
||||
#include <net/tcp.h>
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
#include <net/busy_poll.h>
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
#include "cxgb4.h"
|
||||
#include "t4_regs.h"
|
||||
#include "t4_values.h"
|
||||
|
|
@ -1720,6 +1723,7 @@ static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
|
|||
skb->truesize += skb->data_len;
|
||||
skb->ip_summed = CHECKSUM_UNNECESSARY;
|
||||
skb_record_rx_queue(skb, rxq->rspq.idx);
|
||||
skb_mark_napi_id(skb, &rxq->rspq.napi);
|
||||
if (rxq->rspq.netdev->features & NETIF_F_RXHASH)
|
||||
skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
|
||||
PKT_HASH_TYPE_L3);
|
||||
|
|
@ -1763,6 +1767,7 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
|
|||
csum_ok = pkt->csum_calc && !pkt->err_vec &&
|
||||
(q->netdev->features & NETIF_F_RXCSUM);
|
||||
if ((pkt->l2info & htonl(RXF_TCP_F)) &&
|
||||
!(cxgb_poll_busy_polling(q)) &&
|
||||
(q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
|
||||
do_gro(rxq, si, pkt);
|
||||
return 0;
|
||||
|
|
@ -1801,6 +1806,7 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
|
|||
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
|
||||
rxq->stats.vlan_ex++;
|
||||
}
|
||||
skb_mark_napi_id(skb, &q->napi);
|
||||
netif_receive_skb(skb);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1963,6 +1969,38 @@ static int process_responses(struct sge_rspq *q, int budget)
|
|||
return budget - budget_left;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
int cxgb_busy_poll(struct napi_struct *napi)
|
||||
{
|
||||
struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
|
||||
unsigned int params, work_done;
|
||||
u32 val;
|
||||
|
||||
if (!cxgb_poll_lock_poll(q))
|
||||
return LL_FLUSH_BUSY;
|
||||
|
||||
work_done = process_responses(q, 4);
|
||||
params = QINTR_TIMER_IDX(TIMERREG_COUNTER0_X) | QINTR_CNT_EN;
|
||||
q->next_intr_params = params;
|
||||
val = CIDXINC_V(work_done) | SEINTARM_V(params);
|
||||
|
||||
/* If we don't have access to the new User GTS (T5+), use the old
|
||||
* doorbell mechanism; otherwise use the new BAR2 mechanism.
|
||||
*/
|
||||
if (unlikely(!q->bar2_addr))
|
||||
t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
|
||||
val | INGRESSQID_V((u32)q->cntxt_id));
|
||||
else {
|
||||
writel(val | INGRESSQID_V(q->bar2_qid),
|
||||
q->bar2_addr + SGE_UDB_GTS);
|
||||
wmb();
|
||||
}
|
||||
|
||||
cxgb_poll_unlock_poll(q);
|
||||
return work_done;
|
||||
}
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
|
||||
/**
|
||||
* napi_rx_handler - the NAPI handler for Rx processing
|
||||
* @napi: the napi instance
|
||||
|
|
@ -1978,9 +2016,13 @@ static int napi_rx_handler(struct napi_struct *napi, int budget)
|
|||
{
|
||||
unsigned int params;
|
||||
struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
|
||||
int work_done = process_responses(q, budget);
|
||||
int work_done;
|
||||
u32 val;
|
||||
|
||||
if (!cxgb_poll_lock_napi(q))
|
||||
return budget;
|
||||
|
||||
work_done = process_responses(q, budget);
|
||||
if (likely(work_done < budget)) {
|
||||
int timer_index;
|
||||
|
||||
|
|
@ -2018,6 +2060,7 @@ static int napi_rx_handler(struct napi_struct *napi, int budget)
|
|||
q->bar2_addr + SGE_UDB_GTS);
|
||||
wmb();
|
||||
}
|
||||
cxgb_poll_unlock_napi(q);
|
||||
return work_done;
|
||||
}
|
||||
|
||||
|
|
@ -2341,6 +2384,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
|
|||
goto err;
|
||||
|
||||
netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
|
||||
napi_hash_add(&iq->napi);
|
||||
iq->cur_desc = iq->desc;
|
||||
iq->cidx = 0;
|
||||
iq->gen = 1;
|
||||
|
|
@ -2598,6 +2642,7 @@ static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
|
|||
rq->cntxt_id, fl_id, 0xffff);
|
||||
dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len,
|
||||
rq->desc, rq->phys_addr);
|
||||
napi_hash_del(&rq->napi);
|
||||
netif_napi_del(&rq->napi);
|
||||
rq->netdev = NULL;
|
||||
rq->cntxt_id = rq->abs_id = 0;
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@
|
|||
|
||||
/* GTS register */
|
||||
#define SGE_TIMERREGS 6
|
||||
#define TIMERREG_COUNTER0_X 0
|
||||
|
||||
/* T5 and later support a new BAR2-based doorbell mechanism for Egress Queues.
|
||||
* The User Doorbells are each 128 bytes in length with a Simple Doorbell at
|
||||
|
|
|
|||
Loading…
Reference in a new issue