diff options
author | Björn Töpel <bjorn.topel@intel.com> | 2020-05-20 21:20:55 +0200 |
---|---|---|
committer | Alexei Starovoitov <ast@kernel.org> | 2020-05-21 17:31:26 -0700 |
commit | be1222b585fdc410b8c1dbcc57dd03a00f04eff5 (patch) | |
tree | 6c910bd9b103654a35ce42cce7caa26f016ed912 /drivers/net/ethernet/intel/i40e/i40e_xsk.c | |
parent | e1675f97367bed74d4dcfe08de9ce9b5d6b288c1 (diff) |
i40e: Separate kernel allocated rx_bi rings from AF_XDP rings
Continuing the path to support MEM_TYPE_XSK_BUFF_POOL, the AF_XDP
zero-copy/sk_buff rx_bi rings are now separate. Functions to properly
allocate the different rings are added as well.
v3->v4: Made i40e_fd_handle_status() static. (kbuild test robot)
v4->v5: Fix kdoc for i40e_clean_programming_status(). (Jakub)
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Cc: intel-wired-lan@lists.osuosl.org
Link: https://lore.kernel.org/bpf/20200520192103.355233-8-bjorn.topel@gmail.com
Diffstat (limited to 'drivers/net/ethernet/intel/i40e/i40e_xsk.c')
-rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_xsk.c | 74 |
1 files changed, 40 insertions, 34 deletions
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index d84ec92f8538..4fca52a30ea4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -9,9 +9,23 @@ #include "i40e_txrx_common.h" #include "i40e_xsk.h" -static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx) +int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring) { - return &rx_ring->rx_bi[idx]; + unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count; + + rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL); + return rx_ring->rx_bi_zc ? 0 : -ENOMEM; +} + +void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring) +{ + memset(rx_ring->rx_bi_zc, 0, + sizeof(*rx_ring->rx_bi_zc) * rx_ring->count); +} + +static struct i40e_rx_buffer_zc *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx) +{ + return &rx_ring->rx_bi_zc[idx]; } /** @@ -238,7 +252,7 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) } /** - * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer + * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer_zc * @rx_ring: Rx ring * @bi: Rx buffer to populate * @@ -248,7 +262,7 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) * Returns true for a successful allocation, false otherwise **/ static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi) + struct i40e_rx_buffer_zc *bi) { struct xdp_umem *umem = rx_ring->xsk_umem; void *addr = bi->addr; @@ -279,7 +293,7 @@ static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring, } /** - * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer + * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer_zc * @rx_ring: Rx ring * @bi: Rx buffer to populate * @@ -289,7 +303,7 @@ static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring, * Returns true for a successful allocation, false otherwise **/ static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi) + struct i40e_rx_buffer_zc *bi) { struct xdp_umem *umem = rx_ring->xsk_umem; u64 handle, hr; @@ -318,11 +332,11 @@ static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring, static __always_inline bool __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count, bool alloc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi)) + struct i40e_rx_buffer_zc *bi)) { u16 ntu = rx_ring->next_to_use; union i40e_rx_desc *rx_desc; - struct i40e_rx_buffer *bi; + struct i40e_rx_buffer_zc *bi; bool ok = true; rx_desc = I40E_RX_DESC(rx_ring, ntu); @@ -402,10 +416,11 @@ static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count) * * Returns the received Rx buffer **/ -static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring, - const unsigned int size) +static struct i40e_rx_buffer_zc *i40e_get_rx_buffer_zc( + struct i40e_ring *rx_ring, + const unsigned int size) { - struct i40e_rx_buffer *bi; + struct i40e_rx_buffer_zc *bi; bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); @@ -427,10 +442,10 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring, * recycle queue (next_to_alloc). **/ static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *old_bi) + struct i40e_rx_buffer_zc *old_bi) { - struct i40e_rx_buffer *new_bi = i40e_rx_bi(rx_ring, - rx_ring->next_to_alloc); + struct i40e_rx_buffer_zc *new_bi = i40e_rx_bi(rx_ring, + rx_ring->next_to_alloc); u16 nta = rx_ring->next_to_alloc; /* update, and store next to alloc */ @@ -452,7 +467,7 @@ static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring, **/ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) { - struct i40e_rx_buffer *bi; + struct i40e_rx_buffer_zc *bi; struct i40e_ring *rx_ring; u64 hr, mask; u16 nta; @@ -490,7 +505,7 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) * Returns the skb, or NULL on failure. **/ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, - struct i40e_rx_buffer *bi, + struct i40e_rx_buffer_zc *bi, struct xdp_buff *xdp) { unsigned int metasize = xdp->data - xdp->data_meta; @@ -514,19 +529,6 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, } /** - * i40e_inc_ntc: Advance the next_to_clean index - * @rx_ring: Rx ring - **/ -static void i40e_inc_ntc(struct i40e_ring *rx_ring) -{ - u32 ntc = rx_ring->next_to_clean + 1; - - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - prefetch(I40E_RX_DESC(rx_ring, ntc)); -} - -/** * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring * @rx_ring: Rx ring * @budget: NAPI budget @@ -547,7 +549,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) xdp.frame_sz = xsk_umem_xdp_frame_sz(umem); while (likely(total_rx_packets < (unsigned int)budget)) { - struct i40e_rx_buffer *bi; + struct i40e_rx_buffer_zc *bi; union i40e_rx_desc *rx_desc; unsigned int size; u64 qword; @@ -568,14 +570,18 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) */ dma_rmb(); - bi = i40e_clean_programming_status(rx_ring, rx_desc, - qword); - if (unlikely(bi)) { + if (i40e_rx_is_programming_status(qword)) { + i40e_clean_programming_status(rx_ring, + rx_desc->raw.qword[0], + qword); + bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); + i40e_inc_ntc(rx_ring); i40e_reuse_rx_buffer_zc(rx_ring, bi); cleaned_count++; continue; } + bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean); size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; if (!size) @@ -832,7 +838,7 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring) u16 i; for (i = 0; i < rx_ring->count; i++) { - struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i); + struct i40e_rx_buffer_zc *rx_bi = i40e_rx_bi(rx_ring, i); if (!rx_bi->addr) continue; |