net/mlx5: fix split buffer Rx
Checks
Commit Message
Routine to lookup LKey on Rx was assuming that the mbuf address
always belongs to a single mempool: the one associated with an RxQ
or the MPRQ mempool. This assumption is false for split buffers case.
A wrong LKey was looked up, resulting in completion errors.
Modify lookup routines to lookup LKey in the mbuf->pool
for non-MPRQ cases both on Rx datapath and on queue initialization.
Fixes: fec28ca0e3a9 ("net/mlx5: support mempool registration")
Signed-off-by: Dmitry Kozlyuk <dkozlyuk@nvidia.com>
Reviewed-by: Matan Azrad <matan@nvidia.com>
Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
drivers/net/mlx5/mlx5_rx.c | 5 ++++-
drivers/net/mlx5/mlx5_rx.h | 37 +++++++++++++++++++++++++++++++++++--
2 files changed, 39 insertions(+), 3 deletions(-)
Comments
On 11/1/2021 9:56 AM, Dmitry Kozlyuk wrote:
> Routine to lookup LKey on Rx was assuming that the mbuf address
> always belongs to a single mempool: the one associated with an RxQ
> or the MPRQ mempool. This assumption is false for split buffers case.
> A wrong LKey was looked up, resulting in completion errors.
> Modify lookup routines to lookup LKey in the mbuf->pool
> for non-MPRQ cases both on Rx datapath and on queue initialization.
>
> Fixes: fec28ca0e3a9 ("net/mlx5: support mempool registration")
>
> Signed-off-by: Dmitry Kozlyuk <dkozlyuk@nvidia.com>
> Reviewed-by: Matan Azrad <matan@nvidia.com>
> Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
> ---
> drivers/net/mlx5/mlx5_rx.c | 5 ++++-
> drivers/net/mlx5/mlx5_rx.h | 37 +++++++++++++++++++++++++++++++++++--
> 2 files changed, 39 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
> index 258a645314..7296e81534 100644
> --- a/drivers/net/mlx5/mlx5_rx.c
> +++ b/drivers/net/mlx5/mlx5_rx.c
> @@ -356,6 +356,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
> volatile struct mlx5_wqe_data_seg *scat;
> uintptr_t addr;
> uint32_t byte_count;
> + uint32_t lkey;
>
> if (mlx5_rxq_mprq_enabled(rxq)) {
> struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
> @@ -366,6 +367,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
> 1 << rxq->strd_num_n);
> byte_count = (1 << rxq->strd_sz_n) *
> (1 << rxq->strd_num_n);
> + lkey = mlx5_rx_addr2mr(rxq, addr);
> } else {
> struct rte_mbuf *buf = (*rxq->elts)[i];
>
> @@ -373,13 +375,14 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
> rxq->wqes)[i];
> addr = rte_pktmbuf_mtod(buf, uintptr_t);
> byte_count = DATA_LEN(buf);
> + lkey = mlx5_rx_mb2mr(rxq, buf);
> }
> /* scat->addr must be able to store a pointer. */
> MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
> *scat = (struct mlx5_wqe_data_seg){
> .addr = rte_cpu_to_be_64(addr),
> .byte_count = rte_cpu_to_be_32(byte_count),
> - .lkey = mlx5_rx_addr2mr(rxq, addr),
> + .lkey = lkey,
> };
> }
> rxq->consumed_strd = 0;
> diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
> index 4952fe1455..5903776e7a 100644
> --- a/drivers/net/mlx5/mlx5_rx.h
> +++ b/drivers/net/mlx5/mlx5_rx.h
> @@ -266,7 +266,7 @@ uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
> static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
>
> /**
> - * Query LKey from a packet buffer for Rx. No need to flush local caches
> + * Query LKey for an address on Rx. No need to flush local caches
> * as the Rx mempool database entries are valid for the lifetime of the queue.
> *
> * @param rxq
> @@ -301,7 +301,40 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
> mr_ctrl, mp, addr);
> }
>
> -#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
> +/**
> + * Query LKey from a packet buffer for Rx. No need to flush local caches
> + * as the Rx mempool database entries are valid for the lifetime of the queue.
> + *
> + * @param rxq
> + * Pointer to Rx queue structure.
> + * @param mb
> + * Buffer to search the address of.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + * This function always succeeds on valid input.
> + */
> +static __rte_always_inline uint32_t
> +mlx5_rx_mb2mr(struct mlx5_rxq_data *rxq, struct rte_mbuf *mb)
> +{
> + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
> + uintptr_t addr = (uintptr_t)mb->buf_addr;
> + struct mlx5_rxq_ctrl *rxq_ctrl;
> + uint32_t lkey;
> +
> + /* Linear search on MR cache array. */
> + lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
> + MLX5_MR_CACHE_N, addr);
> + if (likely(lkey != UINT32_MAX))
> + return lkey;
> + /*
> + * Slower search in the mempool database on miss.
> + * During queue creation rxq->sh is not yet set, so we use rxq_ctrl.
> + */
> + rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
> + return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
> + mr_ctrl, mb->pool, addr);
Hi Dmitry,
I am getting build errors with this patch, although I can see CI build is passed,
can you please check if I am missing something, build error is:
../drivers/net/mlx5/mlx5_rx.h: In function ‘mlx5_rx_mb2mr’:
../drivers/net/mlx5/mlx5_rx.h:354:47: error: ‘struct mlx5_rxq_ctrl’ has no member named ‘priv’
354 | return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
| ^~
[...]
> Hi Dmitry,
>
> I am getting build errors with this patch, although I can see CI build is
> passed,
> can you please check if I am missing something, build error is:
>
> ../drivers/net/mlx5/mlx5_rx.h: In function ‘mlx5_rx_mb2mr’:
> ../drivers/net/mlx5/mlx5_rx.h:354:47: error: ‘struct mlx5_rxq_ctrl’ has no
> member named ‘priv’
> 354 | return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev-
> >mr_scache,
> | ^~
Hi Ferruh,
I sent a v2 based on next-net-mlx which Raslan has applied already.
It also applies cleanly to next-net if needed.
On 11/8/2021 1:10 PM, Ferruh Yigit wrote:
> On 11/1/2021 9:56 AM, Dmitry Kozlyuk wrote:
>> Routine to lookup LKey on Rx was assuming that the mbuf address
>> always belongs to a single mempool: the one associated with an RxQ
>> or the MPRQ mempool. This assumption is false for split buffers case.
>> A wrong LKey was looked up, resulting in completion errors.
>> Modify lookup routines to lookup LKey in the mbuf->pool
>> for non-MPRQ cases both on Rx datapath and on queue initialization.
>>
>> Fixes: fec28ca0e3a9 ("net/mlx5: support mempool registration")
>>
>> Signed-off-by: Dmitry Kozlyuk <dkozlyuk@nvidia.com>
>> Reviewed-by: Matan Azrad <matan@nvidia.com>
>> Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
>> ---
>> drivers/net/mlx5/mlx5_rx.c | 5 ++++-
>> drivers/net/mlx5/mlx5_rx.h | 37 +++++++++++++++++++++++++++++++++++--
>> 2 files changed, 39 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
>> index 258a645314..7296e81534 100644
>> --- a/drivers/net/mlx5/mlx5_rx.c
>> +++ b/drivers/net/mlx5/mlx5_rx.c
>> @@ -356,6 +356,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
>> volatile struct mlx5_wqe_data_seg *scat;
>> uintptr_t addr;
>> uint32_t byte_count;
>> + uint32_t lkey;
>> if (mlx5_rxq_mprq_enabled(rxq)) {
>> struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
>> @@ -366,6 +367,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
>> 1 << rxq->strd_num_n);
>> byte_count = (1 << rxq->strd_sz_n) *
>> (1 << rxq->strd_num_n);
>> + lkey = mlx5_rx_addr2mr(rxq, addr);
>> } else {
>> struct rte_mbuf *buf = (*rxq->elts)[i];
>> @@ -373,13 +375,14 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
>> rxq->wqes)[i];
>> addr = rte_pktmbuf_mtod(buf, uintptr_t);
>> byte_count = DATA_LEN(buf);
>> + lkey = mlx5_rx_mb2mr(rxq, buf);
>> }
>> /* scat->addr must be able to store a pointer. */
>> MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
>> *scat = (struct mlx5_wqe_data_seg){
>> .addr = rte_cpu_to_be_64(addr),
>> .byte_count = rte_cpu_to_be_32(byte_count),
>> - .lkey = mlx5_rx_addr2mr(rxq, addr),
>> + .lkey = lkey,
>> };
>> }
>> rxq->consumed_strd = 0;
>> diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
>> index 4952fe1455..5903776e7a 100644
>> --- a/drivers/net/mlx5/mlx5_rx.h
>> +++ b/drivers/net/mlx5/mlx5_rx.h
>> @@ -266,7 +266,7 @@ uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
>> static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
>> /**
>> - * Query LKey from a packet buffer for Rx. No need to flush local caches
>> + * Query LKey for an address on Rx. No need to flush local caches
>> * as the Rx mempool database entries are valid for the lifetime of the queue.
>> *
>> * @param rxq
>> @@ -301,7 +301,40 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
>> mr_ctrl, mp, addr);
>> }
>> -#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
>> +/**
>> + * Query LKey from a packet buffer for Rx. No need to flush local caches
>> + * as the Rx mempool database entries are valid for the lifetime of the queue.
>> + *
>> + * @param rxq
>> + * Pointer to Rx queue structure.
>> + * @param mb
>> + * Buffer to search the address of.
>> + *
>> + * @return
>> + * Searched LKey on success, UINT32_MAX on no match.
>> + * This function always succeeds on valid input.
>> + */
>> +static __rte_always_inline uint32_t
>> +mlx5_rx_mb2mr(struct mlx5_rxq_data *rxq, struct rte_mbuf *mb)
>> +{
>> + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
>> + uintptr_t addr = (uintptr_t)mb->buf_addr;
>> + struct mlx5_rxq_ctrl *rxq_ctrl;
>> + uint32_t lkey;
>> +
>> + /* Linear search on MR cache array. */
>> + lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
>> + MLX5_MR_CACHE_N, addr);
>> + if (likely(lkey != UINT32_MAX))
>> + return lkey;
>> + /*
>> + * Slower search in the mempool database on miss.
>> + * During queue creation rxq->sh is not yet set, so we use rxq_ctrl.
>> + */
>> + rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
>> + return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
>> + mr_ctrl, mb->pool, addr);
>
> Hi Dmitry,
>
> I am getting build errors with this patch, although I can see CI build is passed,
> can you please check if I am missing something, build error is:
>
> ../drivers/net/mlx5/mlx5_rx.h: In function ‘mlx5_rx_mb2mr’:
> ../drivers/net/mlx5/mlx5_rx.h:354:47: error: ‘struct mlx5_rxq_ctrl’ has no member named ‘priv’
> 354 | return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
> | ^~
>
>
My bad, Raslan pointed out that there is a v2 already, will continue with that one.
@@ -356,6 +356,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
volatile struct mlx5_wqe_data_seg *scat;
uintptr_t addr;
uint32_t byte_count;
+ uint32_t lkey;
if (mlx5_rxq_mprq_enabled(rxq)) {
struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
@@ -366,6 +367,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
1 << rxq->strd_num_n);
byte_count = (1 << rxq->strd_sz_n) *
(1 << rxq->strd_num_n);
+ lkey = mlx5_rx_addr2mr(rxq, addr);
} else {
struct rte_mbuf *buf = (*rxq->elts)[i];
@@ -373,13 +375,14 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
rxq->wqes)[i];
addr = rte_pktmbuf_mtod(buf, uintptr_t);
byte_count = DATA_LEN(buf);
+ lkey = mlx5_rx_mb2mr(rxq, buf);
}
/* scat->addr must be able to store a pointer. */
MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
*scat = (struct mlx5_wqe_data_seg){
.addr = rte_cpu_to_be_64(addr),
.byte_count = rte_cpu_to_be_32(byte_count),
- .lkey = mlx5_rx_addr2mr(rxq, addr),
+ .lkey = lkey,
};
}
rxq->consumed_strd = 0;
@@ -266,7 +266,7 @@ uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
/**
- * Query LKey from a packet buffer for Rx. No need to flush local caches
+ * Query LKey for an address on Rx. No need to flush local caches
* as the Rx mempool database entries are valid for the lifetime of the queue.
*
* @param rxq
@@ -301,7 +301,40 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
mr_ctrl, mp, addr);
}
-#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches
+ * as the Rx mempool database entries are valid for the lifetime of the queue.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param mb
+ * Buffer to search the address of.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ * This function always succeeds on valid input.
+ */
+static __rte_always_inline uint32_t
+mlx5_rx_mb2mr(struct mlx5_rxq_data *rxq, struct rte_mbuf *mb)
+{
+ struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ uintptr_t addr = (uintptr_t)mb->buf_addr;
+ struct mlx5_rxq_ctrl *rxq_ctrl;
+ uint32_t lkey;
+
+ /* Linear search on MR cache array. */
+ lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX5_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /*
+ * Slower search in the mempool database on miss.
+ * During queue creation rxq->sh is not yet set, so we use rxq_ctrl.
+ */
+ rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+ return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
+ mr_ctrl, mb->pool, addr);
+}
/**
* Convert timestamp from HW format to linear counter