From patchwork Mon Jul 22 09:13:10 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Matan Azrad X-Patchwork-Id: 56848 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id E28A71BE82; Mon, 22 Jul 2019 11:14:22 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id A644F1BDF4 for ; Mon, 22 Jul 2019 11:13:30 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE2 (envelope-from matan@mellanox.com) with ESMTPS (AES256-SHA encrypted); 22 Jul 2019 12:13:24 +0300 Received: from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx [10.210.16.112]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x6M9DMjm010084; Mon, 22 Jul 2019 12:13:24 +0300 From: Matan Azrad To: Shahaf Shuler , Yongseok Koh , Viacheslav Ovsiienko Cc: dev@dpdk.org, Dekel Peled Date: Mon, 22 Jul 2019 09:13:10 +0000 Message-Id: <1563786795-14027-24-git-send-email-matan@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1563786795-14027-1-git-send-email-matan@mellanox.com> References: <1563786795-14027-1-git-send-email-matan@mellanox.com> Subject: [dpdk-dev] [PATCH 23/28] net/mlx5: replace the external mbuf shared memory X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" As an arrangement to the LRO support when a packet can consume all the stride memory, the external mbuf shared information cannot be anymore in the end of the stride, because the HW may write the packet data to all the stride memory. Move the shared information memory from the stride to the control memory of the external mbuf. Signed-off-by: Matan Azrad Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5_rxq.c | 22 +++++++++++++++------- drivers/net/mlx5/mlx5_rxtx.c | 19 +++++++++++-------- drivers/net/mlx5/mlx5_rxtx.h | 12 +++++++++++- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index c03ecf8..344ff90 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -1361,14 +1361,22 @@ struct mlx5_rxq_obj * * Callback function to initialize mbufs for Multi-Packet RQ. */ static inline void -mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg __rte_unused, +mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg, void *_m, unsigned int i __rte_unused) { struct mlx5_mprq_buf *buf = _m; + struct rte_mbuf_ext_shared_info *shinfo; + unsigned int strd_n = (unsigned int)(uintptr_t)opaque_arg; + unsigned int j; memset(_m, 0, sizeof(*buf)); buf->mp = mp; rte_atomic16_set(&buf->refcnt, 1); + for (j = 0; j != strd_n; ++j) { + shinfo = &buf->shinfos[j]; + shinfo->free_cb = mlx5_mprq_buf_free_cb; + shinfo->fcb_opaque = buf; + } } /** @@ -1463,7 +1471,8 @@ struct mlx5_rxq_obj * } assert(strd_num_n && strd_sz_n); buf_len = (1 << strd_num_n) * (1 << strd_sz_n); - obj_size = buf_len + sizeof(struct mlx5_mprq_buf); + obj_size = sizeof(struct mlx5_mprq_buf) + buf_len + (1 << strd_num_n) * + sizeof(struct rte_mbuf_ext_shared_info) + RTE_PKTMBUF_HEADROOM; /* * Received packets can be either memcpy'd or externally referenced. In * case that the packet is attached to an mbuf as an external buffer, as @@ -1508,7 +1517,8 @@ struct mlx5_rxq_obj * } snprintf(name, sizeof(name), "port-%u-mprq", dev->data->port_id); mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ, - 0, NULL, NULL, mlx5_mprq_buf_init, NULL, + 0, NULL, NULL, mlx5_mprq_buf_init, + (void *)(uintptr_t)(1 << strd_num_n), dev->device->numa_node, 0); if (mp == NULL) { DRV_LOG(ERR, @@ -1594,10 +1604,8 @@ struct mlx5_rxq_ctrl * * Otherwise, enable Rx scatter if necessary. */ assert(mb_len >= RTE_PKTMBUF_HEADROOM); - mprq_stride_size = - dev->data->dev_conf.rxmode.max_rx_pkt_len + - sizeof(struct rte_mbuf_ext_shared_info) + - RTE_PKTMBUF_HEADROOM; + mprq_stride_size = dev->data->dev_conf.rxmode.max_rx_pkt_len + + RTE_PKTMBUF_HEADROOM; if (mprq_en && desc > (1U << config->mprq.stride_num_n) && mprq_stride_size <= (1U << config->mprq.max_stride_size_n)) { diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 584da3e..241e01b 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -100,7 +100,8 @@ enum mlx5_txcmp_code { volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res); static __rte_always_inline void -mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx); +mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx, + const unsigned int strd_n); static int mlx5_queue_state_modify(struct rte_eth_dev *dev, @@ -756,7 +757,8 @@ enum mlx5_txcmp_code { scat = &((volatile struct mlx5_wqe_mprq *) rxq->wqes)[i].dseg; - addr = (uintptr_t)mlx5_mprq_buf_addr(buf); + addr = (uintptr_t)mlx5_mprq_buf_addr(buf, + 1 << rxq->strd_num_n); byte_count = (1 << rxq->strd_sz_n) * (1 << rxq->strd_num_n); } else { @@ -1392,7 +1394,8 @@ enum mlx5_txcmp_code { } static inline void -mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx) +mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx, + const unsigned int strd_n) { struct mlx5_mprq_buf *rep = rxq->mprq_repl; volatile struct mlx5_wqe_data_seg *wqe = @@ -1403,7 +1406,7 @@ enum mlx5_txcmp_code { /* Replace MPRQ buf. */ (*rxq->mprq_bufs)[rq_idx] = rep; /* Replace WQE. */ - addr = mlx5_mprq_buf_addr(rep); + addr = mlx5_mprq_buf_addr(rep, strd_n); wqe->addr = rte_cpu_to_be_64((uintptr_t)addr); /* If there's only one MR, no need to replace LKey in WQE. */ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) @@ -1459,7 +1462,7 @@ enum mlx5_txcmp_code { if (consumed_strd == strd_n) { /* Replace WQE only if the buffer is still in use. */ if (rte_atomic16_read(&buf->refcnt) > 1) { - mprq_buf_replace(rxq, rq_ci & wq_mask); + mprq_buf_replace(rxq, rq_ci & wq_mask, strd_n); /* Release the old buffer. */ mlx5_mprq_buf_free(buf); } else if (unlikely(rxq->mprq_repl == NULL)) { @@ -1521,7 +1524,7 @@ enum mlx5_txcmp_code { if (rxq->crc_present) len -= RTE_ETHER_CRC_LEN; offset = strd_idx * strd_sz + strd_shift; - addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf), offset); + addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n), offset); /* Initialize the offload flag. */ pkt->ol_flags = 0; /* @@ -1557,8 +1560,8 @@ enum mlx5_txcmp_code { */ buf_iova = rte_mempool_virt2iova(buf) + RTE_PTR_DIFF(addr, buf); - shinfo = rte_pktmbuf_ext_shinfo_init_helper(addr, - &buf_len, mlx5_mprq_buf_free_cb, buf); + shinfo = &buf->shinfos[strd_idx]; + rte_mbuf_ext_refcnt_set(shinfo, 1); /* * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when * attaching the stride to mbuf and more offload flags diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index ed5f637..bbd9b31 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -75,10 +75,20 @@ struct mlx5_mprq_buf { struct rte_mempool *mp; rte_atomic16_t refcnt; /* Atomically accessed refcnt. */ uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */ + struct rte_mbuf_ext_shared_info shinfos[]; + /* + * Shared information per stride. + * More memory will be allocated for the first stride head-room and for + * the strides data. + */ } __rte_cache_aligned; /* Get pointer to the first stride. */ -#define mlx5_mprq_buf_addr(ptr) ((ptr) + 1) +#define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \ + sizeof(struct mlx5_mprq_buf) + \ + (strd_n) * \ + sizeof(struct rte_mbuf_ext_shared_info) + \ + RTE_PKTMBUF_HEADROOM)) #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9