diff mbox series

[07/15] net/dpaa2: use internal mempool for SG table

Message ID 20220928052516.1279442-8-g.singh@nxp.com (mailing list archive)
State Superseded, archived
Delegated to: Ferruh Yigit
Headers show
Series DPAA and DPAA2 driver changes | expand

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Gagandeep Singh Sept. 28, 2022, 5:25 a.m. UTC
Creating and using driver's mempool for
allocating the SG table memory required for
FD creation instead of relying on user mempool.

Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
---
 drivers/net/dpaa2/dpaa2_ethdev.c | 14 ++++++++++++++
 drivers/net/dpaa2/dpaa2_ethdev.h |  9 +++++++++
 drivers/net/dpaa2/dpaa2_rxtx.c   | 13 ++++++-------
 3 files changed, 29 insertions(+), 7 deletions(-)

Comments

Ferruh Yigit Oct. 5, 2022, 2:20 p.m. UTC | #1
On 9/28/2022 6:25 AM, Gagandeep Singh wrote:
> Creating and using driver's mempool for
> allocating the SG table memory required for
> FD creation instead of relying on user mempool.
> 

As far as I can see this is in the Tx path, can you please explain why 
driver need an internal pktmbuf pool?

And shouldn't mempool needs to be freed on driver close and release.

Same comment applies for dpaa version of the patch (12/15).

> Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
> ---
>   drivers/net/dpaa2/dpaa2_ethdev.c | 14 ++++++++++++++
>   drivers/net/dpaa2/dpaa2_ethdev.h |  9 +++++++++
>   drivers/net/dpaa2/dpaa2_rxtx.c   | 13 ++++++-------
>   3 files changed, 29 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c
> index 37a8b43114..c7aae70300 100644
> --- a/drivers/net/dpaa2/dpaa2_ethdev.c
> +++ b/drivers/net/dpaa2/dpaa2_ethdev.c
> @@ -78,6 +78,8 @@ bool dpaa2_enable_err_queue;
>   #define MAX_NB_RX_DESC		11264
>   int total_nb_rx_desc;
>   
> +struct rte_mempool *dpaa2_tx_sg_pool;
> +
>   struct rte_dpaa2_xstats_name_off {
>   	char name[RTE_ETH_XSTATS_NAME_SIZE];
>   	uint8_t page_id; /* dpni statistics page id */
> @@ -2907,6 +2909,18 @@ rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv,
>   	/* Invoke PMD device initialization function */
>   	diag = dpaa2_dev_init(eth_dev);
>   	if (diag == 0) {
> +		if (!dpaa2_tx_sg_pool) {
> +			dpaa2_tx_sg_pool =
> +				rte_pktmbuf_pool_create("dpaa2_mbuf_tx_sg_pool",
> +				DPAA2_POOL_SIZE,
> +				DPAA2_POOL_CACHE_SIZE, 0,
> +				DPAA2_MAX_SGS * sizeof(struct qbman_sge),
> +				rte_socket_id());
> +			if (dpaa2_tx_sg_pool == NULL) {
> +				DPAA2_PMD_ERR("SG pool creation failed\n");
> +				return -ENOMEM;
> +			}
> +		}
>   		rte_eth_dev_probing_finish(eth_dev);
>   		return 0;
>   	}
> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h b/drivers/net/dpaa2/dpaa2_ethdev.h
> index 32ae762e4a..872dced517 100644
> --- a/drivers/net/dpaa2/dpaa2_ethdev.h
> +++ b/drivers/net/dpaa2/dpaa2_ethdev.h
> @@ -121,6 +121,15 @@
>   #define DPAA2_PKT_TYPE_VLAN_1		0x0160
>   #define DPAA2_PKT_TYPE_VLAN_2		0x0260
>   
> +/* Global pool used by driver for SG list TX */
> +extern struct rte_mempool *dpaa2_tx_sg_pool;
> +/* Maximum SG segments */
> +#define DPAA2_MAX_SGS 128
> +/* SG pool size */
> +#define DPAA2_POOL_SIZE 2048
> +/* SG pool cache size */
> +#define DPAA2_POOL_CACHE_SIZE 256
> +
>   /* enable timestamp in mbuf*/
>   extern bool dpaa2_enable_ts[];
>   extern uint64_t dpaa2_timestamp_rx_dynflag;
> diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
> index bc0e49b0d4..dcd86c4056 100644
> --- a/drivers/net/dpaa2/dpaa2_rxtx.c
> +++ b/drivers/net/dpaa2/dpaa2_rxtx.c
> @@ -403,7 +403,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd,
>   static int __rte_noinline __rte_hot
>   eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
>   		  struct qbman_fd *fd,
> -		  struct rte_mempool *mp, uint16_t bpid)
> +		  uint16_t bpid)
>   {
>   	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
>   	struct qbman_sge *sgt, *sge = NULL;
> @@ -433,12 +433,12 @@ eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
>   		}
>   		DPAA2_SET_FD_OFFSET(fd, offset);
>   	} else {
> -		temp = rte_pktmbuf_alloc(mp);
> +		temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool);
>   		if (temp == NULL) {
>   			DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
>   			return -ENOMEM;
>   		}
> -		DPAA2_SET_ONLY_FD_BPID(fd, bpid);
> +		DPAA2_SET_ONLY_FD_BPID(fd, mempool_to_bpid(dpaa2_tx_sg_pool));
>   		DPAA2_SET_FD_OFFSET(fd, temp->data_off);
>   #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
>   		rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp),
> @@ -1321,9 +1321,10 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
>   
>   			if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) {
>   				if (unlikely((*bufs)->nb_segs > 1)) {
> +					mp = (*bufs)->pool;
>   					if (eth_mbuf_to_sg_fd(*bufs,
>   							      &fd_arr[loop],
> -							      mp, 0))
> +							      mempool_to_bpid(mp)))
>   						goto send_n_return;
>   				} else {
>   					eth_mbuf_to_fd(*bufs,
> @@ -1372,7 +1373,7 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
>   				if (unlikely((*bufs)->nb_segs > 1)) {
>   					if (eth_mbuf_to_sg_fd(*bufs,
>   							&fd_arr[loop],
> -							mp, bpid))
> +							bpid))
>   						goto send_n_return;
>   				} else {
>   					eth_mbuf_to_fd(*bufs,
> @@ -1646,7 +1647,6 @@ dpaa2_dev_tx_multi_txq_ordered(void **queue,
>   			if (unlikely((*bufs)->nb_segs > 1)) {
>   				if (eth_mbuf_to_sg_fd(*bufs,
>   						      &fd_arr[loop],
> -						      mp,
>   						      bpid))
>   					goto send_frames;
>   			} else {
> @@ -1810,7 +1810,6 @@ dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
>   				if (unlikely((*bufs)->nb_segs > 1)) {
>   					if (eth_mbuf_to_sg_fd(*bufs,
>   							      &fd_arr[loop],
> -							      mp,
>   							      bpid))
>   						goto send_n_return;
>   				} else {
Gagandeep Singh Oct. 6, 2022, 8:49 a.m. UTC | #2
Hi

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@amd.com>
> Sent: Wednesday, October 5, 2022 7:51 PM
> To: Gagandeep Singh <G.Singh@nxp.com>; dev@dpdk.org
> Subject: Re: [PATCH 07/15] net/dpaa2: use internal mempool for SG table
> 
> On 9/28/2022 6:25 AM, Gagandeep Singh wrote:
> > Creating and using driver's mempool for allocating the SG table memory
> > required for FD creation instead of relying on user mempool.
> >
> 
> As far as I can see this is in the Tx path, can you please explain why
> driver need an internal pktmbuf pool?
> 
> And shouldn't mempool needs to be freed on driver close and release.
> 
> Same comment applies for dpaa version of the patch (12/15).
> 

In TX path, in case of SG packets, driver need additional SG table memory to store the data
of all segments to pass to HW. Earlier it was dependent on the user's pool to allocate memory
which is wrong. so allocating the new pool in the driver for this purpose.

We are creating only one pool for all the DPAA net devices on first DPAA net device probe, so it can
be free only on last device probe. So will it be worth to free the pool on last device probe as
memory will be released on process kill as well?

> > Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
> > ---
> >   drivers/net/dpaa2/dpaa2_ethdev.c | 14 ++++++++++++++
> >   drivers/net/dpaa2/dpaa2_ethdev.h |  9 +++++++++
> >   drivers/net/dpaa2/dpaa2_rxtx.c   | 13 ++++++-------
> >   3 files changed, 29 insertions(+), 7 deletions(-)
> >
> > diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c
> b/drivers/net/dpaa2/dpaa2_ethdev.c
> > index 37a8b43114..c7aae70300 100644
> > --- a/drivers/net/dpaa2/dpaa2_ethdev.c
> > +++ b/drivers/net/dpaa2/dpaa2_ethdev.c
> > @@ -78,6 +78,8 @@ bool dpaa2_enable_err_queue;
> >   #define MAX_NB_RX_DESC		11264
> >   int total_nb_rx_desc;
> >
> > +struct rte_mempool *dpaa2_tx_sg_pool;
> > +
> >   struct rte_dpaa2_xstats_name_off {
> >   	char name[RTE_ETH_XSTATS_NAME_SIZE];
> >   	uint8_t page_id; /* dpni statistics page id */
> > @@ -2907,6 +2909,18 @@ rte_dpaa2_probe(struct rte_dpaa2_driver
> *dpaa2_drv,
> >   	/* Invoke PMD device initialization function */
> >   	diag = dpaa2_dev_init(eth_dev);
> >   	if (diag == 0) {
> > +		if (!dpaa2_tx_sg_pool) {
> > +			dpaa2_tx_sg_pool =
> > +
> 	rte_pktmbuf_pool_create("dpaa2_mbuf_tx_sg_pool",
> > +				DPAA2_POOL_SIZE,
> > +				DPAA2_POOL_CACHE_SIZE, 0,
> > +				DPAA2_MAX_SGS * sizeof(struct qbman_sge),
> > +				rte_socket_id());
> > +			if (dpaa2_tx_sg_pool == NULL) {
> > +				DPAA2_PMD_ERR("SG pool creation
> failed\n");
> > +				return -ENOMEM;
> > +			}
> > +		}
> >   		rte_eth_dev_probing_finish(eth_dev);
> >   		return 0;
> >   	}
> > diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h
> b/drivers/net/dpaa2/dpaa2_ethdev.h
> > index 32ae762e4a..872dced517 100644
> > --- a/drivers/net/dpaa2/dpaa2_ethdev.h
> > +++ b/drivers/net/dpaa2/dpaa2_ethdev.h
> > @@ -121,6 +121,15 @@
> >   #define DPAA2_PKT_TYPE_VLAN_1		0x0160
> >   #define DPAA2_PKT_TYPE_VLAN_2		0x0260
> >
> > +/* Global pool used by driver for SG list TX */
> > +extern struct rte_mempool *dpaa2_tx_sg_pool;
> > +/* Maximum SG segments */
> > +#define DPAA2_MAX_SGS 128
> > +/* SG pool size */
> > +#define DPAA2_POOL_SIZE 2048
> > +/* SG pool cache size */
> > +#define DPAA2_POOL_CACHE_SIZE 256
> > +
> >   /* enable timestamp in mbuf*/
> >   extern bool dpaa2_enable_ts[];
> >   extern uint64_t dpaa2_timestamp_rx_dynflag;
> > diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c
> b/drivers/net/dpaa2/dpaa2_rxtx.c
> > index bc0e49b0d4..dcd86c4056 100644
> > --- a/drivers/net/dpaa2/dpaa2_rxtx.c
> > +++ b/drivers/net/dpaa2/dpaa2_rxtx.c
> > @@ -403,7 +403,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd,
> >   static int __rte_noinline __rte_hot
> >   eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
> >   		  struct qbman_fd *fd,
> > -		  struct rte_mempool *mp, uint16_t bpid)
> > +		  uint16_t bpid)
> >   {
> >   	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
> >   	struct qbman_sge *sgt, *sge = NULL;
> > @@ -433,12 +433,12 @@ eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
> >   		}
> >   		DPAA2_SET_FD_OFFSET(fd, offset);
> >   	} else {
> > -		temp = rte_pktmbuf_alloc(mp);
> > +		temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool);
> >   		if (temp == NULL) {
> >   			DPAA2_PMD_DP_DEBUG("No memory to allocate
> S/G table\n");
> >   			return -ENOMEM;
> >   		}
> > -		DPAA2_SET_ONLY_FD_BPID(fd, bpid);
> > +		DPAA2_SET_ONLY_FD_BPID(fd,
> mempool_to_bpid(dpaa2_tx_sg_pool));
> >   		DPAA2_SET_FD_OFFSET(fd, temp->data_off);
> >   #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
> >   		rte_mempool_check_cookies(rte_mempool_from_obj((void
> *)temp),
> > @@ -1321,9 +1321,10 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
> **bufs, uint16_t nb_pkts)
> >
> >   			if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) {
> >   				if (unlikely((*bufs)->nb_segs > 1)) {
> > +					mp = (*bufs)->pool;
> >   					if (eth_mbuf_to_sg_fd(*bufs,
> >   							      &fd_arr[loop],
> > -							      mp, 0))
> > +
> mempool_to_bpid(mp)))
> >   						goto send_n_return;
> >   				} else {
> >   					eth_mbuf_to_fd(*bufs,
> > @@ -1372,7 +1373,7 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
> **bufs, uint16_t nb_pkts)
> >   				if (unlikely((*bufs)->nb_segs > 1)) {
> >   					if (eth_mbuf_to_sg_fd(*bufs,
> >   							&fd_arr[loop],
> > -							mp, bpid))
> > +							bpid))
> >   						goto send_n_return;
> >   				} else {
> >   					eth_mbuf_to_fd(*bufs,
> > @@ -1646,7 +1647,6 @@ dpaa2_dev_tx_multi_txq_ordered(void **queue,
> >   			if (unlikely((*bufs)->nb_segs > 1)) {
> >   				if (eth_mbuf_to_sg_fd(*bufs,
> >   						      &fd_arr[loop],
> > -						      mp,
> >   						      bpid))
> >   					goto send_frames;
> >   			} else {
> > @@ -1810,7 +1810,6 @@ dpaa2_dev_tx_ordered(void *queue, struct
> rte_mbuf **bufs, uint16_t nb_pkts)
> >   				if (unlikely((*bufs)->nb_segs > 1)) {
> >   					if (eth_mbuf_to_sg_fd(*bufs,
> >   							      &fd_arr[loop],
> > -							      mp,
> >   							      bpid))
> >   						goto send_n_return;
> >   				} else {
Ferruh Yigit Oct. 6, 2022, 9:38 a.m. UTC | #3
On 10/6/2022 9:49 AM, Gagandeep Singh wrote:
> Hi
> 
>> -----Original Message-----
>> From: Ferruh Yigit <ferruh.yigit@amd.com>
>> Sent: Wednesday, October 5, 2022 7:51 PM
>> To: Gagandeep Singh <G.Singh@nxp.com>; dev@dpdk.org
>> Subject: Re: [PATCH 07/15] net/dpaa2: use internal mempool for SG table
>>
>> On 9/28/2022 6:25 AM, Gagandeep Singh wrote:
>>> Creating and using driver's mempool for allocating the SG table memory
>>> required for FD creation instead of relying on user mempool.
>>>
>>
>> As far as I can see this is in the Tx path, can you please explain why
>> driver need an internal pktmbuf pool?
>>
>> And shouldn't mempool needs to be freed on driver close and release.
>>
>> Same comment applies for dpaa version of the patch (12/15).
>>
> 
> In TX path, in case of SG packets, driver need additional SG table memory to store the data
> of all segments to pass to HW. Earlier it was dependent on the user's pool to allocate memory
> which is wrong. so allocating the new pool in the driver for this purpose.
> 

Agree to have driver internal mempool for driver internal requirements.

> We are creating only one pool for all the DPAA net devices on first DPAA net device probe, so it can
> be free only on last device probe. So will it be worth to free the pool on last device probe as
> memory will be released on process kill as well?
> 

Yes, it will be cleared but it is good discipline to free memory on 
quit/close.
What do you think to allocate one per-port, and store it in device 
private data, it makes easy to free it when port closed or released?

>>> Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
>>> ---
>>>    drivers/net/dpaa2/dpaa2_ethdev.c | 14 ++++++++++++++
>>>    drivers/net/dpaa2/dpaa2_ethdev.h |  9 +++++++++
>>>    drivers/net/dpaa2/dpaa2_rxtx.c   | 13 ++++++-------
>>>    3 files changed, 29 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c
>> b/drivers/net/dpaa2/dpaa2_ethdev.c
>>> index 37a8b43114..c7aae70300 100644
>>> --- a/drivers/net/dpaa2/dpaa2_ethdev.c
>>> +++ b/drivers/net/dpaa2/dpaa2_ethdev.c
>>> @@ -78,6 +78,8 @@ bool dpaa2_enable_err_queue;
>>>    #define MAX_NB_RX_DESC		11264
>>>    int total_nb_rx_desc;
>>>
>>> +struct rte_mempool *dpaa2_tx_sg_pool;
>>> +
>>>    struct rte_dpaa2_xstats_name_off {
>>>    	char name[RTE_ETH_XSTATS_NAME_SIZE];
>>>    	uint8_t page_id; /* dpni statistics page id */
>>> @@ -2907,6 +2909,18 @@ rte_dpaa2_probe(struct rte_dpaa2_driver
>> *dpaa2_drv,
>>>    	/* Invoke PMD device initialization function */
>>>    	diag = dpaa2_dev_init(eth_dev);
>>>    	if (diag == 0) {
>>> +		if (!dpaa2_tx_sg_pool) {
>>> +			dpaa2_tx_sg_pool =
>>> +
>> 	rte_pktmbuf_pool_create("dpaa2_mbuf_tx_sg_pool",
>>> +				DPAA2_POOL_SIZE,
>>> +				DPAA2_POOL_CACHE_SIZE, 0,
>>> +				DPAA2_MAX_SGS * sizeof(struct qbman_sge),
>>> +				rte_socket_id());
>>> +			if (dpaa2_tx_sg_pool == NULL) {
>>> +				DPAA2_PMD_ERR("SG pool creation
>> failed\n");
>>> +				return -ENOMEM;
>>> +			}
>>> +		}
>>>    		rte_eth_dev_probing_finish(eth_dev);
>>>    		return 0;
>>>    	}
>>> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h
>> b/drivers/net/dpaa2/dpaa2_ethdev.h
>>> index 32ae762e4a..872dced517 100644
>>> --- a/drivers/net/dpaa2/dpaa2_ethdev.h
>>> +++ b/drivers/net/dpaa2/dpaa2_ethdev.h
>>> @@ -121,6 +121,15 @@
>>>    #define DPAA2_PKT_TYPE_VLAN_1		0x0160
>>>    #define DPAA2_PKT_TYPE_VLAN_2		0x0260
>>>
>>> +/* Global pool used by driver for SG list TX */
>>> +extern struct rte_mempool *dpaa2_tx_sg_pool;
>>> +/* Maximum SG segments */
>>> +#define DPAA2_MAX_SGS 128
>>> +/* SG pool size */
>>> +#define DPAA2_POOL_SIZE 2048
>>> +/* SG pool cache size */
>>> +#define DPAA2_POOL_CACHE_SIZE 256
>>> +
>>>    /* enable timestamp in mbuf*/
>>>    extern bool dpaa2_enable_ts[];
>>>    extern uint64_t dpaa2_timestamp_rx_dynflag;
>>> diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c
>> b/drivers/net/dpaa2/dpaa2_rxtx.c
>>> index bc0e49b0d4..dcd86c4056 100644
>>> --- a/drivers/net/dpaa2/dpaa2_rxtx.c
>>> +++ b/drivers/net/dpaa2/dpaa2_rxtx.c
>>> @@ -403,7 +403,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd,
>>>    static int __rte_noinline __rte_hot
>>>    eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
>>>    		  struct qbman_fd *fd,
>>> -		  struct rte_mempool *mp, uint16_t bpid)
>>> +		  uint16_t bpid)
>>>    {
>>>    	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
>>>    	struct qbman_sge *sgt, *sge = NULL;
>>> @@ -433,12 +433,12 @@ eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
>>>    		}
>>>    		DPAA2_SET_FD_OFFSET(fd, offset);
>>>    	} else {
>>> -		temp = rte_pktmbuf_alloc(mp);
>>> +		temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool);
>>>    		if (temp == NULL) {
>>>    			DPAA2_PMD_DP_DEBUG("No memory to allocate
>> S/G table\n");
>>>    			return -ENOMEM;
>>>    		}
>>> -		DPAA2_SET_ONLY_FD_BPID(fd, bpid);
>>> +		DPAA2_SET_ONLY_FD_BPID(fd,
>> mempool_to_bpid(dpaa2_tx_sg_pool));
>>>    		DPAA2_SET_FD_OFFSET(fd, temp->data_off);
>>>    #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
>>>    		rte_mempool_check_cookies(rte_mempool_from_obj((void
>> *)temp),
>>> @@ -1321,9 +1321,10 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
>> **bufs, uint16_t nb_pkts)
>>>
>>>    			if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) {
>>>    				if (unlikely((*bufs)->nb_segs > 1)) {
>>> +					mp = (*bufs)->pool;
>>>    					if (eth_mbuf_to_sg_fd(*bufs,
>>>    							      &fd_arr[loop],
>>> -							      mp, 0))
>>> +
>> mempool_to_bpid(mp)))
>>>    						goto send_n_return;
>>>    				} else {
>>>    					eth_mbuf_to_fd(*bufs,
>>> @@ -1372,7 +1373,7 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
>> **bufs, uint16_t nb_pkts)
>>>    				if (unlikely((*bufs)->nb_segs > 1)) {
>>>    					if (eth_mbuf_to_sg_fd(*bufs,
>>>    							&fd_arr[loop],
>>> -							mp, bpid))
>>> +							bpid))
>>>    						goto send_n_return;
>>>    				} else {
>>>    					eth_mbuf_to_fd(*bufs,
>>> @@ -1646,7 +1647,6 @@ dpaa2_dev_tx_multi_txq_ordered(void **queue,
>>>    			if (unlikely((*bufs)->nb_segs > 1)) {
>>>    				if (eth_mbuf_to_sg_fd(*bufs,
>>>    						      &fd_arr[loop],
>>> -						      mp,
>>>    						      bpid))
>>>    					goto send_frames;
>>>    			} else {
>>> @@ -1810,7 +1810,6 @@ dpaa2_dev_tx_ordered(void *queue, struct
>> rte_mbuf **bufs, uint16_t nb_pkts)
>>>    				if (unlikely((*bufs)->nb_segs > 1)) {
>>>    					if (eth_mbuf_to_sg_fd(*bufs,
>>>    							      &fd_arr[loop],
>>> -							      mp,
>>>    							      bpid))
>>>    						goto send_n_return;
>>>    				} else {
>
Gagandeep Singh Oct. 6, 2022, 11:18 a.m. UTC | #4
> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@amd.com>
> Sent: Thursday, October 6, 2022 3:08 PM
> To: Gagandeep Singh <G.Singh@nxp.com>; dev@dpdk.org
> Subject: Re: [PATCH 07/15] net/dpaa2: use internal mempool for SG table
> 
> On 10/6/2022 9:49 AM, Gagandeep Singh wrote:
> > Hi
> >
> >> -----Original Message-----
> >> From: Ferruh Yigit <ferruh.yigit@amd.com>
> >> Sent: Wednesday, October 5, 2022 7:51 PM
> >> To: Gagandeep Singh <G.Singh@nxp.com>; dev@dpdk.org
> >> Subject: Re: [PATCH 07/15] net/dpaa2: use internal mempool for SG
> >> table
> >>
> >> On 9/28/2022 6:25 AM, Gagandeep Singh wrote:
> >>> Creating and using driver's mempool for allocating the SG table
> >>> memory required for FD creation instead of relying on user mempool.
> >>>
> >>
> >> As far as I can see this is in the Tx path, can you please explain
> >> why driver need an internal pktmbuf pool?
> >>
> >> And shouldn't mempool needs to be freed on driver close and release.
> >>
> >> Same comment applies for dpaa version of the patch (12/15).
> >>
> >
> > In TX path, in case of SG packets, driver need additional SG table
> > memory to store the data of all segments to pass to HW. Earlier it was
> > dependent on the user's pool to allocate memory which is wrong. so
> allocating the new pool in the driver for this purpose.
> >
> 
> Agree to have driver internal mempool for driver internal requirements.
> 
> > We are creating only one pool for all the DPAA net devices on first
> > DPAA net device probe, so it can be free only on last device probe. So
> > will it be worth to free the pool on last device probe as memory will be
> released on process kill as well?
> >
> 
> Yes, it will be cleared but it is good discipline to free memory on quit/close.
> What do you think to allocate one per-port, and store it in device private
> data, it makes easy to free it when port closed or released?
> 
Yes It will be easy to have per device mempool, But due to limited number of
HW pools we cannot allocate as many as eth devices number of pool for internal
Uses.

Ok, I will free the pool on last eth device remove in next version.

> >>> Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
> >>> ---
> >>>    drivers/net/dpaa2/dpaa2_ethdev.c | 14 ++++++++++++++
> >>>    drivers/net/dpaa2/dpaa2_ethdev.h |  9 +++++++++
> >>>    drivers/net/dpaa2/dpaa2_rxtx.c   | 13 ++++++-------
> >>>    3 files changed, 29 insertions(+), 7 deletions(-)
> >>>
> >>> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c
> >> b/drivers/net/dpaa2/dpaa2_ethdev.c
> >>> index 37a8b43114..c7aae70300 100644
> >>> --- a/drivers/net/dpaa2/dpaa2_ethdev.c
> >>> +++ b/drivers/net/dpaa2/dpaa2_ethdev.c
> >>> @@ -78,6 +78,8 @@ bool dpaa2_enable_err_queue;
> >>>    #define MAX_NB_RX_DESC		11264
> >>>    int total_nb_rx_desc;
> >>>
> >>> +struct rte_mempool *dpaa2_tx_sg_pool;
> >>> +
> >>>    struct rte_dpaa2_xstats_name_off {
> >>>    	char name[RTE_ETH_XSTATS_NAME_SIZE];
> >>>    	uint8_t page_id; /* dpni statistics page id */ @@ -2907,6
> >>> +2909,18 @@ rte_dpaa2_probe(struct rte_dpaa2_driver
> >> *dpaa2_drv,
> >>>    	/* Invoke PMD device initialization function */
> >>>    	diag = dpaa2_dev_init(eth_dev);
> >>>    	if (diag == 0) {
> >>> +		if (!dpaa2_tx_sg_pool) {
> >>> +			dpaa2_tx_sg_pool =
> >>> +
> >> 	rte_pktmbuf_pool_create("dpaa2_mbuf_tx_sg_pool",
> >>> +				DPAA2_POOL_SIZE,
> >>> +				DPAA2_POOL_CACHE_SIZE, 0,
> >>> +				DPAA2_MAX_SGS * sizeof(struct qbman_sge),
> >>> +				rte_socket_id());
> >>> +			if (dpaa2_tx_sg_pool == NULL) {
> >>> +				DPAA2_PMD_ERR("SG pool creation
> >> failed\n");
> >>> +				return -ENOMEM;
> >>> +			}
> >>> +		}
> >>>    		rte_eth_dev_probing_finish(eth_dev);
> >>>    		return 0;
> >>>    	}
> >>> diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h
> >> b/drivers/net/dpaa2/dpaa2_ethdev.h
> >>> index 32ae762e4a..872dced517 100644
> >>> --- a/drivers/net/dpaa2/dpaa2_ethdev.h
> >>> +++ b/drivers/net/dpaa2/dpaa2_ethdev.h
> >>> @@ -121,6 +121,15 @@
> >>>    #define DPAA2_PKT_TYPE_VLAN_1		0x0160
> >>>    #define DPAA2_PKT_TYPE_VLAN_2		0x0260
> >>>
> >>> +/* Global pool used by driver for SG list TX */ extern struct
> >>> +rte_mempool *dpaa2_tx_sg_pool;
> >>> +/* Maximum SG segments */
> >>> +#define DPAA2_MAX_SGS 128
> >>> +/* SG pool size */
> >>> +#define DPAA2_POOL_SIZE 2048
> >>> +/* SG pool cache size */
> >>> +#define DPAA2_POOL_CACHE_SIZE 256
> >>> +
> >>>    /* enable timestamp in mbuf*/
> >>>    extern bool dpaa2_enable_ts[];
> >>>    extern uint64_t dpaa2_timestamp_rx_dynflag; diff --git
> >>> a/drivers/net/dpaa2/dpaa2_rxtx.c
> >> b/drivers/net/dpaa2/dpaa2_rxtx.c
> >>> index bc0e49b0d4..dcd86c4056 100644
> >>> --- a/drivers/net/dpaa2/dpaa2_rxtx.c
> >>> +++ b/drivers/net/dpaa2/dpaa2_rxtx.c
> >>> @@ -403,7 +403,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd,
> >>>    static int __rte_noinline __rte_hot
> >>>    eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
> >>>    		  struct qbman_fd *fd,
> >>> -		  struct rte_mempool *mp, uint16_t bpid)
> >>> +		  uint16_t bpid)
> >>>    {
> >>>    	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
> >>>    	struct qbman_sge *sgt, *sge = NULL; @@ -433,12 +433,12 @@
> >>> eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
> >>>    		}
> >>>    		DPAA2_SET_FD_OFFSET(fd, offset);
> >>>    	} else {
> >>> -		temp = rte_pktmbuf_alloc(mp);
> >>> +		temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool);
> >>>    		if (temp == NULL) {
> >>>    			DPAA2_PMD_DP_DEBUG("No memory to allocate
> >> S/G table\n");
> >>>    			return -ENOMEM;
> >>>    		}
> >>> -		DPAA2_SET_ONLY_FD_BPID(fd, bpid);
> >>> +		DPAA2_SET_ONLY_FD_BPID(fd,
> >> mempool_to_bpid(dpaa2_tx_sg_pool));
> >>>    		DPAA2_SET_FD_OFFSET(fd, temp->data_off);
> >>>    #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
> >>>    		rte_mempool_check_cookies(rte_mempool_from_obj((void
> >> *)temp),
> >>> @@ -1321,9 +1321,10 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
> >> **bufs, uint16_t nb_pkts)
> >>>
> >>>    			if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) {
> >>>    				if (unlikely((*bufs)->nb_segs > 1)) {
> >>> +					mp = (*bufs)->pool;
> >>>    					if (eth_mbuf_to_sg_fd(*bufs,
> >>>    							      &fd_arr[loop],
> >>> -							      mp, 0))
> >>> +
> >> mempool_to_bpid(mp)))
> >>>    						goto send_n_return;
> >>>    				} else {
> >>>    					eth_mbuf_to_fd(*bufs,
> >>> @@ -1372,7 +1373,7 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf
> >> **bufs, uint16_t nb_pkts)
> >>>    				if (unlikely((*bufs)->nb_segs > 1)) {
> >>>    					if (eth_mbuf_to_sg_fd(*bufs,
> >>>    							&fd_arr[loop],
> >>> -							mp, bpid))
> >>> +							bpid))
> >>>    						goto send_n_return;
> >>>    				} else {
> >>>    					eth_mbuf_to_fd(*bufs,
> >>> @@ -1646,7 +1647,6 @@ dpaa2_dev_tx_multi_txq_ordered(void
> **queue,
> >>>    			if (unlikely((*bufs)->nb_segs > 1)) {
> >>>    				if (eth_mbuf_to_sg_fd(*bufs,
> >>>    						      &fd_arr[loop],
> >>> -						      mp,
> >>>    						      bpid))
> >>>    					goto send_frames;
> >>>    			} else {
> >>> @@ -1810,7 +1810,6 @@ dpaa2_dev_tx_ordered(void *queue, struct
> >> rte_mbuf **bufs, uint16_t nb_pkts)
> >>>    				if (unlikely((*bufs)->nb_segs > 1)) {
> >>>    					if (eth_mbuf_to_sg_fd(*bufs,
> >>>    							      &fd_arr[loop],
> >>> -							      mp,
> >>>    							      bpid))
> >>>    						goto send_n_return;
> >>>    				} else {
> >
diff mbox series

Patch

diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c
index 37a8b43114..c7aae70300 100644
--- a/drivers/net/dpaa2/dpaa2_ethdev.c
+++ b/drivers/net/dpaa2/dpaa2_ethdev.c
@@ -78,6 +78,8 @@  bool dpaa2_enable_err_queue;
 #define MAX_NB_RX_DESC		11264
 int total_nb_rx_desc;
 
+struct rte_mempool *dpaa2_tx_sg_pool;
+
 struct rte_dpaa2_xstats_name_off {
 	char name[RTE_ETH_XSTATS_NAME_SIZE];
 	uint8_t page_id; /* dpni statistics page id */
@@ -2907,6 +2909,18 @@  rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv,
 	/* Invoke PMD device initialization function */
 	diag = dpaa2_dev_init(eth_dev);
 	if (diag == 0) {
+		if (!dpaa2_tx_sg_pool) {
+			dpaa2_tx_sg_pool =
+				rte_pktmbuf_pool_create("dpaa2_mbuf_tx_sg_pool",
+				DPAA2_POOL_SIZE,
+				DPAA2_POOL_CACHE_SIZE, 0,
+				DPAA2_MAX_SGS * sizeof(struct qbman_sge),
+				rte_socket_id());
+			if (dpaa2_tx_sg_pool == NULL) {
+				DPAA2_PMD_ERR("SG pool creation failed\n");
+				return -ENOMEM;
+			}
+		}
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
diff --git a/drivers/net/dpaa2/dpaa2_ethdev.h b/drivers/net/dpaa2/dpaa2_ethdev.h
index 32ae762e4a..872dced517 100644
--- a/drivers/net/dpaa2/dpaa2_ethdev.h
+++ b/drivers/net/dpaa2/dpaa2_ethdev.h
@@ -121,6 +121,15 @@ 
 #define DPAA2_PKT_TYPE_VLAN_1		0x0160
 #define DPAA2_PKT_TYPE_VLAN_2		0x0260
 
+/* Global pool used by driver for SG list TX */
+extern struct rte_mempool *dpaa2_tx_sg_pool;
+/* Maximum SG segments */
+#define DPAA2_MAX_SGS 128
+/* SG pool size */
+#define DPAA2_POOL_SIZE 2048
+/* SG pool cache size */
+#define DPAA2_POOL_CACHE_SIZE 256
+
 /* enable timestamp in mbuf*/
 extern bool dpaa2_enable_ts[];
 extern uint64_t dpaa2_timestamp_rx_dynflag;
diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
index bc0e49b0d4..dcd86c4056 100644
--- a/drivers/net/dpaa2/dpaa2_rxtx.c
+++ b/drivers/net/dpaa2/dpaa2_rxtx.c
@@ -403,7 +403,7 @@  eth_fd_to_mbuf(const struct qbman_fd *fd,
 static int __rte_noinline __rte_hot
 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 		  struct qbman_fd *fd,
-		  struct rte_mempool *mp, uint16_t bpid)
+		  uint16_t bpid)
 {
 	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 	struct qbman_sge *sgt, *sge = NULL;
@@ -433,12 +433,12 @@  eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 		}
 		DPAA2_SET_FD_OFFSET(fd, offset);
 	} else {
-		temp = rte_pktmbuf_alloc(mp);
+		temp = rte_pktmbuf_alloc(dpaa2_tx_sg_pool);
 		if (temp == NULL) {
 			DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
 			return -ENOMEM;
 		}
-		DPAA2_SET_ONLY_FD_BPID(fd, bpid);
+		DPAA2_SET_ONLY_FD_BPID(fd, mempool_to_bpid(dpaa2_tx_sg_pool));
 		DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
 		rte_mempool_check_cookies(rte_mempool_from_obj((void *)temp),
@@ -1321,9 +1321,10 @@  dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 
 			if (unlikely(RTE_MBUF_HAS_EXTBUF(*bufs))) {
 				if (unlikely((*bufs)->nb_segs > 1)) {
+					mp = (*bufs)->pool;
 					if (eth_mbuf_to_sg_fd(*bufs,
 							      &fd_arr[loop],
-							      mp, 0))
+							      mempool_to_bpid(mp)))
 						goto send_n_return;
 				} else {
 					eth_mbuf_to_fd(*bufs,
@@ -1372,7 +1373,7 @@  dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				if (unlikely((*bufs)->nb_segs > 1)) {
 					if (eth_mbuf_to_sg_fd(*bufs,
 							&fd_arr[loop],
-							mp, bpid))
+							bpid))
 						goto send_n_return;
 				} else {
 					eth_mbuf_to_fd(*bufs,
@@ -1646,7 +1647,6 @@  dpaa2_dev_tx_multi_txq_ordered(void **queue,
 			if (unlikely((*bufs)->nb_segs > 1)) {
 				if (eth_mbuf_to_sg_fd(*bufs,
 						      &fd_arr[loop],
-						      mp,
 						      bpid))
 					goto send_frames;
 			} else {
@@ -1810,7 +1810,6 @@  dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				if (unlikely((*bufs)->nb_segs > 1)) {
 					if (eth_mbuf_to_sg_fd(*bufs,
 							      &fd_arr[loop],
-							      mp,
 							      bpid))
 						goto send_n_return;
 				} else {