@@ -670,6 +670,7 @@ Kai Ji <kai.ji@intel.com>
Kaiwen Deng <kaiwenx.deng@intel.com>
Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Kamalakannan R <kamalakannan.r@intel.com>
+Kamalakshitha Aligeri <kamalakshitha.aligeri@arm.com>
Kamil Bednarczyk <kamil.bednarczyk@intel.com>
Kamil Chalupnik <kamilx.chalupnik@intel.com>
Kamil Rytarowski <kamil.rytarowski@caviumnetworks.com>
@@ -35,6 +35,9 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
if (unlikely(!cache))
return i40e_rxq_rearm_common(rxq, true);
+ void **cache_objs;
+
+ cache_objs = rte_mempool_cache_zc_get_bulk(cache, rxq->mp, RTE_I40E_RXQ_REARM_THRESH);
/* We need to pull 'n' more MBUFs into the software ring from mempool
* We inline the mempool function here, so we can vectorize the copy
@@ -45,15 +48,12 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
/* No. Backfill the cache first, and then fill from it */
uint32_t req = RTE_I40E_RXQ_REARM_THRESH + (cache->size -
cache->len);
-
/* How many do we require
* i.e. number to fill the cache + the request
*/
int ret = rte_mempool_ops_dequeue_bulk(rxq->mp,
- &cache->objs[cache->len], req);
- if (ret == 0) {
- cache->len += req;
- } else {
+ cache_objs[cache->len], req);
+ if (ret != 0) {
if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
rxq->nb_rx_desc) {
__m128i dma_addr0;
@@ -63,11 +63,11 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
rxep[i].mbuf = &rxq->fake_mbuf;
_mm_store_si128
((__m128i *)&rxdp[i].read,
- dma_addr0);
+ dma_addr0);
}
}
rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
- RTE_I40E_RXQ_REARM_THRESH;
+ RTE_I40E_RXQ_REARM_THRESH;
return;
}
}
@@ -90,7 +90,7 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
*/
for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH / 8; i++) {
const __m512i mbuf_ptrs = _mm512_loadu_si512
- (&cache->objs[cache->len - 8]);
+ (cache_objs[cache->len - 8]);
_mm512_store_si512(rxep, mbuf_ptrs);
/* gather iova of mbuf0-7 into one zmm reg */
@@ -906,21 +906,16 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq)
struct rte_mempool_cache *cache = rte_mempool_default_cache(mp,
rte_lcore_id());
- if (!cache || cache->len == 0)
+ if (!cache)
goto normal;
- cache_objs = &cache->objs[cache->len];
+ cache_objs = rte_mempool_cache_zc_put_bulk(cache, mp, n);
- if (n > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ if (!cache_objs) {
rte_mempool_ops_enqueue_bulk(mp, (void *)txep, n);
goto done;
}
- /* The cache follows the following algorithm
- * 1. Add the objects to the cache
- * 2. Anything greater than the cache min value (if it
- * crosses the cache flush threshold) is flushed to the ring.
- */
/* Add elements back into the cache */
uint32_t copied = 0;
/* n is multiple of 32 */
@@ -936,14 +931,6 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq)
_mm512_storeu_si512(&cache_objs[copied + 24], d);
copied += 32;
}
- cache->len += n;
-
- if (cache->len >= cache->flushthresh) {
- rte_mempool_ops_enqueue_bulk
- (mp, &cache->objs[cache->size],
- cache->len - cache->size);
- cache->len = cache->size;
- }
goto done;
}
@@ -95,17 +95,24 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
n = txq->tx_rs_thresh;
- /* first buffer to free from S/W ring is at index
- * tx_next_dd - (tx_rs_thresh-1)
- */
+ /* first buffer to free from S/W ring is at index
+ * tx_next_dd - (tx_rs_thresh-1)
+ */
txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) {
- for (i = 0; i < n; i++) {
- free[i] = txep[i].mbuf;
- /* no need to reset txep[i].mbuf in vector path */
+ struct rte_mempool *mp = txep[0].mbuf->pool;
+ struct rte_mempool_cache *cache = rte_mempool_default_cache(mp, rte_lcore_id());
+ void **cache_objs;
+
+ cache_objs = rte_mempool_cache_zc_put_bulk(cache, mp, n);
+ if (cache_objs) {
+ for (i = 0; i < n; i++) {
+ cache_objs[i] = txep->mbuf;
+ /* no need to reset txep[i].mbuf in vector path */
+ txep++;
+ }
}
- rte_mempool_put_bulk(free[0]->pool, (void **)free, n);
goto done;
}
@@ -121,7 +128,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
} else {
rte_mempool_put_bulk(free[0]->pool,
(void *)free,
- nb_free);
+ nb_free);
free[0] = m;
nb_free = 1;
}
@@ -30,23 +30,32 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
uint64_t paddr;
rxdp = rxq->rx_ring + rxq->rxrearm_start;
-
- /* Pull 'n' more MBUFs into the software ring */
- if (unlikely(rte_mempool_get_bulk(rxq->mp,
- (void *)rxep,
- RTE_I40E_RXQ_REARM_THRESH) < 0)) {
- if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
- rxq->nb_rx_desc) {
- for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {
- rxep[i].mbuf = &rxq->fake_mbuf;
- vst1q_u64((uint64_t *)&rxdp[i].read, zero);
+ struct rte_mempool_cache *cache = rte_mempool_default_cache(rxq->mp, rte_lcore_id());
+
+ /*When no cache provided, get the objects directly from backend */
+ if (!cache) {
+ int ret = rte_mempool_ops_dequeue_bulk(rxq->mp, (void *)rxep,
+ RTE_I40E_RXQ_REARM_THRESH);
+ /* Pull 'n' more MBUFs into the software ring */
+ if (ret != 0) {
+ if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
+ rxq->nb_rx_desc) {
+ for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {
+ rxep[i].mbuf = &rxq->fake_mbuf;
+ vst1q_u64((uint64_t *)&rxdp[i].read, zero);
+ }
}
+ rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
+ RTE_I40E_RXQ_REARM_THRESH;
+ return;
}
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
- RTE_I40E_RXQ_REARM_THRESH;
- return;
}
+ void **cache_objs;
+
+ cache_objs = rte_mempool_cache_zc_get_bulk(cache, rxq->mp, RTE_I40E_RXQ_REARM_THRESH);
+ rte_memcpy(rxep, cache_objs, RTE_I40E_RXQ_REARM_THRESH * sizeof(void *));
+
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) {
mb0 = rxep[0].mbuf;