@@ -1295,7 +1295,8 @@ show_mempool(char *name)
"\t -- No cache align (%c)\n"
"\t -- SP put (%c), SC get (%c)\n"
"\t -- Pool created (%c)\n"
- "\t -- No IOVA config (%c)\n",
+ "\t -- No IOVA config (%c)\n"
+ "\t -- Not used for IO (%c)\n",
ptr->name,
ptr->socket_id,
(flags & MEMPOOL_F_NO_SPREAD) ? 'y' : 'n',
@@ -1303,7 +1304,8 @@ show_mempool(char *name)
(flags & MEMPOOL_F_SP_PUT) ? 'y' : 'n',
(flags & MEMPOOL_F_SC_GET) ? 'y' : 'n',
(flags & MEMPOOL_F_POOL_CREATED) ? 'y' : 'n',
- (flags & MEMPOOL_F_NO_IOVA_CONTIG) ? 'y' : 'n');
+ (flags & MEMPOOL_F_NO_IOVA_CONTIG) ? 'y' : 'n',
+ (flags & MEMPOOL_F_NON_IO) ? 'y' : 'n');
printf(" - Size %u Cache %u element %u\n"
" - header %u trailer %u\n"
" - private data size %u\n",
@@ -12,6 +12,7 @@
#include <sys/queue.h>
#include <rte_common.h>
+#include <rte_eal_paging.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_errno.h>
@@ -729,6 +730,109 @@ test_mempool_events_safety(void)
#pragma pop_macro("RTE_TEST_TRACE_FAILURE")
}
+#pragma push_macro("RTE_TEST_TRACE_FAILURE")
+#undef RTE_TEST_TRACE_FAILURE
+#define RTE_TEST_TRACE_FAILURE(...) do { \
+ ret = TEST_FAILED; \
+ goto exit; \
+ } while (0)
+
+static int
+test_mempool_flag_non_io_set_when_no_iova_contig_set(void)
+{
+ struct rte_mempool *mp = NULL;
+ int ret;
+
+ mp = rte_mempool_create_empty("empty", MEMPOOL_SIZE,
+ MEMPOOL_ELT_SIZE, 0, 0,
+ SOCKET_ID_ANY, MEMPOOL_F_NO_IOVA_CONTIG);
+ RTE_TEST_ASSERT_NOT_NULL(mp, "Cannot create mempool: %s",
+ rte_strerror(rte_errno));
+ rte_mempool_set_ops_byname(mp, rte_mbuf_best_mempool_ops(), NULL);
+ ret = rte_mempool_populate_default(mp);
+ RTE_TEST_ASSERT(ret > 0, "Failed to populate mempool: %s",
+ rte_strerror(rte_errno));
+ RTE_TEST_ASSERT(mp->flags & MEMPOOL_F_NON_IO,
+ "NON_IO flag is not set when NO_IOVA_CONTIG is set");
+ ret = TEST_SUCCESS;
+exit:
+ rte_mempool_free(mp);
+ return ret;
+}
+
+static int
+test_mempool_flag_non_io_unset_when_populated_with_valid_iova(void)
+{
+ const struct rte_memzone *mz;
+ void *virt;
+ rte_iova_t iova;
+ size_t page_size = RTE_PGSIZE_2M;
+ struct rte_mempool *mp = NULL;
+ int ret;
+
+ mz = rte_memzone_reserve("test_mempool", 3 * page_size, SOCKET_ID_ANY,
+ RTE_MEMZONE_IOVA_CONTIG);
+ RTE_TEST_ASSERT_NOT_NULL(mz, "Cannot allocate memory");
+ virt = mz->addr;
+ iova = rte_mem_virt2iova(virt);
+ RTE_TEST_ASSERT_NOT_EQUAL(iova, RTE_BAD_IOVA, "Cannot get IOVA");
+ mp = rte_mempool_create_empty("empty", MEMPOOL_SIZE,
+ MEMPOOL_ELT_SIZE, 0, 0,
+ SOCKET_ID_ANY, 0);
+ RTE_TEST_ASSERT_NOT_NULL(mp, "Cannot create mempool: %s",
+ rte_strerror(rte_errno));
+
+ ret = rte_mempool_populate_iova(mp, RTE_PTR_ADD(virt, 1 * page_size),
+ RTE_BAD_IOVA, page_size, NULL, NULL);
+ RTE_TEST_ASSERT(ret > 0, "Failed to populate mempool: %s",
+ rte_strerror(rte_errno));
+ RTE_TEST_ASSERT(mp->flags & MEMPOOL_F_NON_IO,
+ "NON_IO flag is not set when mempool is populated with only RTE_BAD_IOVA");
+
+ ret = rte_mempool_populate_iova(mp, virt, iova, page_size, NULL, NULL);
+ RTE_TEST_ASSERT(ret > 0, "Failed to populate mempool: %s",
+ rte_strerror(rte_errno));
+ RTE_TEST_ASSERT(!(mp->flags & MEMPOOL_F_NON_IO),
+ "NON_IO flag is not unset when mempool is populated with valid IOVA");
+
+ ret = rte_mempool_populate_iova(mp, RTE_PTR_ADD(virt, 2 * page_size),
+ RTE_BAD_IOVA, page_size, NULL, NULL);
+ RTE_TEST_ASSERT(ret > 0, "Failed to populate mempool: %s",
+ rte_strerror(rte_errno));
+ RTE_TEST_ASSERT(!(mp->flags & MEMPOOL_F_NON_IO),
+ "NON_IO flag is set even when some objects have valid IOVA");
+ ret = TEST_SUCCESS;
+
+exit:
+ rte_mempool_free(mp);
+ rte_memzone_free(mz);
+ return ret;
+}
+
+static int
+test_mempool_flag_non_io_unset_by_default(void)
+{
+ struct rte_mempool *mp;
+ int ret;
+
+ mp = rte_mempool_create_empty("empty", MEMPOOL_SIZE,
+ MEMPOOL_ELT_SIZE, 0, 0,
+ SOCKET_ID_ANY, 0);
+ RTE_TEST_ASSERT_NOT_NULL(mp, "Cannot create mempool: %s",
+ rte_strerror(rte_errno));
+ ret = rte_mempool_populate_default(mp);
+ RTE_TEST_ASSERT_EQUAL(ret, (int)mp->size, "Failed to populate mempool: %s",
+ rte_strerror(rte_errno));
+ RTE_TEST_ASSERT(!(mp->flags & MEMPOOL_F_NON_IO),
+ "NON_IO flag is set by default");
+ ret = TEST_SUCCESS;
+exit:
+ rte_mempool_free(mp);
+ return ret;
+}
+
+#pragma pop_macro("RTE_TEST_TRACE_FAILURE")
+
static int
test_mempool(void)
{
@@ -914,6 +1018,14 @@ test_mempool(void)
if (test_mempool_events_safety() < 0)
GOTO_ERR(ret, err);
+ /* test NON_IO flag inference */
+ if (test_mempool_flag_non_io_unset_by_default() < 0)
+ GOTO_ERR(ret, err);
+ if (test_mempool_flag_non_io_set_when_no_iova_contig_set() < 0)
+ GOTO_ERR(ret, err);
+ if (test_mempool_flag_non_io_unset_when_populated_with_valid_iova() < 0)
+ GOTO_ERR(ret, err);
+
rte_mempool_list_dump(stdout);
ret = 0;
@@ -229,6 +229,9 @@ API Changes
the crypto/security operation. This field will be used to communicate
events such as soft expiry with IPsec in lookaside mode.
+* mempool: Added ``MEMPOOL_F_NON_IO`` flag to give a hint to DPDK components
+ that objects from this pool will not be used for device IO (e.g. DMA).
+
ABI Changes
-----------
@@ -372,6 +372,10 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next);
mp->nb_mem_chunks++;
+ /* At least some objects in the pool can now be used for IO. */
+ if (iova != RTE_BAD_IOVA)
+ mp->flags &= ~MEMPOOL_F_NON_IO;
+
/* Report the mempool as ready only when fully populated. */
if (mp->populated_size >= mp->size)
mempool_event_callback_invoke(RTE_MEMPOOL_EVENT_READY, mp);
@@ -851,6 +855,12 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
return NULL;
}
+ /*
+ * No objects in the pool can be used for IO until it's populated
+ * with at least some objects with valid IOVA.
+ */
+ flags |= MEMPOOL_F_NON_IO;
+
/* "no cache align" imply "no spread" */
if (flags & MEMPOOL_F_NO_CACHE_ALIGN)
flags |= MEMPOOL_F_NO_SPREAD;
@@ -257,6 +257,8 @@ struct rte_mempool {
#define MEMPOOL_F_SC_GET 0x0008 /**< Default get is "single-consumer".*/
#define MEMPOOL_F_POOL_CREATED 0x0010 /**< Internal: pool is created. */
#define MEMPOOL_F_NO_IOVA_CONTIG 0x0020 /**< Don't need IOVA contiguous objs. */
+/** Internal: no object from the pool can be used for device IO (DMA). */
+#define MEMPOOL_F_NON_IO 0x0040
/**
* @internal When debug is enabled, store some statistics.