@@ -167,6 +167,7 @@
#include <rte_common.h>
#include <rte_config.h>
#include <rte_power_intrinsics.h>
+#include <rte_mempool.h>
#include "rte_ethdev_trace_fp.h"
#include "rte_dev_info.h"
@@ -6334,6 +6335,8 @@ rte_eth_rx_burst(uint16_t port_id, uint16_t queue_id,
nb_rx = p->rx_pkt_burst(qd, rx_pkts, nb_pkts);
+ rte_mempool_history_bulk((void **)rx_pkts, nb_rx, RTE_MEMPOOL_APP_RX);
+
#ifdef RTE_ETHDEV_RXTX_CALLBACKS
{
void *cb;
@@ -6692,8 +6695,19 @@ rte_eth_tx_burst(uint16_t port_id, uint16_t queue_id,
}
#endif
+#if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+ uint16_t requested_pkts = nb_pkts;
+ rte_mempool_history_bulk((void **)tx_pkts, nb_pkts, RTE_MEMPOOL_PMD_TX);
+#endif
+
nb_pkts = p->tx_pkt_burst(qd, tx_pkts, nb_pkts);
+#if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+ if (requested_pkts > nb_pkts)
+ rte_mempool_history_bulk((void **)tx_pkts + nb_pkts,
+ requested_pkts - nb_pkts, RTE_MEMPOOL_BUSY_TX);
+#endif
+
rte_ethdev_trace_tx_burst(port_id, queue_id, (void **)tx_pkts, nb_pkts);
return nb_pkts;
}
@@ -32,6 +32,7 @@
#include "mempool_trace.h"
#include "rte_mempool.h"
+
RTE_EXPORT_SYMBOL(rte_mempool_logtype)
RTE_LOG_REGISTER_DEFAULT(rte_mempool_logtype, INFO);
@@ -1632,3 +1633,113 @@ RTE_INIT(mempool_init_telemetry)
rte_telemetry_register_cmd("/mempool/info", mempool_handle_info,
"Returns mempool info. Parameters: pool_name");
}
+
+#if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+static void
+rte_mempool_get_object_history_stat(FILE *f, struct rte_mempool *mp)
+{
+ struct rte_mempool_objhdr *hdr;
+
+ uint64_t n_never = 0; /* never been allocated. */
+ uint64_t n_free = 0; /* returned to the pool. */
+ uint64_t n_alloc = 0; /* allocated from the pool. */
+ uint64_t n_ref = 0; /* freed by pmd, not returned to the pool. */
+ uint64_t n_pmd_tx = 0; /* owned by PMD Tx. */
+ uint64_t n_pmd_rx = 0; /* owned by PMD Rx. */
+ uint64_t n_app_rx = 0; /* owned by Application on Rx. */
+ uint64_t n_app_alloc = 0; /* owned by Application on Alloc. */
+ uint64_t n_busy_tx = 0; /* owned by Application on Busy Tx. */
+ uint64_t n_total = 0; /* Total amount. */
+
+ if (f == NULL)
+ return;
+
+ STAILQ_FOREACH(hdr, &mp->elt_list, next) {
+ uint64_t hs = hdr->history;
+
+ rte_rmb();
+ n_total++;
+ if (hs == 0) {
+ n_never++;
+ continue;
+ }
+ switch (hs & RTE_MEMPOOL_HISTORY_MASK) {
+ case RTE_MEMPOOL_FREE:
+ n_free++;
+ break;
+ case RTE_MEMPOOL_PMD_FREE:
+ n_alloc++;
+ n_ref++;
+ break;
+ case RTE_MEMPOOL_PMD_TX:
+ n_alloc++;
+ n_pmd_tx++;
+ break;
+ case RTE_MEMPOOL_APP_RX:
+ n_alloc++;
+ n_app_rx++;
+ break;
+ case RTE_MEMPOOL_PMD_ALLOC:
+ n_alloc++;
+ n_pmd_rx++;
+ break;
+ case RTE_MEMPOOL_ALLOC:
+ n_alloc++;
+ n_app_alloc++;
+ break;
+ case RTE_MEMPOOL_BUSY_TX:
+ n_alloc++;
+ n_busy_tx++;
+ break;
+ default:
+ break;
+ }
+ fprintf(f, "%016" PRIX64 "\n", hs);
+ }
+
+ fprintf(f, "\n"
+ "Populated: %u\n"
+ "Never allocated: %" PRIu64 "\n"
+ "Free: %" PRIu64 "\n"
+ "Allocated: %" PRIu64 "\n"
+ "Referenced free: %" PRIu64 "\n"
+ "PMD owned Tx: %" PRIu64 "\n"
+ "PMD owned Rx: %" PRIu64 "\n"
+ "App owned alloc: %" PRIu64 "\n"
+ "App owned Rx: %" PRIu64 "\n"
+ "App owned busy: %" PRIu64 "\n"
+ "Counted total: %" PRIu64 "\n",
+ mp->populated_size, n_never, n_free + n_never, n_alloc,
+ n_ref, n_pmd_tx, n_pmd_rx, n_app_alloc, n_app_rx,
+ n_busy_tx, n_total);
+}
+#endif
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_mempool_objects_dump, 24.07)
+void
+rte_mempool_objects_dump(__rte_unused FILE *f)
+{
+ #if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+ if (f == NULL) {
+ RTE_MEMPOOL_LOG(ERR, "Invalid file pointer");
+ return;
+ }
+
+ struct rte_mempool *mp = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_mempool_list *mempool_list;
+
+ mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
+
+ rte_mcfg_mempool_read_lock();
+
+ TAILQ_FOREACH(te, mempool_list, next) {
+ mp = (struct rte_mempool *) te->data;
+ rte_mempool_get_object_history_stat(f, mp);
+ }
+
+ rte_mcfg_mempool_read_unlock();
+#else
+ RTE_MEMPOOL_LOG(INFO, "Mempool history recorder is not supported");
+#endif
+}
@@ -60,6 +60,26 @@ extern "C" {
#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL /**< Header cookie. */
#define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
+/**
+ * Mempool trace operation bits and masks.
+ * Used to record the lifecycle of mempool objects through the system.
+ */
+#define RTE_MEMPOOL_HISTORY_BITS 4 /*Number of bits for history operation*/
+#define RTE_MEMPOOL_HISTORY_MASK ((1ULL << RTE_MEMPOOL_HISTORY_BITS) - 1)
+
+/* History operation types */
+enum rte_mempool_history_op {
+ RTE_MEMPOOL_NEVER = 0, /* Initial state - never allocated */
+ RTE_MEMPOOL_FREE = 1, /* Freed back to mempool */
+ RTE_MEMPOOL_PMD_FREE = 2, /* Freed by PMD back to mempool */
+ RTE_MEMPOOL_PMD_TX = 3, /* Sent to PMD for Tx */
+ RTE_MEMPOOL_APP_RX = 4, /* Returned to application on Rx */
+ RTE_MEMPOOL_PMD_ALLOC = 5, /* Allocated by PMD for Rx */
+ RTE_MEMPOOL_ALLOC = 6, /* Allocated by application */
+ RTE_MEMPOOL_BUSY_TX = 7, /* Returned to app due to Tx busy */
+ RTE_MEMPOOL_MAX = 8 /* Maximum trace operation value */
+};
+
#ifdef RTE_LIBRTE_MEMPOOL_STATS
/**
* A structure that stores the mempool statistics (per-lcore).
@@ -157,6 +177,9 @@ struct rte_mempool_objhdr {
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
uint64_t cookie; /**< Debug cookie. */
#endif
+#if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+ uint64_t history; /**< Debug object history. */
+#endif
};
/**
@@ -457,6 +480,83 @@ void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
#define RTE_MEMPOOL_OPS_NAMESIZE 32 /**< Max length of ops struct name. */
+
+#if RTE_MEMPOOL_DEBUG_OBJECTS_HISTORY
+/**
+ * Get the history value from a mempool object header.
+ *
+ * @param obj
+ * Pointer to the mempool object.
+ * @return
+ * The history value from the object header.
+ */
+static inline uint64_t rte_mempool_history_get(void *obj)
+{
+ struct rte_mempool_objhdr *hdr;
+
+ if (unlikely(obj == NULL))
+ return 0;
+
+ hdr = rte_mempool_get_header(obj);
+ return hdr->history;
+}
+
+/**
+ * Mark a mempool object with the history value.
+ *
+ * @param obj
+ * Pointer to the mempool object.
+ * @param mark
+ * The history mark value to add.
+ */
+static inline void rte_mempool_history_mark(void *obj, uint32_t mark)
+{
+ struct rte_mempool_objhdr *hdr;
+
+ if (unlikely(obj == NULL))
+ return;
+
+ hdr = rte_mempool_get_header(obj);
+ hdr->history = (hdr->history << RTE_MEMPOOL_HISTORY_BITS) | mark;
+}
+
+/**
+ * Mark multiple mempool objects with the history value.
+ *
+ * @param b
+ * Array of pointers to mempool objects.
+ * @param n
+ * Number of objects to mark.
+ * @param mark
+ * The history mark value to add to each object.
+ */
+static inline void rte_mempool_history_bulk(void * const *b, uint32_t n, uint32_t mark)
+{
+ if (unlikely(b == NULL))
+ return;
+
+ while (n--)
+ rte_mempool_history_mark(*b++, mark);
+}
+#else
+static inline uint64_t rte_mempool_history_get(void *obj)
+{
+ RTE_SET_USED(obj);
+ return 0;
+}
+static inline void rte_mempool_history_mark(void *obj, uint32_t mark)
+{
+ RTE_SET_USED(obj);
+ RTE_SET_USED(mark);
+}
+static inline void rte_mempool_history_bulk(void * const *b, uint32_t n, uint32_t mark)
+{
+ RTE_SET_USED(b);
+ RTE_SET_USED(n);
+ RTE_SET_USED(mark);
+}
+#endif
+
/**
* Prototype for implementation specific data provisioning function.
*
@@ -1395,6 +1495,7 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
/* Increment stats now, adding in mempool always succeeds. */
RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
+ rte_mempool_history_bulk(obj_table, n, RTE_MEMPOOL_FREE);
__rte_assume(cache->flushthresh <= RTE_MEMPOOL_CACHE_MAX_SIZE * 2);
__rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE * 2);
@@ -1661,6 +1762,7 @@ rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
if (likely(ret == 0))
RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
+ rte_mempool_history_bulk(obj_table, n, RTE_MEMPOOL_ALLOC);
rte_mempool_trace_generic_get(mp, obj_table, n, cache);
return ret;
}
@@ -1876,6 +1978,10 @@ static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
}
+__rte_experimental
+void
+rte_mempool_objects_dump(FILE *f);
+
/**
* Dump the status of all mempools on the console
*