@@ -1001,6 +1001,17 @@ Driver options
Enabled by default.
+- ``mr_mempool_reg_en`` parameter [int]
+
+ A nonzero value enables implicit registration of DMA memory of all mempools
+ except those having ``MEMPOOL_F_NON_IO``. The effect is that when a packet
+ from a mempool is transmitted, its memory is already registered for DMA
+ in the PMD and no registration will happen on the data path. The tradeoff is
+ extra work on the creation of each mempool and increased HW resource use
+ if some mempools are not used with MLX5 devices.
+
+ Enabled by default.
+
- ``representor`` parameter [list]
This parameter can be used to instantiate DPDK Ethernet devices from
@@ -55,6 +55,12 @@ New Features
Also, make sure to start the actual text at the margin.
=======================================================
+* **Updated Mellanox mlx5 driver.**
+
+ Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+ * Added implicit mempool registration to avoid data path hiccups (opt-out).
+
Removed Items
-------------
@@ -20,6 +20,45 @@
#include "mlx5_tx.h"
#include "mlx5_utils.h"
+/**
+ * Handle a port-agnostic message.
+ *
+ * @return
+ * 0 on success, 1 when message is not port-agnostic, (-1) on error.
+ */
+static int
+mlx5_mp_os_handle_port_agnostic(const struct rte_mp_msg *mp_msg,
+ const void *peer)
+{
+ struct rte_mp_msg mp_res;
+ struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param;
+ const struct mlx5_mp_param *param =
+ (const struct mlx5_mp_param *)mp_msg->param;
+ const struct mlx5_mp_arg_mempool_reg *mpr;
+ struct mlx5_mp_id mp_id;
+
+ switch (param->type) {
+ case MLX5_MP_REQ_MEMPOOL_REGISTER:
+ mlx5_mp_id_init(&mp_id, param->port_id);
+ mp_init_msg(&mp_id, &mp_res, param->type);
+ mpr = ¶m->args.mempool_reg;
+ res->result = mlx5_mr_mempool_register(mpr->share_cache,
+ mpr->pd, mpr->mempool,
+ NULL);
+ return rte_mp_reply(&mp_res, peer);
+ case MLX5_MP_REQ_MEMPOOL_UNREGISTER:
+ mlx5_mp_id_init(&mp_id, param->port_id);
+ mp_init_msg(&mp_id, &mp_res, param->type);
+ mpr = ¶m->args.mempool_reg;
+ res->result = mlx5_mr_mempool_unregister(mpr->share_cache,
+ mpr->mempool, NULL);
+ return rte_mp_reply(&mp_res, peer);
+ default:
+ return 1;
+ }
+ return -1;
+}
+
int
mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
{
@@ -34,6 +73,11 @@ mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
int ret;
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /* Port-agnostic messages. */
+ ret = mlx5_mp_os_handle_port_agnostic(mp_msg, peer);
+ if (ret <= 0)
+ return ret;
+ /* Port-specific messages. */
if (!rte_eth_dev_is_valid_port(param->port_id)) {
rte_errno = ENODEV;
DRV_LOG(ERR, "port %u invalid port ID", param->port_id);
@@ -1034,8 +1034,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
err = mlx5_proc_priv_init(eth_dev);
if (err)
return NULL;
- mp_id.port_id = eth_dev->data->port_id;
- strlcpy(mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
+ mlx5_mp_id_init(&mp_id, eth_dev->data->port_id);
/* Receive command fd from primary process */
err = mlx5_mp_req_verbs_cmd_fd(&mp_id);
if (err < 0)
@@ -2136,6 +2135,7 @@ mlx5_os_config_default(struct mlx5_dev_config *config)
config->txqs_inline = MLX5_ARG_UNSET;
config->vf_nl_en = 1;
config->mr_ext_memseg_en = 1;
+ config->mr_mempool_reg_en = 1;
config->mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN;
config->mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS;
config->dv_esw_en = 1;
@@ -20,5 +20,7 @@ enum {
#define MLX5_NAMESIZE IF_NAMESIZE
int mlx5_auxiliary_get_ifindex(const char *sf_name);
+void mlx5_mempool_event_cb(enum rte_mempool_event event,
+ struct rte_mempool *mp, void *arg);
#endif /* RTE_PMD_MLX5_OS_H_ */
@@ -178,6 +178,9 @@
/* Device parameter to configure allow or prevent duplicate rules pattern. */
#define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern"
+/* Device parameter to configure implicit registration of mempool memory. */
+#define MLX5_MR_MEMPOOL_REG_EN "mr_mempool_reg_en"
+
/* Shared memory between primary and secondary processes. */
struct mlx5_shared_data *mlx5_shared_data;
@@ -1085,6 +1088,120 @@ mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh,
return err;
}
+/**
+ * Register the mempool for the protection domain.
+ *
+ * @param sh
+ * Pointer to the device shared context.
+ * @param mp
+ * Mempool being registered.
+ */
+static void
+mlx5_dev_ctx_shared_mempool_register(struct mlx5_dev_ctx_shared *sh,
+ struct rte_mempool *mp)
+{
+ struct mlx5_mp_id mp_id;
+
+ mlx5_mp_id_init(&mp_id, 0);
+ if (mlx5_mr_mempool_register(&sh->share_cache, sh->pd, mp, &mp_id) < 0)
+ DRV_LOG(ERR, "Failed to register mempool %s for PD %p: %s",
+ mp->name, sh->pd, rte_strerror(rte_errno));
+}
+
+/**
+ * Unregister the mempool from the protection domain.
+ *
+ * @param sh
+ * Pointer to the device shared context.
+ * @param mp
+ * Mempool being unregistered.
+ */
+static void
+mlx5_dev_ctx_shared_mempool_unregister(struct mlx5_dev_ctx_shared *sh,
+ struct rte_mempool *mp)
+{
+ struct mlx5_mp_id mp_id;
+
+ mlx5_mp_id_init(&mp_id, 0);
+ if (mlx5_mr_mempool_unregister(&sh->share_cache, mp, &mp_id) < 0)
+ DRV_LOG(WARNING, "Failed to unregister mempool %s for PD %p: %s",
+ mp->name, sh->pd, rte_strerror(rte_errno));
+}
+
+/**
+ * rte_mempool_walk() callback to register mempools
+ * for the protection domain.
+ *
+ * @param mp
+ * The mempool being walked.
+ * @param arg
+ * Pointer to the device shared context.
+ */
+static void
+mlx5_dev_ctx_shared_mempool_register_cb(struct rte_mempool *mp, void *arg)
+{
+ mlx5_dev_ctx_shared_mempool_register
+ ((struct mlx5_dev_ctx_shared *)arg, mp);
+}
+
+/**
+ * rte_mempool_walk() callback to unregister mempools
+ * from the protection domain.
+ *
+ * @param mp
+ * The mempool being walked.
+ * @param arg
+ * Pointer to the device shared context.
+ */
+static void
+mlx5_dev_ctx_shared_mempool_unregister_cb(struct rte_mempool *mp, void *arg)
+{
+ mlx5_dev_ctx_shared_mempool_unregister
+ ((struct mlx5_dev_ctx_shared *)arg, mp);
+}
+
+/**
+ * Mempool life cycle callback for Ethernet devices.
+ *
+ * @param event
+ * Mempool life cycle event.
+ * @param mp
+ * Associated mempool.
+ * @param arg
+ * Pointer to a device shared context.
+ */
+static void
+mlx5_dev_ctx_shared_mempool_event_cb(enum rte_mempool_event event,
+ struct rte_mempool *mp, void *arg)
+{
+ struct mlx5_dev_ctx_shared *sh = arg;
+
+ switch (event) {
+ case RTE_MEMPOOL_EVENT_CREATE:
+ mlx5_dev_ctx_shared_mempool_register(sh, mp);
+ break;
+ case RTE_MEMPOOL_EVENT_DESTROY:
+ mlx5_dev_ctx_shared_mempool_unregister(sh, mp);
+ break;
+ }
+}
+
+int
+mlx5_dev_ctx_shared_mempool_subscribe(struct mlx5_dev_ctx_shared *sh)
+{
+ int ret;
+
+ /* Callback for this shared context may be already registered. */
+ ret = rte_mempool_event_callback_register
+ (mlx5_dev_ctx_shared_mempool_event_cb, sh);
+ if (ret != 0 && rte_errno != EEXIST)
+ return ret;
+ /* Register mempools only once for this shared context. */
+ if (ret == 0)
+ rte_mempool_walk(mlx5_dev_ctx_shared_mempool_register_cb, sh);
+ return 0;
+}
+
/**
* Allocate shared device context. If there is multiport device the
* master and representors will share this context, if there is single
@@ -1282,6 +1399,8 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
void
mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
{
+ int ret;
+
pthread_mutex_lock(&mlx5_dev_ctx_list_mutex);
#ifdef RTE_LIBRTE_MLX5_DEBUG
/* Check the object presence in the list. */
@@ -1302,6 +1421,12 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
if (--sh->refcnt)
goto exit;
+ /* Stop watching for mempool events and unregister all mempools. */
+ ret = rte_mempool_event_callback_unregister
+ (mlx5_dev_ctx_shared_mempool_event_cb, sh);
+ if (ret == 0 || rte_errno != ENOENT)
+ rte_mempool_walk(mlx5_dev_ctx_shared_mempool_unregister_cb,
+ sh);
/* Remove from memory callback device list. */
rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
LIST_REMOVE(sh, mem_event_cb);
@@ -1991,6 +2116,8 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
config->decap_en = !!tmp;
} else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) {
config->allow_duplicate_pattern = !!tmp;
+ } else if (strcmp(MLX5_MR_MEMPOOL_REG_EN, key) == 0) {
+ config->mr_mempool_reg_en = !!tmp;
} else {
DRV_LOG(WARNING, "%s: unknown parameter", key);
rte_errno = EINVAL;
@@ -2051,6 +2178,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
MLX5_SYS_MEM_EN,
MLX5_DECAP_EN,
MLX5_ALLOW_DUPLICATE_PATTERN,
+ MLX5_MR_MEMPOOL_REG_EN,
NULL,
};
struct rte_kvargs *kvlist;
@@ -155,6 +155,13 @@ struct mlx5_flow_dump_ack {
/** Key string for IPC. */
#define MLX5_MP_NAME "net_mlx5_mp"
+/** Initialize a multi-process ID. */
+static inline void
+mlx5_mp_id_init(struct mlx5_mp_id *mp_id, uint16_t port_id)
+{
+ mp_id->port_id = port_id;
+ strlcpy(mp_id->name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
+}
LIST_HEAD(mlx5_dev_list, mlx5_dev_ctx_shared);
@@ -175,6 +182,9 @@ struct mlx5_local_data {
extern struct mlx5_shared_data *mlx5_shared_data;
+/* Exposed to copy into the shared data in OS-specific module. */
+extern int mlx5_net_mempool_slot;
+
/* Dev ops structs */
extern const struct eth_dev_ops mlx5_dev_ops;
extern const struct eth_dev_ops mlx5_dev_sec_ops;
@@ -270,6 +280,8 @@ struct mlx5_dev_config {
unsigned int dv_miss_info:1; /* restore packet after partial hw miss */
unsigned int allow_duplicate_pattern:1;
/* Allow/Prevent the duplicate rules pattern. */
+ unsigned int mr_mempool_reg_en:1;
+ /* Allow/prevent implicit mempool memory registration. */
struct {
unsigned int enabled:1; /* Whether MPRQ is enabled. */
unsigned int stride_num_n; /* Number of strides. */
@@ -1497,6 +1509,7 @@ struct mlx5_dev_ctx_shared *
mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
const struct mlx5_dev_config *config);
void mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh);
+int mlx5_dev_ctx_shared_mempool_subscribe(struct mlx5_dev_ctx_shared *sh);
void mlx5_free_table_hash_list(struct mlx5_priv *priv);
int mlx5_alloc_table_hash_list(struct mlx5_priv *priv);
void mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
@@ -128,9 +128,36 @@ mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
uint32_t
mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
{
+ struct mlx5_txq_ctrl *txq_ctrl =
+ container_of(txq, struct mlx5_txq_ctrl, txq);
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct mlx5_priv *priv = txq_ctrl->priv;
uintptr_t addr = (uintptr_t)mb->buf_addr;
uint32_t lkey;
+ if (priv->config.mr_mempool_reg_en) {
+ struct rte_mempool *mp = NULL;
+ struct mlx5_mprq_buf *buf;
+
+ if (!RTE_MBUF_HAS_EXTBUF(mb)) {
+ mp = mlx5_mb2mp(mb);
+ } else if (mb->shinfo->free_cb == mlx5_mprq_buf_free_cb) {
+ /* Recover MPRQ mempool. */
+ buf = mb->shinfo->fcb_opaque;
+ mp = buf->mp;
+ }
+ if (mp != NULL) {
+ lkey = mlx5_mr_mempool2mr_bh(&priv->sh->share_cache,
+ mr_ctrl, mp, addr);
+ /*
+ * Lookup can only fail on invalid input, e.g. "addr"
+ * is not from "mp" or "mp" has MEMPOOL_F_NON_IO set.
+ */
+ if (lkey != UINT32_MAX)
+ return lkey;
+ }
+ /* Fallback for generic mechanism in corner cases. */
+ }
lkey = mlx5_tx_addr2mr_bh(txq, addr);
if (lkey == UINT32_MAX && rte_errno == ENXIO) {
/* Mempool may have externally allocated memory. */
@@ -1124,6 +1124,13 @@ mlx5_dev_start(struct rte_eth_dev *dev)
dev->data->port_id, strerror(rte_errno));
goto error;
}
+ if (priv->config.mr_mempool_reg_en) {
+ if (mlx5_dev_ctx_shared_mempool_subscribe(priv->sh) != 0) {
+ DRV_LOG(ERR, "port %u failed to subscribe for mempool life cycle: %s",
+ dev->data->port_id, rte_strerror(rte_errno));
+ goto error;
+ }
+ }
rte_wmb();
dev->tx_pkt_burst = mlx5_select_tx_function(dev);
dev->rx_pkt_burst = mlx5_select_rx_function(dev);
@@ -1193,11 +1200,10 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
if (priv->obj_ops.lb_dummy_queue_release)
priv->obj_ops.lb_dummy_queue_release(dev);
mlx5_txpp_stop(dev);
-
return 0;
}
-/**
+/*
* Enable traffic flows configured by control plane
*
* @param dev