@@ -315,3 +315,4 @@
" libmlx5)");
mlx5_glue = NULL;
}
+
@@ -7,8 +7,11 @@
#include <stdio.h>
#include <rte_errno.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
#include "mlx5_common.h"
+#include "mlx5_common_os.h"
#include "mlx5_common_utils.h"
int mlx5_common_logtype;
@@ -150,3 +153,122 @@ static inline void mlx5_cpu_id(unsigned int level,
#endif
haswell_broadwell_cpu = 0;
}
+
+/**
+ * Allocate page of door-bells and register it using DevX API.
+ *
+ * @param [in] ctx
+ * Pointer to the device context.
+ *
+ * @return
+ * Pointer to new page on success, NULL otherwise.
+ */
+static struct mlx5_devx_dbr_page *
+mlx5_alloc_dbr_page(void *ctx)
+{
+ struct mlx5_devx_dbr_page *page;
+
+ /* Allocate space for door-bell page and management data. */
+ page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (!page) {
+ DRV_LOG(ERR, "cannot allocate dbr page");
+ return NULL;
+ }
+ /* Register allocated memory. */
+ page->umem = mlx5_glue->devx_umem_reg(ctx, page->dbrs,
+ MLX5_DBR_PAGE_SIZE, 0);
+ if (!page->umem) {
+ DRV_LOG(ERR, "cannot umem reg dbr page");
+ rte_free(page);
+ return NULL;
+ }
+ return page;
+}
+
+/**
+ * Find the next available door-bell, allocate new page if needed.
+ *
+ * @param [in] ctx
+ * Pointer to device context.
+ * @param [in] head
+ * Pointer to the head of dbr pages list.
+ * @param [out] dbr_page
+ * Door-bell page containing the page data.
+ *
+ * @return
+ * Door-bell address offset on success, a negative error value otherwise.
+ */
+int64_t
+mlx5_get_dbr(void *ctx, struct mlx5_dbr_page_list *head,
+ struct mlx5_devx_dbr_page **dbr_page)
+{
+ struct mlx5_devx_dbr_page *page = NULL;
+ uint32_t i, j;
+
+ LIST_FOREACH(page, head, next)
+ if (page->dbr_count < MLX5_DBR_PER_PAGE)
+ break;
+ if (!page) { /* No page with free door-bell exists. */
+ page = mlx5_alloc_dbr_page(ctx);
+ if (!page) /* Failed to allocate new page. */
+ return (-1);
+ LIST_INSERT_HEAD(head, page, next);
+ }
+ /* Loop to find bitmap part with clear bit. */
+ for (i = 0;
+ i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
+ i++)
+ ; /* Empty. */
+ /* Find the first clear bit. */
+ MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
+ j = rte_bsf64(~page->dbr_bitmap[i]);
+ page->dbr_bitmap[i] |= (UINT64_C(1) << j);
+ page->dbr_count++;
+ *dbr_page = page;
+ return (((i * 64) + j) * sizeof(uint64_t));
+}
+
+/**
+ * Release a door-bell record.
+ *
+ * @param [in] head
+ * Pointer to the head of dbr pages list.
+ * @param [in] umem_id
+ * UMEM ID of page containing the door-bell record to release.
+ * @param [in] offset
+ * Offset of door-bell record in page.
+ *
+ * @return
+ * 0 on success, a negative error value otherwise.
+ */
+int32_t
+mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+ uint64_t offset)
+{
+ struct mlx5_devx_dbr_page *page = NULL;
+ int ret = 0;
+
+ LIST_FOREACH(page, head, next)
+ /* Find the page this address belongs to. */
+ if (mlx5_os_get_umem_id(page->umem) == umem_id)
+ break;
+ if (!page)
+ return -EINVAL;
+ page->dbr_count--;
+ if (!page->dbr_count) {
+ /* Page not used, free it and remove from list. */
+ LIST_REMOVE(page, next);
+ if (page->umem)
+ ret = -mlx5_glue->devx_umem_dereg(page->umem);
+ rte_free(page);
+ } else {
+ /* Mark in bitmap that this door-bell is not in use. */
+ offset /= MLX5_DBR_SIZE;
+ int i = offset / 64;
+ int j = offset % 64;
+
+ page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
+ }
+ return ret;
+}
@@ -15,6 +15,7 @@
#include <rte_devargs.h>
#include "mlx5_prm.h"
+#include "mlx5_devx_cmds.h"
/* Bit-field manipulation. */
@@ -207,6 +208,36 @@ enum mlx5_class {
MLX5_CLASS_INVALID,
};
+#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
+#define MLX5_DBR_SIZE 8
+#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
+#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
+
+struct mlx5_devx_dbr_page {
+ /* Door-bell records, must be first member in structure. */
+ uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
+ LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
+ void *umem;
+ uint32_t dbr_count; /* Number of door-bell records in use. */
+ /* 1 bit marks matching door-bell is in use. */
+ uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
+};
+
+/* devX creation object */
+struct mlx5_devx_obj {
+ void *obj; /* The DV object. */
+ int id; /* The object ID. */
+};
+
+/* UMR memory buffer used to define 1 entry in indirect mkey. */
+struct mlx5_klm {
+ uint32_t byte_count;
+ uint32_t mkey;
+ uint64_t address;
+};
+
+LIST_HEAD(mlx5_dbr_page_list, mlx5_devx_dbr_page);
+
__rte_internal
enum mlx5_class mlx5_class_get(struct rte_devargs *devargs);
__rte_internal
@@ -214,7 +245,12 @@ void mlx5_translate_port_name(const char *port_name_in,
struct mlx5_switch_info *port_info_out);
void mlx5_glue_constructor(void);
size_t mlx5_os_get_page_size(void);
-
+__rte_internal
+int64_t mlx5_get_dbr(void *ctx, struct mlx5_dbr_page_list *head,
+ struct mlx5_devx_dbr_page **dbr_page);
+__rte_internal
+int32_t mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+ uint64_t offset);
extern uint8_t haswell_broadwell_cpu;
#endif /* RTE_PMD_MLX5_COMMON_H_ */
@@ -9,19 +9,6 @@
#include "mlx5_prm.h"
-/* devX creation object */
-struct mlx5_devx_obj {
- void *obj; /* The DV object. */
- int id; /* The object ID. */
-};
-
-/* UMR memory buffer used to define 1 entry in indirect mkey. */
-struct mlx5_klm {
- uint32_t byte_count;
- uint32_t mkey;
- uint64_t address;
-};
-
/* This is limitation of libibverbs: in length variable type is u16. */
#define MLX5_DEVX_MAX_KLM_ENTRIES ((UINT16_MAX - \
MLX5_ST_SZ_DW(create_mkey_in) * 4) / (MLX5_ST_SZ_DW(klm) * 4))
@@ -34,6 +34,9 @@ INTERNAL {
mlx5_devx_cmd_query_virtq;
mlx5_devx_get_out_command_status;
+ mlx5_get_dbr;
+ mlx5_release_dbr;
+
mlx5_mp_init_primary;
mlx5_mp_uninit_primary;
mlx5_mp_init_secondary;
@@ -1666,126 +1666,6 @@ struct mlx5_dev_ctx_shared *
DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
}
-/**
- * Allocate page of door-bells and register it using DevX API.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- *
- * @return
- * Pointer to new page on success, NULL otherwise.
- */
-static struct mlx5_devx_dbr_page *
-mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page;
-
- /* Allocate space for door-bell page and management data. */
- page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
- RTE_CACHE_LINE_SIZE, dev->device->numa_node);
- if (!page) {
- DRV_LOG(ERR, "port %u cannot allocate dbr page",
- dev->data->port_id);
- return NULL;
- }
- /* Register allocated memory. */
- page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
- MLX5_DBR_PAGE_SIZE, 0);
- if (!page->umem) {
- DRV_LOG(ERR, "port %u cannot umem reg dbr page",
- dev->data->port_id);
- rte_free(page);
- return NULL;
- }
- return page;
-}
-
-/**
- * Find the next available door-bell, allocate new page if needed.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [out] dbr_page
- * Door-bell page containing the page data.
- *
- * @return
- * Door-bell address offset on success, a negative error value otherwise.
- */
-int64_t
-mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- uint32_t i, j;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- if (page->dbr_count < MLX5_DBR_PER_PAGE)
- break;
- if (!page) { /* No page with free door-bell exists. */
- page = mlx5_alloc_dbr_page(dev);
- if (!page) /* Failed to allocate new page. */
- return (-1);
- LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
- }
- /* Loop to find bitmap part with clear bit. */
- for (i = 0;
- i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
- i++)
- ; /* Empty. */
- /* Find the first clear bit. */
- MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
- j = rte_bsf64(~page->dbr_bitmap[i]);
- page->dbr_bitmap[i] |= (UINT64_C(1) << j);
- page->dbr_count++;
- *dbr_page = page;
- return (((i * 64) + j) * sizeof(uint64_t));
-}
-
-/**
- * Release a door-bell record.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [in] umem_id
- * UMEM ID of page containing the door-bell record to release.
- * @param [in] offset
- * Offset of door-bell record in page.
- *
- * @return
- * 0 on success, a negative error value otherwise.
- */
-int32_t
-mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- int ret = 0;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- /* Find the page this address belongs to. */
- if (mlx5_os_get_umem_id(page->umem) == umem_id)
- break;
- if (!page)
- return -EINVAL;
- page->dbr_count--;
- if (!page->dbr_count) {
- /* Page not used, free it and remove from list. */
- LIST_REMOVE(page, next);
- if (page->umem)
- ret = -mlx5_glue->devx_umem_dereg(page->umem);
- rte_free(page);
- } else {
- /* Mark in bitmap that this door-bell is not in use. */
- offset /= MLX5_DBR_SIZE;
- int i = offset / 64;
- int j = offset % 64;
-
- page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
- }
- return ret;
-}
-
int
rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n)
{
@@ -500,21 +500,6 @@ struct mlx5_flow_tbl_resource {
#define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
#define MLX5_MAX_TABLES_FDB UINT16_MAX
-#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
-#define MLX5_DBR_SIZE 8
-#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
-#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
-
-struct mlx5_devx_dbr_page {
- /* Door-bell records, must be first member in structure. */
- uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
- LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
- void *umem;
- uint32_t dbr_count; /* Number of door-bell records in use. */
- /* 1 bit marks matching door-bell is in use. */
- uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
-};
-
/* ID generation structure. */
struct mlx5_flow_id_pool {
uint32_t *free_arr; /**< Pointer to the a array of free values. */
@@ -655,7 +640,7 @@ struct mlx5_priv {
/* Context for Verbs allocator. */
int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
- LIST_HEAD(dbrpage, mlx5_devx_dbr_page) dbrpgs; /* Door-bell pages. */
+ struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
struct mlx5_flow_id_pool *qrss_id_pool;
struct mlx5_hlist *mreg_cp_tbl;
@@ -682,10 +667,6 @@ struct mlx5_priv {
int mlx5_getenv_int(const char *);
int mlx5_proc_priv_init(struct rte_eth_dev *dev);
-int64_t mlx5_get_dbr(struct rte_eth_dev *dev,
- struct mlx5_devx_dbr_page **dbr_page);
-int32_t mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id,
- uint64_t offset);
int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
struct rte_eth_udp_tunnel *udp_tunnel);
uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
@@ -1414,7 +1414,8 @@ struct mlx5_rxq_obj *
struct mlx5_devx_dbr_page *dbr_page;
int64_t dbr_offset;
- dbr_offset = mlx5_get_dbr(dev, &dbr_page);
+ dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,
+ &dbr_page);
if (dbr_offset < 0)
goto error;
rxq_ctrl->dbr_offset = dbr_offset;
@@ -2102,7 +2103,8 @@ struct mlx5_rxq_ctrl *
rxq_ctrl->obj = NULL;
if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
if (rxq_ctrl->dbr_umem_id_valid)
- claim_zero(mlx5_release_dbr(dev, rxq_ctrl->dbr_umem_id,
+ claim_zero(mlx5_release_dbr(&priv->dbrpgs,
+ rxq_ctrl->dbr_umem_id,
rxq_ctrl->dbr_offset));
if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD)
mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);