[4/5] vdpa/mlx5: workaround guest MR registrations
Checks
Commit Message
From: Michael Baum <michaelba@oss.nvidia.com>
Due to kernel issue in direct MKEY creation using the DevX API, this
patch replaces the virtio MR creation to use Verbs API.
Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")
Cc: stable@dpdk.org
Signed-off-by: Michael Baum <michaelba@oss.nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
drivers/vdpa/mlx5/mlx5_vdpa.h | 8 +++---
drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 41 +++++++++----------------------
2 files changed, 16 insertions(+), 33 deletions(-)
@@ -59,10 +59,10 @@ struct mlx5_vdpa_event_qp {
struct mlx5_vdpa_query_mr {
SLIST_ENTRY(mlx5_vdpa_query_mr) next;
- void *addr;
- uint64_t length;
- struct mlx5dv_devx_umem *umem;
- struct mlx5_devx_obj *mkey;
+ union {
+ struct ibv_mr *mr;
+ struct mlx5_devx_obj *mkey;
+ };
int is_indirect;
};
@@ -23,9 +23,10 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
entry = SLIST_FIRST(&priv->mr_list);
while (entry) {
next = SLIST_NEXT(entry, next);
- claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
- if (!entry->is_indirect)
- claim_zero(mlx5_glue->devx_umem_dereg(entry->umem));
+ if (entry->is_indirect)
+ claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
+ else
+ claim_zero(mlx5_glue->dereg_mr(entry->mr));
SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
rte_free(entry);
entry = next;
@@ -202,7 +203,6 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
goto error;
}
DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
- memset(&mkey_attr, 0, sizeof(mkey_attr));
for (i = 0; i < mem->nregions; i++) {
reg = &mem->regions[i];
entry = rte_zmalloc(__func__, sizeof(*entry), 0);
@@ -211,28 +211,15 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
DRV_LOG(ERR, "Failed to allocate mem entry memory.");
goto error;
}
- entry->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
- (void *)(uintptr_t)reg->host_user_addr,
- reg->size, IBV_ACCESS_LOCAL_WRITE);
- if (!entry->umem) {
- DRV_LOG(ERR, "Failed to register Umem by Devx.");
- ret = -errno;
- goto error;
- }
- mkey_attr.addr = (uintptr_t)(reg->guest_phys_addr);
- mkey_attr.size = reg->size;
- mkey_attr.umem_id = entry->umem->umem_id;
- mkey_attr.pd = priv->cdev->pdn;
- mkey_attr.pg_access = 1;
- entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx,
- &mkey_attr);
- if (!entry->mkey) {
+ entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
+ (void *)(uintptr_t)(reg->host_user_addr),
+ reg->size, reg->guest_phys_addr,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!entry->mr) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
ret = -rte_errno;
goto error;
}
- entry->addr = (void *)(uintptr_t)(reg->host_user_addr);
- entry->length = reg->size;
entry->is_indirect = 0;
if (i > 0) {
uint64_t sadd;
@@ -262,12 +249,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
for (k = 0; k < reg->size; k += klm_size) {
klm_array[klm_index].byte_count = k + klm_size >
reg->size ? reg->size - k : klm_size;
- klm_array[klm_index].mkey = entry->mkey->id;
+ klm_array[klm_index].mkey = entry->mr->lkey;
klm_array[klm_index].address = reg->guest_phys_addr + k;
klm_index++;
}
SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
}
+ memset(&mkey_attr, 0, sizeof(mkey_attr));
mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
mkey_attr.size = mem_size;
mkey_attr.pd = priv->cdev->pdn;
@@ -295,13 +283,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
priv->gpa_mkey_index = entry->mkey->id;
return 0;
error:
- if (entry) {
- if (entry->mkey)
- mlx5_devx_cmd_destroy(entry->mkey);
- if (entry->umem)
- mlx5_glue->devx_umem_dereg(entry->umem);
+ if (entry)
rte_free(entry);
- }
mlx5_vdpa_mem_dereg(priv);
rte_errno = -ret;
return ret;