@@ -80,6 +80,7 @@ enum {
/* Vdpa task types. */
enum mlx5_vdpa_task_type {
MLX5_VDPA_TASK_REG_MR = 1,
+ MLX5_VDPA_TASK_SETUP_VIRTQ,
};
/* Generic task information and size must be multiple of 4B. */
@@ -117,12 +118,12 @@ struct mlx5_vdpa_vmem_info {
struct mlx5_vdpa_virtq {
SLIST_ENTRY(mlx5_vdpa_virtq) next;
- uint8_t enable;
uint16_t index;
uint16_t vq_size;
uint8_t notifier_state;
- bool stopped;
uint32_t configured:1;
+ uint32_t enable:1;
+ uint32_t stopped:1;
uint32_t version;
pthread_mutex_t virtq_lock;
struct mlx5_vdpa_priv *priv;
@@ -565,11 +566,13 @@ bool
mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv,
uint32_t thrd_idx,
enum mlx5_vdpa_task_type task_type,
- uint32_t *bulk_refcnt, uint32_t *bulk_err_cnt,
+ uint32_t *remaining_cnt, uint32_t *err_cnt,
void **task_data, uint32_t num);
int
mlx5_vdpa_register_mr(struct mlx5_vdpa_priv *priv, uint32_t idx);
bool
mlx5_vdpa_c_thread_wait_bulk_tasks_done(uint32_t *remaining_cnt,
uint32_t *err_cnt, uint32_t sleep_time);
+int
+mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index, bool reg_kick);
#endif /* RTE_PMD_MLX5_VDPA_H_ */
@@ -100,6 +100,7 @@ mlx5_vdpa_c_thread_handle(void *arg)
{
struct mlx5_vdpa_conf_thread_mng *multhrd = arg;
pthread_t thread_id = pthread_self();
+ struct mlx5_vdpa_virtq *virtq;
struct mlx5_vdpa_priv *priv;
struct mlx5_vdpa_task task;
struct rte_ring *rng;
@@ -139,6 +140,19 @@ mlx5_vdpa_c_thread_handle(void *arg)
__ATOMIC_RELAXED);
}
break;
+ case MLX5_VDPA_TASK_SETUP_VIRTQ:
+ virtq = &priv->virtqs[task.idx];
+ pthread_mutex_lock(&virtq->virtq_lock);
+ ret = mlx5_vdpa_virtq_setup(priv,
+ task.idx, false);
+ if (ret) {
+ DRV_LOG(ERR,
+ "Failed to setup virtq %d.", task.idx);
+ __atomic_fetch_add(
+ task.err_cnt, 1, __ATOMIC_RELAXED);
+ }
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ break;
default:
DRV_LOG(ERR, "Invalid vdpa task type %d.",
task.type);
@@ -371,7 +371,7 @@ mlx5_vdpa_err_interrupt_handler(void *cb_arg __rte_unused)
goto unlock;
if (rte_rdtsc() / rte_get_tsc_hz() < MLX5_VDPA_ERROR_TIME_SEC)
goto unlock;
- virtq->stopped = true;
+ virtq->stopped = 1;
/* Query error info. */
if (mlx5_vdpa_virtq_query(priv, vq_index))
goto log;
@@ -111,8 +111,9 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
for (i = 0; i < priv->caps.max_num_virtio_queues; i++) {
struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i];
+ if (virtq->index != i)
+ continue;
pthread_mutex_lock(&virtq->virtq_lock);
- virtq->configured = 0;
for (j = 0; j < RTE_DIM(virtq->umems); ++j) {
if (virtq->umems[j].obj) {
claim_zero(mlx5_glue->devx_umem_dereg
@@ -131,7 +132,6 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
}
}
-
static int
mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
{
@@ -191,7 +191,7 @@ mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index)
ret = mlx5_vdpa_virtq_modify(virtq, 0);
if (ret)
return -1;
- virtq->stopped = true;
+ virtq->stopped = 1;
DRV_LOG(DEBUG, "vid %u virtq %u was stopped.", priv->vid, index);
return mlx5_vdpa_virtq_query(priv, index);
}
@@ -411,7 +411,38 @@ mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv *priv)
}
static int
-mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index)
+mlx5_vdpa_virtq_doorbell_setup(struct mlx5_vdpa_virtq *virtq,
+ struct rte_vhost_vring *vq, int index)
+{
+ virtq->intr_handle =
+ rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+ if (virtq->intr_handle == NULL) {
+ DRV_LOG(ERR, "Fail to allocate intr_handle");
+ return -1;
+ }
+ if (rte_intr_fd_set(virtq->intr_handle, vq->kickfd))
+ return -1;
+ if (rte_intr_fd_get(virtq->intr_handle) == -1) {
+ DRV_LOG(WARNING, "Virtq %d kickfd is invalid.", index);
+ } else {
+ if (rte_intr_type_set(virtq->intr_handle,
+ RTE_INTR_HANDLE_EXT))
+ return -1;
+ if (rte_intr_callback_register(virtq->intr_handle,
+ mlx5_vdpa_virtq_kick_handler, virtq)) {
+ (void)rte_intr_fd_set(virtq->intr_handle, -1);
+ DRV_LOG(ERR, "Failed to register virtq %d interrupt.",
+ index);
+ return -1;
+ }
+ DRV_LOG(DEBUG, "Register fd %d interrupt for virtq %d.",
+ rte_intr_fd_get(virtq->intr_handle), index);
+ }
+ return 0;
+}
+
+int
+mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index, bool reg_kick)
{
struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index];
struct rte_vhost_vring vq;
@@ -455,33 +486,11 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index)
rte_write32(virtq->index, priv->virtq_db_addr);
rte_spinlock_unlock(&priv->db_lock);
/* Setup doorbell mapping. */
- virtq->intr_handle =
- rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
- if (virtq->intr_handle == NULL) {
- DRV_LOG(ERR, "Fail to allocate intr_handle");
- goto error;
- }
-
- if (rte_intr_fd_set(virtq->intr_handle, vq.kickfd))
- goto error;
-
- if (rte_intr_fd_get(virtq->intr_handle) == -1) {
- DRV_LOG(WARNING, "Virtq %d kickfd is invalid.", index);
- } else {
- if (rte_intr_type_set(virtq->intr_handle, RTE_INTR_HANDLE_EXT))
- goto error;
-
- if (rte_intr_callback_register(virtq->intr_handle,
- mlx5_vdpa_virtq_kick_handler,
- virtq)) {
- (void)rte_intr_fd_set(virtq->intr_handle, -1);
+ if (reg_kick) {
+ if (mlx5_vdpa_virtq_doorbell_setup(virtq, &vq, index)) {
DRV_LOG(ERR, "Failed to register virtq %d interrupt.",
index);
goto error;
- } else {
- DRV_LOG(DEBUG, "Register fd %d interrupt for virtq %d.",
- rte_intr_fd_get(virtq->intr_handle),
- index);
}
}
/* Subscribe virtq error event. */
@@ -497,7 +506,6 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index)
rte_errno = errno;
goto error;
}
- virtq->stopped = false;
/* Initial notification to ask Qemu handling completed buffers. */
if (virtq->eqp.cq.callfd != -1)
eventfd_write(virtq->eqp.cq.callfd, (eventfd_t)1);
@@ -567,10 +575,12 @@ mlx5_vdpa_features_validate(struct mlx5_vdpa_priv *priv)
int
mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
{
- uint32_t i;
- uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
int ret = rte_vhost_get_negotiated_features(priv->vid, &priv->features);
+ uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
+ uint32_t remaining_cnt = 0, err_cnt = 0, task_num = 0;
+ uint32_t i, thrd_idx, data[1];
struct mlx5_vdpa_virtq *virtq;
+ struct rte_vhost_vring vq;
if (ret || mlx5_vdpa_features_validate(priv)) {
DRV_LOG(ERR, "Failed to configure negotiated features.");
@@ -590,16 +600,83 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
return -1;
}
priv->nr_virtqs = nr_vring;
- for (i = 0; i < nr_vring; i++) {
- virtq = &priv->virtqs[i];
- if (virtq->enable) {
+ if (priv->use_c_thread) {
+ uint32_t main_task_idx[nr_vring];
+
+ for (i = 0; i < nr_vring; i++) {
+ virtq = &priv->virtqs[i];
+ if (!virtq->enable)
+ continue;
+ thrd_idx = i % (conf_thread_mng.max_thrds + 1);
+ if (!thrd_idx) {
+ main_task_idx[task_num] = i;
+ task_num++;
+ continue;
+ }
+ thrd_idx = priv->last_c_thrd_idx + 1;
+ if (thrd_idx >= conf_thread_mng.max_thrds)
+ thrd_idx = 0;
+ priv->last_c_thrd_idx = thrd_idx;
+ data[0] = i;
+ if (mlx5_vdpa_task_add(priv, thrd_idx,
+ MLX5_VDPA_TASK_SETUP_VIRTQ,
+ &remaining_cnt, &err_cnt,
+ (void **)&data, 1)) {
+ DRV_LOG(ERR, "Fail to add "
+ "task setup virtq (%d).", i);
+ main_task_idx[task_num] = i;
+ task_num++;
+ }
+ }
+ for (i = 0; i < task_num; i++) {
+ virtq = &priv->virtqs[main_task_idx[i]];
pthread_mutex_lock(&virtq->virtq_lock);
- if (mlx5_vdpa_virtq_setup(priv, i)) {
+ if (mlx5_vdpa_virtq_setup(priv,
+ main_task_idx[i], false)) {
pthread_mutex_unlock(&virtq->virtq_lock);
goto error;
}
pthread_mutex_unlock(&virtq->virtq_lock);
}
+ if (mlx5_vdpa_c_thread_wait_bulk_tasks_done(&remaining_cnt,
+ &err_cnt, 2000)) {
+ DRV_LOG(ERR,
+ "Failed to wait virt-queue setup tasks ready.");
+ goto error;
+ }
+ for (i = 0; i < nr_vring; i++) {
+ /* Setup doorbell mapping in order for Qume. */
+ virtq = &priv->virtqs[i];
+ pthread_mutex_lock(&virtq->virtq_lock);
+ if (!virtq->enable || !virtq->configured) {
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ continue;
+ }
+ if (rte_vhost_get_vhost_vring(priv->vid, i, &vq)) {
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ goto error;
+ }
+ if (mlx5_vdpa_virtq_doorbell_setup(virtq, &vq, i)) {
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ DRV_LOG(ERR,
+ "Failed to register virtq %d interrupt.", i);
+ goto error;
+ }
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ }
+ } else {
+ for (i = 0; i < nr_vring; i++) {
+ virtq = &priv->virtqs[i];
+ pthread_mutex_lock(&virtq->virtq_lock);
+ if (virtq->enable) {
+ if (mlx5_vdpa_virtq_setup(priv, i, true)) {
+ pthread_mutex_unlock(
+ &virtq->virtq_lock);
+ goto error;
+ }
+ }
+ pthread_mutex_unlock(&virtq->virtq_lock);
+ }
}
return 0;
error:
@@ -663,7 +740,7 @@ mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
mlx5_vdpa_virtq_unset(virtq);
}
if (enable) {
- ret = mlx5_vdpa_virtq_setup(priv, index);
+ ret = mlx5_vdpa_virtq_setup(priv, index, true);
if (ret) {
DRV_LOG(ERR, "Failed to setup virtq %d.", index);
return ret;