diff mbox series

[3/7] vdpa/mlx5: no kick handling during shutdown

Message ID 20220224132820.1939650-4-xuemingl@nvidia.com (mailing list archive)
State Superseded, archived
Delegated to: Maxime Coquelin
Headers
Series vdpa/mlx5: improve device shutdown time |

Checks

Context Check Description
ci/checkpatch warning coding style issues

Commit Message

Xueming Li Feb. 24, 2022, 1:28 p.m. UTC 
  When Qemu suspend a VM, hw notifier is un-mmapped while vCPU thread may
still active and write notifier through kick socket.

PMD kick handler thread tries to install hw notifier through slave
socket in such case will timeout and slow down device close.

This patch skips hw notifier install if VQ or device in middle of
shutdown.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.c       | 17 ++++++++++-------
 drivers/vdpa/mlx5/mlx5_vdpa.h       |  8 +++++++-
 drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 12 +++++++++++-
 3 files changed, 28 insertions(+), 9 deletions(-)
diff mbox series

Patch

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 749c9d097cf..48f20d9ecdb 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -252,13 +252,15 @@  mlx5_vdpa_dev_close(int vid)
 	}
 	mlx5_vdpa_err_event_unset(priv);
 	mlx5_vdpa_cqe_event_unset(priv);
-	if (priv->configured)
+	if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
 		ret |= mlx5_vdpa_lm_log(priv);
+		priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
+	}
 	mlx5_vdpa_steer_unset(priv);
 	mlx5_vdpa_virtqs_release(priv);
 	mlx5_vdpa_event_qp_global_release(priv);
 	mlx5_vdpa_mem_dereg(priv);
-	priv->configured = 0;
+	priv->state = MLX5_VDPA_STATE_PROBED;
 	priv->vid = 0;
 	/* The mutex may stay locked after event thread cancel - initiate it. */
 	pthread_mutex_init(&priv->vq_config_lock, NULL);
@@ -277,7 +279,8 @@  mlx5_vdpa_dev_config(int vid)
 		DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 		return -EINVAL;
 	}
-	if (priv->configured && mlx5_vdpa_dev_close(vid)) {
+	if (priv->state == MLX5_VDPA_STATE_CONFIGURED &&
+	    mlx5_vdpa_dev_close(vid)) {
 		DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
 		return -1;
 	}
@@ -291,7 +294,7 @@  mlx5_vdpa_dev_config(int vid)
 		mlx5_vdpa_dev_close(vid);
 		return -1;
 	}
-	priv->configured = 1;
+	priv->state = MLX5_VDPA_STATE_CONFIGURED;
 	DRV_LOG(INFO, "vDPA device %d was configured.", vid);
 	return 0;
 }
@@ -373,7 +376,7 @@  mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
 		DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 		return -ENODEV;
 	}
-	if (!priv->configured) {
+	if (priv->state == MLX5_VDPA_STATE_PROBED) {
 		DRV_LOG(ERR, "Device %s was not configured.",
 				vdev->device->name);
 		return -ENODATA;
@@ -401,7 +404,7 @@  mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
 		DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
 		return -ENODEV;
 	}
-	if (!priv->configured) {
+	if (priv->state == MLX5_VDPA_STATE_PROBED) {
 		DRV_LOG(ERR, "Device %s was not configured.",
 				vdev->device->name);
 		return -ENODATA;
@@ -594,7 +597,7 @@  mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
 		TAILQ_REMOVE(&priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (found) {
-		if (priv->configured)
+		if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
 			mlx5_vdpa_dev_close(priv->vid);
 		if (priv->var) {
 			mlx5_glue->dv_free_var(priv->var);
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 22617924eac..cc83d7cba3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -113,9 +113,15 @@  enum {
 	MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT
 };
 
+enum mlx5_dev_state {
+	MLX5_VDPA_STATE_PROBED = 0,
+	MLX5_VDPA_STATE_CONFIGURED,
+	MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */
+};
+
 struct mlx5_vdpa_priv {
 	TAILQ_ENTRY(mlx5_vdpa_priv) next;
-	uint8_t configured;
+	enum mlx5_dev_state state;
 	pthread_mutex_t vq_config_lock;
 	uint64_t no_traffic_counter;
 	pthread_t timer_tid;
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
index e1e05924a40..b1d584ca8b0 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
@@ -25,6 +25,11 @@  mlx5_vdpa_virtq_kick_handler(void *cb_arg)
 	int nbytes;
 	int retry;
 
+	if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+		DRV_LOG(ERR,  "device %d queue %d down, skip kick handling",
+			priv->vid, virtq->index);
+		return;
+	}
 	if (rte_intr_fd_get(virtq->intr_handle) < 0)
 		return;
 	for (retry = 0; retry < 3; ++retry) {
@@ -43,6 +48,11 @@  mlx5_vdpa_virtq_kick_handler(void *cb_arg)
 	if (nbytes < 0)
 		return;
 	rte_write32(virtq->index, priv->virtq_db_addr);
+	if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
+		DRV_LOG(ERR,  "device %d queue %d down, skip kick handling",
+			priv->vid, virtq->index);
+		return;
+	}
 	if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
 		if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
 			virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
@@ -541,7 +551,7 @@  mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
 
 	DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
 		virtq->enable ? "en" : "dis", enable ? "en" : "dis");
-	if (!priv->configured) {
+	if (priv->state == MLX5_VDPA_STATE_PROBED) {
 		virtq->enable = !!enable;
 		return 0;
 	}