From patchwork Tue Mar 24 14:24:35 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Asaf Penso X-Patchwork-Id: 67070 X-Patchwork-Delegate: maxime.coquelin@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 34F66A057C; Tue, 24 Mar 2020 15:25:06 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id DB67F1C0AF; Tue, 24 Mar 2020 15:24:50 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 4148E2BAE for ; Tue, 24 Mar 2020 15:24:47 +0100 (CET) Received: from Internal Mail-Server by MTLPINE2 (envelope-from asafp@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2020 16:24:42 +0200 Received: from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx [10.210.16.112]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 02OEOgQd022008; Tue, 24 Mar 2020 16:24:42 +0200 From: Asaf Penso To: dev@dpdk.org Cc: Viacheslav Ovsiienko , Shahaf Shuler , Maxime Coquelin , Matan Azrad Date: Tue, 24 Mar 2020 14:24:35 +0000 Message-Id: <1585059877-2369-3-git-send-email-asafp@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1585059877-2369-1-git-send-email-asafp@mellanox.com> References: <1585059877-2369-1-git-send-email-asafp@mellanox.com> Subject: [dpdk-dev] [PATCH 2/4] vdpa/mlx5: support direct HW notifications X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Matan Azrad Add support for the next 2 callbacks: get_vfio_device_fd and get_notify_area. This will allow direct HW doorbell ringing from guest and will save CPU usage in host. By this patch, the QEMU will map the physical address of the virtio device in guest directly to the physical address of the HW device doorbell. The guest doorbell write is 2 bytes transaction while some Mellanox nics support only 4 bytes transactions. Remove ConnectX-5 and BF1 devices support which don't support 2B doorbell writes for HW triggering. Signed-off-by: Matan Azrad Acked-by: Viacheslav Ovsiienko Reviewed-by: Maxime Coquelin --- drivers/vdpa/mlx5/mlx5_vdpa.c | 74 ++++++++++++++++++++++++++++++++++++------- drivers/vdpa/mlx5/mlx5_vdpa.h | 1 + 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c index 5542c29..4eb6abf 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa.c +++ b/drivers/vdpa/mlx5/mlx5_vdpa.c @@ -133,6 +133,29 @@ } static int +mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv) +{ + int ret; + + if (priv->direct_notifier) { + ret = rte_vhost_host_notifier_ctrl(priv->vid, false); + if (ret != 0) { + DRV_LOG(INFO, "Direct HW notifier FD cannot be " + "destroyed for device %d: %d.", priv->vid, ret); + return -1; + } + priv->direct_notifier = 0; + } + ret = rte_vhost_host_notifier_ctrl(priv->vid, true); + if (ret != 0) + DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for" + " device %d: %d.", priv->vid, ret); + else + priv->direct_notifier = 1; + return 0; +} + +static int mlx5_vdpa_features_set(int vid) { int did = rte_vhost_get_vdpa_device_id(vid); @@ -209,8 +232,9 @@ return -1; } priv->vid = vid; - if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_virtqs_prepare(priv) || - mlx5_vdpa_steer_setup(priv) || mlx5_vdpa_cqe_event_setup(priv)) { + if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) || + mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) || + mlx5_vdpa_cqe_event_setup(priv)) { mlx5_vdpa_dev_close(vid); return -1; } @@ -218,6 +242,40 @@ return 0; } +static int +mlx5_vdpa_get_device_fd(int vid) +{ + int did = rte_vhost_get_vdpa_device_id(vid); + struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did); + + if (priv == NULL) { + DRV_LOG(ERR, "Invalid device id: %d.", did); + return -EINVAL; + } + return priv->ctx->cmd_fd; +} + +static int +mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size) +{ + int did = rte_vhost_get_vdpa_device_id(vid); + struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did); + + RTE_SET_USED(qid); + if (priv == NULL) { + DRV_LOG(ERR, "Invalid device id: %d.", did); + return -EINVAL; + } + if (!priv->var) { + DRV_LOG(ERR, "VAR was not created for device %d, is the device" + " configured?.", did); + return -EINVAL; + } + *offset = priv->var->mmap_off; + *size = priv->var->length; + return 0; +} + static struct rte_vdpa_dev_ops mlx5_vdpa_ops = { .get_queue_num = mlx5_vdpa_get_queue_num, .get_features = mlx5_vdpa_get_vdpa_features, @@ -228,8 +286,8 @@ .set_features = mlx5_vdpa_features_set, .migration_done = NULL, .get_vfio_group_fd = NULL, - .get_vfio_device_fd = NULL, - .get_notify_area = NULL, + .get_vfio_device_fd = mlx5_vdpa_get_device_fd, + .get_notify_area = mlx5_vdpa_get_notify_area, }; static struct ibv_device * @@ -520,14 +578,6 @@ static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = { { RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, - PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) - }, - { - RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, - PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF) - }, - { - RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX6) }, { diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h index 3324c9d..75af410 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/mlx5_vdpa.h @@ -100,6 +100,7 @@ struct mlx5_vdpa_steer { struct mlx5_vdpa_priv { TAILQ_ENTRY(mlx5_vdpa_priv) next; uint8_t configured; + uint8_t direct_notifier; /* Whether direct notifier is on or off. */ int id; /* vDPA device id. */ int vid; /* vhost device id. */ struct ibv_context *ctx; /* Device context. */