Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/111182/?format=api
{ "id": 111182, "url": "http://patchwork.dpdk.org/api/patches/111182/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20220516111041.63914-5-xuan.ding@intel.com/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20220516111041.63914-5-xuan.ding@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20220516111041.63914-5-xuan.ding@intel.com", "date": "2022-05-16T11:10:40", "name": "[v8,4/5] vhost: support async dequeue for split ring", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "606ff8546391a881c32bb14a9769d8f907c6b3ec", "submitter": { "id": 1401, "url": "http://patchwork.dpdk.org/api/people/1401/?format=api", "name": "Ding, Xuan", "email": "xuan.ding@intel.com" }, "delegate": { "id": 2642, "url": "http://patchwork.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20220516111041.63914-5-xuan.ding@intel.com/mbox/", "series": [ { "id": 22951, "url": "http://patchwork.dpdk.org/api/series/22951/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=22951", "date": "2022-05-16T11:10:36", "name": "vhost: support async dequeue data path", "version": 8, "mbox": "http://patchwork.dpdk.org/series/22951/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/111182/comments/", "check": "success", "checks": "http://patchwork.dpdk.org/api/patches/111182/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 4C103A00BE;\n\tMon, 16 May 2022 13:16:12 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3CFBB42831;\n\tMon, 16 May 2022 13:15:56 +0200 (CEST)", "from mga01.intel.com (mga01.intel.com [192.55.52.88])\n by mails.dpdk.org (Postfix) with ESMTP id C77BB42830\n for <dev@dpdk.org>; Mon, 16 May 2022 13:15:52 +0200 (CEST)", "from orsmga007.jf.intel.com ([10.7.209.58])\n by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 16 May 2022 04:15:48 -0700", "from npg-dpdk-xuan-cbdma.sh.intel.com ([10.67.110.228])\n by orsmga007.jf.intel.com with ESMTP; 16 May 2022 04:15:45 -0700" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1652699753; x=1684235753;\n h=from:to:cc:subject:date:message-id:in-reply-to: references;\n bh=Pma87OusbLJjRuel1ya9VMNtUGgRuHpM/Q2BQ8lyDI0=;\n b=hCRVUCXqaxvcPKqnk+xB0Q34Q4U6XGPygs7cHs7rINulFHyvCuHk8Jhm\n /IntuAh3ABus7Arpba2/JvQj4L2WvIRLpshvWjkD7nS5BGXDs2ZNr2D7Z\n Ai7mAPoPf0TG/XJdxpz99MEXwtGUjTYIQA2zgpPVDxbRSn9HNlDx7NoSO\n +EPItJxLNRbsdMmA6RCn74+w4caO9tQsH6DsayMyxd4IZuKFtMG0msiSF\n Gdya2KZ/UN3/NcefZcbPKbKDd7OBzttX4JGYzlHKvq9Ym5xdqwWO/c/OJ\n ue0npX12brXk9sGaH9afQ4L8FvU/e3lL+79kjZFJDSWuuRs7OGvjUjH9F A==;", "X-IronPort-AV": [ "E=McAfee;i=\"6400,9594,10348\"; a=\"296063103\"", "E=Sophos;i=\"5.91,229,1647327600\"; d=\"scan'208\";a=\"296063103\"", "E=Sophos;i=\"5.91,229,1647327600\"; d=\"scan'208\";a=\"568272394\"" ], "X-ExtLoop1": "1", "From": "xuan.ding@intel.com", "To": "maxime.coquelin@redhat.com,\n\tchenbo.xia@intel.com", "Cc": "dev@dpdk.org, jiayu.hu@intel.com, cheng1.jiang@intel.com,\n sunil.pai.g@intel.com, liangma@liangbit.com,\n Xuan Ding <xuan.ding@intel.com>, Yuan Wang <yuanx.wang@intel.com>", "Subject": "[PATCH v8 4/5] vhost: support async dequeue for split ring", "Date": "Mon, 16 May 2022 11:10:40 +0000", "Message-Id": "<20220516111041.63914-5-xuan.ding@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20220516111041.63914-1-xuan.ding@intel.com>", "References": "<20220407152546.38167-1-xuan.ding@intel.com>\n <20220516111041.63914-1-xuan.ding@intel.com>", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "From: Xuan Ding <xuan.ding@intel.com>\n\nThis patch implements asynchronous dequeue data path for vhost split\nring, a new API rte_vhost_async_try_dequeue_burst() is introduced.\n\nSigned-off-by: Xuan Ding <xuan.ding@intel.com>\nSigned-off-by: Yuan Wang <yuanx.wang@intel.com>\nTested-by: Yvonne Yang <yvonnex.yang@intel.com>\nReviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>\n---\n doc/guides/prog_guide/vhost_lib.rst | 6 +\n doc/guides/rel_notes/release_22_07.rst | 5 +\n lib/vhost/rte_vhost_async.h | 37 +++\n lib/vhost/version.map | 2 +-\n lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++\n 5 files changed, 386 insertions(+), 1 deletion(-)", "diff": "diff --git a/doc/guides/prog_guide/vhost_lib.rst b/doc/guides/prog_guide/vhost_lib.rst\nindex f287b76ebf..98f4509d1a 100644\n--- a/doc/guides/prog_guide/vhost_lib.rst\n+++ b/doc/guides/prog_guide/vhost_lib.rst\n@@ -282,6 +282,12 @@ The following is an overview of some key Vhost API functions:\n Clear inflight packets which are submitted to DMA engine in vhost async data\n path. Completed packets are returned to applications through ``pkts``.\n \n+* ``rte_vhost_async_try_dequeue_burst(vid, queue_id, mbuf_pool, pkts, count,\n+ nr_inflight, dma_id, vchan_id)``\n+\n+ Receive ``count`` packets from guest to host in async data path,\n+ and store them at ``pkts``.\n+\n Vhost-user Implementations\n --------------------------\n \ndiff --git a/doc/guides/rel_notes/release_22_07.rst b/doc/guides/rel_notes/release_22_07.rst\nindex 88b1e478d4..564d88623e 100644\n--- a/doc/guides/rel_notes/release_22_07.rst\n+++ b/doc/guides/rel_notes/release_22_07.rst\n@@ -70,6 +70,11 @@ New Features\n Added an API which can get the number of inflight packets in\n vhost async data path without using lock.\n \n+* **Added vhost async dequeue API to receive pkts from guest.**\n+\n+ Added vhost async dequeue API which can leverage DMA devices to\n+ accelerate receiving pkts from guest.\n+\n Removed Items\n -------------\n \ndiff --git a/lib/vhost/rte_vhost_async.h b/lib/vhost/rte_vhost_async.h\nindex 70234debf9..a1e7f674ed 100644\n--- a/lib/vhost/rte_vhost_async.h\n+++ b/lib/vhost/rte_vhost_async.h\n@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,\n __rte_experimental\n int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);\n \n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice\n+ *\n+ * This function tries to receive packets from the guest with offloading\n+ * copies to the DMA vChannels. Successfully dequeued packets are returned\n+ * in \"pkts\". The other packets that their copies are submitted to\n+ * the DMA vChannels but not completed are called \"in-flight packets\".\n+ * This function will not return in-flight packets until their copies are\n+ * completed by the DMA vChannels.\n+ *\n+ * @param vid\n+ * ID of vhost device to dequeue data\n+ * @param queue_id\n+ * ID of virtqueue to dequeue data\n+ * @param mbuf_pool\n+ * Mbuf_pool where host mbuf is allocated\n+ * @param pkts\n+ * Blank array to keep successfully dequeued packets\n+ * @param count\n+ * Size of the packet array\n+ * @param nr_inflight\n+ * >= 0: The amount of in-flight packets\n+ * -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id\n+ * @param dma_id\n+ * The identifier of DMA device\n+ * @param vchan_id\n+ * The identifier of virtual DMA channel\n+ * @return\n+ * Number of successfully dequeued packets\n+ */\n+__rte_experimental\n+uint16_t\n+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,\n+\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,\n+\tint *nr_inflight, int16_t dma_id, uint16_t vchan_id);\n+\n #ifdef __cplusplus\n }\n #endif\ndiff --git a/lib/vhost/version.map b/lib/vhost/version.map\nindex 5841315386..8c7211bf0d 100644\n--- a/lib/vhost/version.map\n+++ b/lib/vhost/version.map\n@@ -90,7 +90,7 @@ EXPERIMENTAL {\n \n \t# added in 22.07\n \trte_vhost_async_get_inflight_thread_unsafe;\n-\n+\trte_vhost_async_try_dequeue_burst;\n };\n \n INTERNAL {\ndiff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c\nindex 5904839d5c..c6b11bcb6f 100644\n--- a/lib/vhost/virtio_net.c\n+++ b/lib/vhost/virtio_net.c\n@@ -3171,3 +3171,340 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \n \treturn count;\n }\n+\n+static __rte_always_inline uint16_t\n+async_poll_dequeue_completed_split(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\tstruct rte_mbuf **pkts, uint16_t count, int16_t dma_id,\n+\t\tuint16_t vchan_id, bool legacy_ol_flags)\n+{\n+\tuint16_t start_idx, from, i;\n+\tuint16_t nr_cpl_pkts = 0;\n+\tstruct async_inflight_info *pkts_info = vq->async->pkts_info;\n+\n+\tvhost_async_dma_check_completed(dev, dma_id, vchan_id, VHOST_DMA_MAX_COPY_COMPLETE);\n+\n+\tstart_idx = async_get_first_inflight_pkt_idx(vq);\n+\n+\tfrom = start_idx;\n+\twhile (vq->async->pkts_cmpl_flag[from] && count--) {\n+\t\tvq->async->pkts_cmpl_flag[from] = false;\n+\t\tfrom = (from + 1) & (vq->size - 1);\n+\t\tnr_cpl_pkts++;\n+\t}\n+\n+\tif (nr_cpl_pkts == 0)\n+\t\treturn 0;\n+\n+\tfor (i = 0; i < nr_cpl_pkts; i++) {\n+\t\tfrom = (start_idx + i) & (vq->size - 1);\n+\t\tpkts[i] = pkts_info[from].mbuf;\n+\n+\t\tif (virtio_net_with_host_offload(dev))\n+\t\t\tvhost_dequeue_offload(dev, &pkts_info[from].nethdr, pkts[i],\n+\t\t\t\t\t legacy_ol_flags);\n+\t}\n+\n+\t/* write back completed descs to used ring and update used idx */\n+\twrite_back_completed_descs_split(vq, nr_cpl_pkts);\n+\t__atomic_add_fetch(&vq->used->idx, nr_cpl_pkts, __ATOMIC_RELEASE);\n+\tvhost_vring_call_split(dev, vq);\n+\n+\tvq->async->pkts_inflight_n -= nr_cpl_pkts;\n+\n+\treturn nr_cpl_pkts;\n+}\n+\n+static __rte_always_inline uint16_t\n+virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,\n+\t\tint16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags)\n+{\n+\tstatic bool allocerr_warned;\n+\tbool dropped = false;\n+\tuint16_t free_entries;\n+\tuint16_t pkt_idx, slot_idx = 0;\n+\tuint16_t nr_done_pkts = 0;\n+\tuint16_t pkt_err = 0;\n+\tuint16_t n_xfer;\n+\tstruct vhost_async *async = vq->async;\n+\tstruct async_inflight_info *pkts_info = async->pkts_info;\n+\tstruct rte_mbuf *pkts_prealloc[MAX_PKT_BURST];\n+\tuint16_t pkts_size = count;\n+\n+\t/**\n+\t * The ordering between avail index and\n+\t * desc reads needs to be enforced.\n+\t */\n+\tfree_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) -\n+\t\t\tvq->last_avail_idx;\n+\tif (free_entries == 0)\n+\t\tgoto out;\n+\n+\trte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);\n+\n+\tasync_iter_reset(async);\n+\n+\tcount = RTE_MIN(count, MAX_PKT_BURST);\n+\tcount = RTE_MIN(count, free_entries);\n+\tVHOST_LOG_DATA(DEBUG, \"(%s) about to dequeue %u buffers\\n\",\n+\t\t\tdev->ifname, count);\n+\n+\tif (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts_prealloc, count))\n+\t\tgoto out;\n+\n+\tfor (pkt_idx = 0; pkt_idx < count; pkt_idx++) {\n+\t\tuint16_t head_idx = 0;\n+\t\tuint16_t nr_vec = 0;\n+\t\tuint16_t to;\n+\t\tuint32_t buf_len;\n+\t\tint err;\n+\t\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n+\t\tstruct rte_mbuf *pkt = pkts_prealloc[pkt_idx];\n+\n+\t\tif (unlikely(fill_vec_buf_split(dev, vq, vq->last_avail_idx,\n+\t\t\t\t\t\t&nr_vec, buf_vec,\n+\t\t\t\t\t\t&head_idx, &buf_len,\n+\t\t\t\t\t\tVHOST_ACCESS_RO) < 0)) {\n+\t\t\tdropped = true;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\terr = virtio_dev_pktmbuf_prep(dev, pkt, buf_len);\n+\t\tif (unlikely(err)) {\n+\t\t\t/**\n+\t\t\t * mbuf allocation fails for jumbo packets when external\n+\t\t\t * buffer allocation is not allowed and linear buffer\n+\t\t\t * is required. Drop this packet.\n+\t\t\t */\n+\t\t\tif (!allocerr_warned) {\n+\t\t\t\tVHOST_LOG_DATA(ERR,\n+\t\t\t\t\t\"(%s) %s: Failed mbuf alloc of size %d from %s\\n\",\n+\t\t\t\t\tdev->ifname, __func__, buf_len, mbuf_pool->name);\n+\t\t\t\tallocerr_warned = true;\n+\t\t\t}\n+\t\t\tdropped = true;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tslot_idx = (async->pkts_idx + pkt_idx) & (vq->size - 1);\n+\t\terr = desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkt, mbuf_pool,\n+\t\t\t\t\tlegacy_ol_flags, slot_idx, true);\n+\t\tif (unlikely(err)) {\n+\t\t\tif (!allocerr_warned) {\n+\t\t\t\tVHOST_LOG_DATA(ERR,\n+\t\t\t\t\t\"(%s) %s: Failed to offload copies to async channel.\\n\",\n+\t\t\t\t\tdev->ifname, __func__);\n+\t\t\t\tallocerr_warned = true;\n+\t\t\t}\n+\t\t\tdropped = true;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tpkts_info[slot_idx].mbuf = pkt;\n+\n+\t\t/* store used descs */\n+\t\tto = async->desc_idx_split & (vq->size - 1);\n+\t\tasync->descs_split[to].id = head_idx;\n+\t\tasync->descs_split[to].len = 0;\n+\t\tasync->desc_idx_split++;\n+\n+\t\tvq->last_avail_idx++;\n+\t}\n+\n+\tif (unlikely(dropped))\n+\t\trte_pktmbuf_free_bulk(&pkts_prealloc[pkt_idx], count - pkt_idx);\n+\n+\tn_xfer = vhost_async_dma_transfer(dev, vq, dma_id, vchan_id, async->pkts_idx,\n+\t\t\t\t\t async->iov_iter, pkt_idx);\n+\n+\tasync->pkts_inflight_n += n_xfer;\n+\n+\tpkt_err = pkt_idx - n_xfer;\n+\tif (unlikely(pkt_err)) {\n+\t\tVHOST_LOG_DATA(DEBUG, \"(%s) %s: failed to transfer data.\\n\",\n+\t\t\t\tdev->ifname, __func__);\n+\n+\t\tpkt_idx = n_xfer;\n+\t\t/* recover available ring */\n+\t\tvq->last_avail_idx -= pkt_err;\n+\n+\t\t/**\n+\t\t * recover async channel copy related structures and free pktmbufs\n+\t\t * for error pkts.\n+\t\t */\n+\t\tasync->desc_idx_split -= pkt_err;\n+\t\twhile (pkt_err-- > 0) {\n+\t\t\trte_pktmbuf_free(pkts_info[slot_idx & (vq->size - 1)].mbuf);\n+\t\t\tslot_idx--;\n+\t\t}\n+\t}\n+\n+\tasync->pkts_idx += pkt_idx;\n+\tif (async->pkts_idx >= vq->size)\n+\t\tasync->pkts_idx -= vq->size;\n+\n+out:\n+\t/* DMA device may serve other queues, unconditionally check completed. */\n+\tnr_done_pkts = async_poll_dequeue_completed_split(dev, vq, pkts, pkts_size,\n+\t\t\t\t\t\t\t dma_id, vchan_id, legacy_ol_flags);\n+\n+\treturn nr_done_pkts;\n+}\n+\n+__rte_noinline\n+static uint16_t\n+virtio_dev_tx_async_split_legacy(struct virtio_net *dev,\n+\t\tstruct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool,\n+\t\tstruct rte_mbuf **pkts, uint16_t count,\n+\t\tint16_t dma_id, uint16_t vchan_id)\n+{\n+\treturn virtio_dev_tx_async_split(dev, vq, mbuf_pool,\n+\t\t\t\tpkts, count, dma_id, vchan_id, true);\n+}\n+\n+__rte_noinline\n+static uint16_t\n+virtio_dev_tx_async_split_compliant(struct virtio_net *dev,\n+\t\tstruct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool,\n+\t\tstruct rte_mbuf **pkts, uint16_t count,\n+\t\tint16_t dma_id, uint16_t vchan_id)\n+{\n+\treturn virtio_dev_tx_async_split(dev, vq, mbuf_pool,\n+\t\t\t\tpkts, count, dma_id, vchan_id, false);\n+}\n+\n+uint16_t\n+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,\n+\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,\n+\tint *nr_inflight, int16_t dma_id, uint16_t vchan_id)\n+{\n+\tstruct virtio_net *dev;\n+\tstruct rte_mbuf *rarp_mbuf = NULL;\n+\tstruct vhost_virtqueue *vq;\n+\tint16_t success = 1;\n+\n+\tdev = get_device(vid);\n+\tif (!dev || !nr_inflight)\n+\t\treturn 0;\n+\n+\t*nr_inflight = -1;\n+\n+\tif (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%s) %s: built-in vhost net backend is disabled.\\n\",\n+\t\t\t\tdev->ifname, __func__);\n+\t\treturn 0;\n+\t}\n+\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%s) %s: invalid virtqueue idx %d.\\n\",\n+\t\t\t\tdev->ifname, __func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\tif (unlikely(dma_id < 0 || dma_id >= RTE_DMADEV_DEFAULT_MAX)) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%s) %s: invalid dma id %d.\\n\",\n+\t\t\t\tdev->ifname, __func__, dma_id);\n+\t\treturn 0;\n+\t}\n+\n+\tif (unlikely(!dma_copy_track[dma_id].vchans ||\n+\t\t\t\t!dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr)) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%s) %s: invalid channel %d:%u.\\n\", dev->ifname, __func__,\n+\t\t\t\tdma_id, vchan_id);\n+\t\treturn 0;\n+\t}\n+\n+\tvq = dev->virtqueue[queue_id];\n+\n+\tif (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))\n+\t\treturn 0;\n+\n+\tif (unlikely(vq->enabled == 0)) {\n+\t\tcount = 0;\n+\t\tgoto out_access_unlock;\n+\t}\n+\n+\tif (unlikely(!vq->async)) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%s) %s: async not registered for queue id %d.\\n\",\n+\t\t\t\tdev->ifname, __func__, queue_id);\n+\t\tcount = 0;\n+\t\tgoto out_access_unlock;\n+\t}\n+\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_lock(vq);\n+\n+\tif (unlikely(vq->access_ok == 0))\n+\t\tif (unlikely(vring_translate(dev, vq) < 0)) {\n+\t\t\tcount = 0;\n+\t\t\tgoto out;\n+\t\t}\n+\n+\t/*\n+\t * Construct a RARP broadcast packet, and inject it to the \"pkts\"\n+\t * array, to looks like that guest actually send such packet.\n+\t *\n+\t * Check user_send_rarp() for more information.\n+\t *\n+\t * broadcast_rarp shares a cacheline in the virtio_net structure\n+\t * with some fields that are accessed during enqueue and\n+\t * __atomic_compare_exchange_n causes a write if performed compare\n+\t * and exchange. This could result in false sharing between enqueue\n+\t * and dequeue.\n+\t *\n+\t * Prevent unnecessary false sharing by reading broadcast_rarp first\n+\t * and only performing compare and exchange if the read indicates it\n+\t * is likely to be set.\n+\t */\n+\tif (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) &&\n+\t\t\t__atomic_compare_exchange_n(&dev->broadcast_rarp,\n+\t\t\t&success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) {\n+\n+\t\trarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);\n+\t\tif (rarp_mbuf == NULL) {\n+\t\t\tVHOST_LOG_DATA(ERR, \"Failed to make RARP packet.\\n\");\n+\t\t\tcount = 0;\n+\t\t\tgoto out;\n+\t\t}\n+\t\t/*\n+\t\t * Inject it to the head of \"pkts\" array, so that switch's mac\n+\t\t * learning table will get updated first.\n+\t\t */\n+\t\tpkts[0] = rarp_mbuf;\n+\t\tpkts++;\n+\t\tcount -= 1;\n+\t}\n+\n+\tif (unlikely(vq_is_packed(dev))) {\n+\t\tstatic bool not_support_pack_log;\n+\t\tif (!not_support_pack_log) {\n+\t\t\tVHOST_LOG_DATA(ERR,\n+\t\t\t\t\"(%s) %s: async dequeue does not support packed ring.\\n\",\n+\t\t\t\tdev->ifname, __func__);\n+\t\t\tnot_support_pack_log = true;\n+\t\t}\n+\t\tcount = 0;\n+\t\tgoto out;\n+\t}\n+\n+\tif (dev->flags & VIRTIO_DEV_LEGACY_OL_FLAGS)\n+\t\tcount = virtio_dev_tx_async_split_legacy(dev, vq, mbuf_pool, pkts,\n+\t\t\t\t\t\t\t count, dma_id, vchan_id);\n+\telse\n+\t\tcount = virtio_dev_tx_async_split_compliant(dev, vq, mbuf_pool, pkts,\n+\t\t\t\t\t\t\t count, dma_id, vchan_id);\n+\n+\t*nr_inflight = vq->async->pkts_inflight_n;\n+\n+out:\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_unlock(vq);\n+\n+out_access_unlock:\n+\trte_spinlock_unlock(&vq->access_lock);\n+\n+\tif (unlikely(rarp_mbuf != NULL))\n+\t\tcount += 1;\n+\n+\treturn count;\n+}\n", "prefixes": [ "v8", "4/5" ] }