Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 92740,
    "url": "http://patchwork.dpdk.org/api/patches/92740/?format=api",
    "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20210504131458.593429-13-bruce.richardson@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20210504131458.593429-13-bruce.richardson@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20210504131458.593429-13-bruce.richardson@intel.com",
    "date": "2021-05-04T13:14:58",
    "name": "[v5,12/12] raw/ioat: report status of completed jobs",
    "commit_ref": null,
    "pull_url": null,
    "state": "accepted",
    "archived": true,
    "hash": "bfab12aef13cff4b5f40c473919b748143d74948",
    "submitter": {
        "id": 20,
        "url": "http://patchwork.dpdk.org/api/people/20/?format=api",
        "name": "Bruce Richardson",
        "email": "bruce.richardson@intel.com"
    },
    "delegate": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/users/1/?format=api",
        "username": "tmonjalo",
        "first_name": "Thomas",
        "last_name": "Monjalon",
        "email": "thomas@monjalon.net"
    },
    "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20210504131458.593429-13-bruce.richardson@intel.com/mbox/",
    "series": [
        {
            "id": 16807,
            "url": "http://patchwork.dpdk.org/api/series/16807/?format=api",
            "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=16807",
            "date": "2021-05-04T13:14:50",
            "name": "ioat driver updates",
            "version": 5,
            "mbox": "http://patchwork.dpdk.org/series/16807/mbox/"
        }
    ],
    "comments": "http://patchwork.dpdk.org/api/patches/92740/comments/",
    "check": "success",
    "checks": "http://patchwork.dpdk.org/api/patches/92740/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id D683DA0562;\n\tTue,  4 May 2021 15:17:04 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 4FAB141156;\n\tTue,  4 May 2021 15:15:50 +0200 (CEST)",
            "from mga07.intel.com (mga07.intel.com [134.134.136.100])\n by mails.dpdk.org (Postfix) with ESMTP id D8C03410FE\n for <dev@dpdk.org>; Tue,  4 May 2021 15:15:43 +0200 (CEST)",
            "from orsmga002.jf.intel.com ([10.7.209.21])\n by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 04 May 2021 06:15:43 -0700",
            "from silpixa00399126.ir.intel.com ([10.237.223.78])\n by orsmga002.jf.intel.com with ESMTP; 04 May 2021 06:15:41 -0700"
        ],
        "IronPort-SDR": [
            "\n GFz4aWIyBGL9r7KASXIAzJcnkaTwtnbLX7taQKHUDJJipQ3vpPXsSm8K5QErBt4TFn27kU2PYd\n k6SH5T3rJhmQ==",
            "\n tiyHSbY7EvB8o2wf5bPJaSvz3sC/Gkhy7w4Q6KZJM/JO5QD6m1rIZt0AIeuS+cArochp5OxPAZ\n efxxRRHx+QIg=="
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6200,9189,9973\"; a=\"261922883\"",
            "E=Sophos;i=\"5.82,272,1613462400\"; d=\"scan'208\";a=\"261922883\"",
            "E=Sophos;i=\"5.82,272,1613462400\"; d=\"scan'208\";a=\"406105641\""
        ],
        "X-ExtLoop1": "1",
        "From": "Bruce Richardson <bruce.richardson@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "kevin.laatz@intel.com, sunil.pai.g@intel.com, jiayu.hu@intel.com,\n Bruce Richardson <bruce.richardson@intel.com>",
        "Date": "Tue,  4 May 2021 14:14:58 +0100",
        "Message-Id": "<20210504131458.593429-13-bruce.richardson@intel.com>",
        "X-Mailer": "git-send-email 2.30.2",
        "In-Reply-To": "<20210504131458.593429-1-bruce.richardson@intel.com>",
        "References": "<20210318182042.43658-1-bruce.richardson@intel.com>\n <20210504131458.593429-1-bruce.richardson@intel.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "Subject": "[dpdk-dev] [PATCH v5 12/12] raw/ioat: report status of completed\n jobs",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "Add improved error handling to rte_ioat_completed_ops(). This patch adds\nnew parameters to the function to enable the user to track the completion\nstatus of each individual operation in a batch. With this addition, the\nfunction can help the user to determine firstly, how many operations may\nhave failed or been skipped and then secondly, which specific operations\ndid not complete successfully.\n\nSigned-off-by: Kevin Laatz <kevin.laatz@intel.com>\nSigned-off-by: Bruce Richardson <bruce.richardson@intel.com>\n---\n doc/guides/rel_notes/release_21_05.rst |   5 +\n drivers/raw/ioat/ioat_common.c         |   9 +\n drivers/raw/ioat/ioat_rawdev_test.c    | 274 ++++++++++++++++++++++++-\n drivers/raw/ioat/rte_idxd_rawdev_fns.h | 151 ++++++++++----\n drivers/raw/ioat/rte_ioat_rawdev.h     |  53 ++++-\n drivers/raw/ioat/rte_ioat_rawdev_fns.h |  15 +-\n examples/ioat/ioatfwd.c                |  14 +-\n examples/vhost/ioat.c                  |   2 +-\n 8 files changed, 455 insertions(+), 68 deletions(-)",
    "diff": "diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst\nindex cd33898c55..064810a8f9 100644\n--- a/doc/guides/rel_notes/release_21_05.rst\n+++ b/doc/guides/rel_notes/release_21_05.rst\n@@ -333,6 +333,11 @@ API Changes\n   it's not supported on the current platform. Instead ``rte_stack_create()``\n   fails and ``rte_errno`` is set to ``ENOTSUP``.\n \n+* raw/ioat: The experimental function ``rte_ioat_completed_ops()`` now\n+  supports two additional parameters, ``status`` and ``num_unsuccessful``,\n+  to allow the reporting of errors from hardware when performing copy\n+  operations.\n+\n \n ABI Changes\n -----------\ndiff --git a/drivers/raw/ioat/ioat_common.c b/drivers/raw/ioat/ioat_common.c\nindex fcb30572e6..d01c1ee367 100644\n--- a/drivers/raw/ioat/ioat_common.c\n+++ b/drivers/raw/ioat/ioat_common.c\n@@ -162,6 +162,15 @@ idxd_dev_configure(const struct rte_rawdev *dev,\n \t\trte_idxd->desc_ring = NULL;\n \t\treturn -ENOMEM;\n \t}\n+\trte_idxd->hdl_ring_flags = rte_zmalloc(NULL,\n+\t\t\tsizeof(*rte_idxd->hdl_ring_flags) * max_desc, 0);\n+\tif (rte_idxd->hdl_ring_flags == NULL) {\n+\t\trte_free(rte_idxd->desc_ring);\n+\t\trte_free(rte_idxd->hdl_ring);\n+\t\trte_idxd->desc_ring = NULL;\n+\t\trte_idxd->hdl_ring = NULL;\n+\t\treturn -ENOMEM;\n+\t}\n \trte_idxd->hdls_read = rte_idxd->batch_start = 0;\n \trte_idxd->batch_size = 0;\n \ndiff --git a/drivers/raw/ioat/ioat_rawdev_test.c b/drivers/raw/ioat/ioat_rawdev_test.c\nindex 839a716a21..5e33669699 100644\n--- a/drivers/raw/ioat/ioat_rawdev_test.c\n+++ b/drivers/raw/ioat/ioat_rawdev_test.c\n@@ -73,13 +73,15 @@ do_multi_copies(int dev_id, int split_batches, int split_completions)\n \tif (split_completions) {\n \t\t/* gather completions in two halves */\n \t\tuint16_t half_len = RTE_DIM(srcs) / 2;\n-\t\tif (rte_ioat_completed_ops(dev_id, half_len, (void *)completed_src,\n+\t\tif (rte_ioat_completed_ops(dev_id, half_len, NULL, NULL,\n+\t\t\t\t(void *)completed_src,\n \t\t\t\t(void *)completed_dst) != half_len) {\n \t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops - first half request\\n\");\n \t\t\trte_rawdev_dump(dev_id, stdout);\n \t\t\treturn -1;\n \t\t}\n-\t\tif (rte_ioat_completed_ops(dev_id, half_len, (void *)&completed_src[half_len],\n+\t\tif (rte_ioat_completed_ops(dev_id, half_len, NULL, NULL,\n+\t\t\t\t(void *)&completed_src[half_len],\n \t\t\t\t(void *)&completed_dst[half_len]) != half_len) {\n \t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops - second half request\\n\");\n \t\t\trte_rawdev_dump(dev_id, stdout);\n@@ -87,7 +89,8 @@ do_multi_copies(int dev_id, int split_batches, int split_completions)\n \t\t}\n \t} else {\n \t\t/* gather all completions in one go */\n-\t\tif (rte_ioat_completed_ops(dev_id, 64, (void *)completed_src,\n+\t\tif (rte_ioat_completed_ops(dev_id, RTE_DIM(completed_src), NULL, NULL,\n+\t\t\t\t(void *)completed_src,\n \t\t\t\t(void *)completed_dst) != RTE_DIM(srcs)) {\n \t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n \t\t\trte_rawdev_dump(dev_id, stdout);\n@@ -151,7 +154,7 @@ test_enqueue_copies(int dev_id)\n \t\trte_ioat_perform_ops(dev_id);\n \t\tusleep(10);\n \n-\t\tif (rte_ioat_completed_ops(dev_id, 1, (void *)&completed[0],\n+\t\tif (rte_ioat_completed_ops(dev_id, 1, NULL, NULL, (void *)&completed[0],\n \t\t\t\t(void *)&completed[1]) != 1) {\n \t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n \t\t\treturn -1;\n@@ -170,6 +173,13 @@ test_enqueue_copies(int dev_id)\n \t\t\t}\n \t\trte_pktmbuf_free(src);\n \t\trte_pktmbuf_free(dst);\n+\n+\t\t/* check ring is now empty */\n+\t\tif (rte_ioat_completed_ops(dev_id, 1, NULL, NULL, (void *)&completed[0],\n+\t\t\t\t(void *)&completed[1]) != 0) {\n+\t\t\tPRINT_ERR(\"Error: got unexpected returned handles from rte_ioat_completed_ops\\n\");\n+\t\t\treturn -1;\n+\t\t}\n \t} while (0);\n \n \t/* test doing a multiple single copies */\n@@ -203,7 +213,8 @@ test_enqueue_copies(int dev_id)\n \t\t}\n \t\tusleep(10);\n \n-\t\tif (rte_ioat_completed_ops(dev_id, max_completions, (void *)&completed[0],\n+\t\tif (rte_ioat_completed_ops(dev_id, max_completions, NULL, NULL,\n+\t\t\t\t(void *)&completed[0],\n \t\t\t\t(void *)&completed[max_completions]) != max_ops) {\n \t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n \t\t\trte_rawdev_dump(dev_id, stdout);\n@@ -256,7 +267,7 @@ test_enqueue_fill(int dev_id)\n \t\trte_ioat_perform_ops(dev_id);\n \t\tusleep(100);\n \n-\t\tif (rte_ioat_completed_ops(dev_id, 1, (void *)&completed[0],\n+\t\tif (rte_ioat_completed_ops(dev_id, 1, NULL, NULL, (void *)&completed[0],\n \t\t\t(void *)&completed[1]) != 1) {\n \t\t\tPRINT_ERR(\"Error with completed ops\\n\");\n \t\t\treturn -1;\n@@ -266,8 +277,7 @@ test_enqueue_fill(int dev_id)\n \t\t\tchar pat_byte = ((char *)&pattern)[j % 8];\n \t\t\tif (dst_data[j] != pat_byte) {\n \t\t\t\tPRINT_ERR(\"Error with fill operation (lengths = %u): got (%x), not (%x)\\n\",\n-\t\t\t\t\t\tlengths[i], dst_data[j],\n-\t\t\t\t\t\tpat_byte);\n+\t\t\t\t\t\tlengths[i], dst_data[j], pat_byte);\n \t\t\t\treturn -1;\n \t\t\t}\n \t\t}\n@@ -323,6 +333,7 @@ test_burst_capacity(int dev_id)\n \t\tusleep(100);\n \t\tfor (i = 0; i < ring_space / (2 * BURST_SIZE); i++) {\n \t\t\tif (rte_ioat_completed_ops(dev_id, BURST_SIZE,\n+\t\t\t\t\tNULL, NULL,\n \t\t\t\t\tcompletions, completions) != BURST_SIZE) {\n \t\t\t\tPRINT_ERR(\"Error with completions\\n\");\n \t\t\t\treturn -1;\n@@ -341,10 +352,248 @@ test_burst_capacity(int dev_id)\n \treturn 0;\n }\n \n+static int\n+test_completion_status(int dev_id)\n+{\n+#define COMP_BURST_SZ\t16\n+\tconst unsigned int fail_copy[] = {0, 7, 15};\n+\tstruct rte_mbuf *srcs[COMP_BURST_SZ], *dsts[COMP_BURST_SZ];\n+\tstruct rte_mbuf *completed_src[COMP_BURST_SZ * 2];\n+\tstruct rte_mbuf *completed_dst[COMP_BURST_SZ * 2];\n+\tunsigned int length = 1024;\n+\tunsigned int i;\n+\tuint8_t not_ok = 0;\n+\n+\t/* Test single full batch statuses */\n+\tfor (i = 0; i < RTE_DIM(fail_copy); i++) {\n+\t\tuint32_t status[COMP_BURST_SZ] = {0};\n+\t\tunsigned int j;\n+\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\tsrcs[j] = rte_pktmbuf_alloc(pool);\n+\t\t\tdsts[j] = rte_pktmbuf_alloc(pool);\n+\n+\t\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\t\t(j == fail_copy[i] ? (phys_addr_t)NULL :\n+\t\t\t\t\t\t\t(srcs[j]->buf_iova + srcs[j]->data_off)),\n+\t\t\t\t\tdsts[j]->buf_iova + dsts[j]->data_off,\n+\t\t\t\t\tlength,\n+\t\t\t\t\t(uintptr_t)srcs[j],\n+\t\t\t\t\t(uintptr_t)dsts[j]) != 1) {\n+\t\t\t\tPRINT_ERR(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\", j);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t}\n+\t\trte_ioat_perform_ops(dev_id);\n+\t\tusleep(100);\n+\n+\t\tif (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ, status, &not_ok,\n+\t\t\t\t(void *)completed_src, (void *)completed_dst) != COMP_BURST_SZ) {\n+\t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (not_ok != 1 || status[fail_copy[i]] == RTE_IOAT_OP_SUCCESS) {\n+\t\t\tunsigned int j;\n+\t\t\tPRINT_ERR(\"Error, missing expected failed copy, %u\\n\", fail_copy[i]);\n+\t\t\tfor (j = 0; j < COMP_BURST_SZ; j++)\n+\t\t\t\tprintf(\"%u \", status[j]);\n+\t\t\tprintf(\"<-- Statuses\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\trte_pktmbuf_free(completed_src[j]);\n+\t\t\trte_pktmbuf_free(completed_dst[j]);\n+\t\t}\n+\t}\n+\n+\t/* Test gathering status for two batches at once */\n+\tfor (i = 0; i < RTE_DIM(fail_copy); i++) {\n+\t\tuint32_t status[COMP_BURST_SZ] = {0};\n+\t\tunsigned int batch, j;\n+\t\tunsigned int expected_failures = 0;\n+\n+\t\tfor (batch = 0; batch < 2; batch++) {\n+\t\t\tfor (j = 0; j < COMP_BURST_SZ/2; j++) {\n+\t\t\t\tsrcs[j] = rte_pktmbuf_alloc(pool);\n+\t\t\t\tdsts[j] = rte_pktmbuf_alloc(pool);\n+\n+\t\t\t\tif (j == fail_copy[i])\n+\t\t\t\t\texpected_failures++;\n+\t\t\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\t\t\t(j == fail_copy[i] ? (phys_addr_t)NULL :\n+\t\t\t\t\t\t\t(srcs[j]->buf_iova + srcs[j]->data_off)),\n+\t\t\t\t\t\tdsts[j]->buf_iova + dsts[j]->data_off,\n+\t\t\t\t\t\tlength,\n+\t\t\t\t\t\t(uintptr_t)srcs[j],\n+\t\t\t\t\t\t(uintptr_t)dsts[j]) != 1) {\n+\t\t\t\t\tPRINT_ERR(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\",\n+\t\t\t\t\t\t\tj);\n+\t\t\t\t\treturn -1;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\trte_ioat_perform_ops(dev_id);\n+\t\t}\n+\t\tusleep(100);\n+\n+\t\tif (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ, status, &not_ok,\n+\t\t\t\t(void *)completed_src, (void *)completed_dst) != COMP_BURST_SZ) {\n+\t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (not_ok != expected_failures) {\n+\t\t\tunsigned int j;\n+\t\t\tPRINT_ERR(\"Error, missing expected failed copy, got %u, not %u\\n\",\n+\t\t\t\t\tnot_ok, expected_failures);\n+\t\t\tfor (j = 0; j < COMP_BURST_SZ; j++)\n+\t\t\t\tprintf(\"%u \", status[j]);\n+\t\t\tprintf(\"<-- Statuses\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\trte_pktmbuf_free(completed_src[j]);\n+\t\t\trte_pktmbuf_free(completed_dst[j]);\n+\t\t}\n+\t}\n+\n+\t/* Test gathering status for half batch at a time */\n+\tfor (i = 0; i < RTE_DIM(fail_copy); i++) {\n+\t\tuint32_t status[COMP_BURST_SZ] = {0};\n+\t\tunsigned int j;\n+\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\tsrcs[j] = rte_pktmbuf_alloc(pool);\n+\t\t\tdsts[j] = rte_pktmbuf_alloc(pool);\n+\n+\t\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\t\t(j == fail_copy[i] ? (phys_addr_t)NULL :\n+\t\t\t\t\t\t\t(srcs[j]->buf_iova + srcs[j]->data_off)),\n+\t\t\t\t\tdsts[j]->buf_iova + dsts[j]->data_off,\n+\t\t\t\t\tlength,\n+\t\t\t\t\t(uintptr_t)srcs[j],\n+\t\t\t\t\t(uintptr_t)dsts[j]) != 1) {\n+\t\t\t\tPRINT_ERR(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\", j);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t}\n+\t\trte_ioat_perform_ops(dev_id);\n+\t\tusleep(100);\n+\n+\t\tif (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ / 2, status, &not_ok,\n+\t\t\t\t(void *)completed_src,\n+\t\t\t\t(void *)completed_dst) != (COMP_BURST_SZ / 2)) {\n+\t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (fail_copy[i] < COMP_BURST_SZ / 2 &&\n+\t\t\t\t(not_ok != 1 || status[fail_copy[i]] == RTE_IOAT_OP_SUCCESS)) {\n+\t\t\tPRINT_ERR(\"Missing expected failure in first half-batch\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ / 2, status, &not_ok,\n+\t\t\t\t(void *)&completed_src[COMP_BURST_SZ / 2],\n+\t\t\t\t(void *)&completed_dst[COMP_BURST_SZ / 2]) != (COMP_BURST_SZ / 2)) {\n+\t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (fail_copy[i] >= COMP_BURST_SZ / 2 && (not_ok != 1 ||\n+\t\t\t\tstatus[fail_copy[i] - (COMP_BURST_SZ / 2)]\n+\t\t\t\t\t== RTE_IOAT_OP_SUCCESS)) {\n+\t\t\tPRINT_ERR(\"Missing expected failure in second half-batch\\n\");\n+\t\t\trte_rawdev_dump(dev_id, stdout);\n+\t\t\treturn -1;\n+\t\t}\n+\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\trte_pktmbuf_free(completed_src[j]);\n+\t\t\trte_pktmbuf_free(completed_dst[j]);\n+\t\t}\n+\t}\n+\n+\t/* Test gathering statuses with fence */\n+\tfor (i = 1; i < RTE_DIM(fail_copy); i++) {\n+\t\tuint32_t status[COMP_BURST_SZ * 2] = {0};\n+\t\tunsigned int j;\n+\t\tuint16_t count;\n+\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\tsrcs[j] = rte_pktmbuf_alloc(pool);\n+\t\t\tdsts[j] = rte_pktmbuf_alloc(pool);\n+\n+\t\t\t/* always fail the first copy */\n+\t\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\t\t(j == 0 ? (phys_addr_t)NULL :\n+\t\t\t\t\t\t(srcs[j]->buf_iova + srcs[j]->data_off)),\n+\t\t\t\t\tdsts[j]->buf_iova + dsts[j]->data_off,\n+\t\t\t\t\tlength,\n+\t\t\t\t\t(uintptr_t)srcs[j],\n+\t\t\t\t\t(uintptr_t)dsts[j]) != 1) {\n+\t\t\t\tPRINT_ERR(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\", j);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t\t/* put in a fence which will stop any further transactions\n+\t\t\t * because we had a previous failure.\n+\t\t\t */\n+\t\t\tif (j == fail_copy[i])\n+\t\t\t\trte_ioat_fence(dev_id);\n+\t\t}\n+\t\trte_ioat_perform_ops(dev_id);\n+\t\tusleep(100);\n+\n+\t\tcount = rte_ioat_completed_ops(dev_id, COMP_BURST_SZ * 2, status, &not_ok,\n+\t\t\t\t(void *)completed_src, (void *)completed_dst);\n+\t\tif (count != COMP_BURST_SZ) {\n+\t\t\tPRINT_ERR(\"Error with rte_ioat_completed_ops, got %u not %u\\n\",\n+\t\t\t\t\tcount, COMP_BURST_SZ);\n+\t\t\tfor (j = 0; j < count; j++)\n+\t\t\t\tprintf(\"%u \", status[j]);\n+\t\t\tprintf(\"<-- Statuses\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (not_ok != COMP_BURST_SZ - fail_copy[i]) {\n+\t\t\tPRINT_ERR(\"Unexpected failed copy count, got %u, expected %u\\n\",\n+\t\t\t\t\tnot_ok, COMP_BURST_SZ - fail_copy[i]);\n+\t\t\tfor (j = 0; j < COMP_BURST_SZ; j++)\n+\t\t\t\tprintf(\"%u \", status[j]);\n+\t\t\tprintf(\"<-- Statuses\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (status[0] == RTE_IOAT_OP_SUCCESS || status[0] == RTE_IOAT_OP_SKIPPED) {\n+\t\t\tPRINT_ERR(\"Error, op 0 unexpectedly did not fail.\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tfor (j = 1; j <= fail_copy[i]; j++) {\n+\t\t\tif (status[j] != RTE_IOAT_OP_SUCCESS) {\n+\t\t\t\tPRINT_ERR(\"Error, op %u unexpectedly failed\\n\", j);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t}\n+\t\tfor (j = fail_copy[i] + 1; j < COMP_BURST_SZ; j++) {\n+\t\t\tif (status[j] != RTE_IOAT_OP_SKIPPED) {\n+\t\t\t\tPRINT_ERR(\"Error, all descriptors after fence should be invalid\\n\");\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t}\n+\t\tfor (j = 0; j < COMP_BURST_SZ; j++) {\n+\t\t\trte_pktmbuf_free(completed_src[j]);\n+\t\t\trte_pktmbuf_free(completed_dst[j]);\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n int\n ioat_rawdev_test(uint16_t dev_id)\n {\n #define IOAT_TEST_RINGSIZE 512\n+\tconst struct rte_idxd_rawdev *idxd =\n+\t\t\t(struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;\n+\tconst enum rte_ioat_dev_type ioat_type = idxd->type;\n \tstruct rte_ioat_rawdev_config p = { .ring_size = -1 };\n \tstruct rte_rawdev_info info = { .dev_private = &p };\n \tstruct rte_rawdev_xstats_name *snames = NULL;\n@@ -453,6 +702,15 @@ ioat_rawdev_test(uint16_t dev_id)\n \tif (test_burst_capacity(dev_id) != 0)\n \t\tgoto err;\n \n+\t/* only DSA devices report address errors, and we can only use null pointers\n+\t * to generate those errors when DPDK is in VA mode.\n+\t */\n+\tif (rte_eal_iova_mode() == RTE_IOVA_VA && ioat_type == RTE_IDXD_DEV) {\n+\t\tprintf(\"Running Completions Status Test\\n\");\n+\t\tif (test_completion_status(dev_id) != 0)\n+\t\t\tgoto err;\n+\t}\n+\n \trte_rawdev_stop(dev_id);\n \tif (rte_rawdev_xstats_reset(dev_id, NULL, 0) != 0) {\n \t\tPRINT_ERR(\"Error resetting xstat values\\n\");\ndiff --git a/drivers/raw/ioat/rte_idxd_rawdev_fns.h b/drivers/raw/ioat/rte_idxd_rawdev_fns.h\nindex 0bd9cfbd0d..862e0eb41d 100644\n--- a/drivers/raw/ioat/rte_idxd_rawdev_fns.h\n+++ b/drivers/raw/ioat/rte_idxd_rawdev_fns.h\n@@ -115,8 +115,17 @@ struct rte_idxd_rawdev {\n \n \tstruct rte_idxd_hw_desc *desc_ring;\n \tstruct rte_idxd_user_hdl *hdl_ring;\n+\t/* flags to indicate handle validity. Kept separate from ring, to avoid\n+\t * using 8 bytes per flag. Upper 8 bits holds error code if any.\n+\t */\n+\tuint16_t *hdl_ring_flags;\n };\n \n+#define RTE_IDXD_HDL_NORMAL     0\n+#define RTE_IDXD_HDL_INVALID    (1 << 0) /* no handle stored for this element */\n+#define RTE_IDXD_HDL_OP_FAILED  (1 << 1) /* return failure for this one */\n+#define RTE_IDXD_HDL_OP_SKIPPED (1 << 2) /* this op was skipped */\n+\n static __rte_always_inline uint16_t\n __idxd_burst_capacity(int dev_id)\n {\n@@ -135,8 +144,10 @@ __idxd_burst_capacity(int dev_id)\n \t\twrite_idx += idxd->desc_ring_mask + 1;\n \tused_space = write_idx - idxd->hdls_read;\n \n-\t/* Return amount of free space in the descriptor ring */\n-\treturn idxd->desc_ring_mask - used_space;\n+\t/* Return amount of free space in the descriptor ring\n+\t * subtract 1 for space for batch descriptor and 1 for possible null desc\n+\t */\n+\treturn idxd->desc_ring_mask - used_space - 2;\n }\n \n static __rte_always_inline rte_iova_t\n@@ -156,23 +167,28 @@ __idxd_write_desc(int dev_id,\n \tstruct rte_idxd_rawdev *idxd =\n \t\t\t(struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;\n \tuint16_t write_idx = idxd->batch_start + idxd->batch_size;\n+\tuint16_t mask = idxd->desc_ring_mask;\n \n \t/* first check batch ring space then desc ring space */\n \tif ((idxd->batch_idx_read == 0 && idxd->batch_idx_write == idxd->max_batches) ||\n \t\t\tidxd->batch_idx_write + 1 == idxd->batch_idx_read)\n \t\tgoto failed;\n-\tif (((write_idx + 1) & idxd->desc_ring_mask) == idxd->hdls_read)\n+\t/* for descriptor ring, we always need a slot for batch completion */\n+\tif (((write_idx + 2) & mask) == idxd->hdls_read)\n \t\tgoto failed;\n \n \t/* write desc and handle. Note, descriptors don't wrap */\n \tidxd->desc_ring[write_idx].pasid = 0;\n \tidxd->desc_ring[write_idx].op_flags = op_flags | IDXD_FLAG_COMPLETION_ADDR_VALID;\n-\tidxd->desc_ring[write_idx].completion = __desc_idx_to_iova(idxd, write_idx);\n+\tidxd->desc_ring[write_idx].completion = __desc_idx_to_iova(idxd, write_idx & mask);\n \tidxd->desc_ring[write_idx].src = src;\n \tidxd->desc_ring[write_idx].dst = dst;\n \tidxd->desc_ring[write_idx].size = size;\n \n-\tidxd->hdl_ring[write_idx & idxd->desc_ring_mask] = *hdl;\n+\tif (hdl == NULL)\n+\t\tidxd->hdl_ring_flags[write_idx & mask] = RTE_IDXD_HDL_INVALID;\n+\telse\n+\t\tidxd->hdl_ring[write_idx & mask] = *hdl;\n \tidxd->batch_size++;\n \n \tidxd->xstats.enqueued++;\n@@ -214,9 +230,8 @@ __idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst,\n static __rte_always_inline int\n __idxd_fence(int dev_id)\n {\n-\tstatic const struct rte_idxd_user_hdl null_hdl;\n \t/* only op field needs filling - zero src, dst and length */\n-\treturn __idxd_write_desc(dev_id, IDXD_FLAG_FENCE, 0, 0, 0, &null_hdl);\n+\treturn __idxd_write_desc(dev_id, IDXD_FLAG_FENCE, 0, 0, 0, NULL);\n }\n \n static __rte_always_inline void\n@@ -233,42 +248,37 @@ __idxd_perform_ops(int dev_id)\n {\n \tstruct rte_idxd_rawdev *idxd =\n \t\t\t(struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;\n-\t/* write completion to last desc in the batch */\n-\tuint16_t comp_idx = idxd->batch_start + idxd->batch_size - 1;\n-\tif (comp_idx > idxd->desc_ring_mask) {\n-\t\tcomp_idx &= idxd->desc_ring_mask;\n-\t\t*((uint64_t *)&idxd->desc_ring[comp_idx]) = 0; /* zero start of desc */\n-\t}\n+\n+\tif (!idxd->cfg.no_prefetch_completions)\n+\t\trte_prefetch1(&idxd->desc_ring[idxd->batch_idx_ring[idxd->batch_idx_read]]);\n \n \tif (idxd->batch_size == 0)\n \t\treturn 0;\n \n-\t_mm_sfence(); /* fence before writing desc to device */\n-\tif (idxd->batch_size > 1) {\n-\t\tstruct rte_idxd_hw_desc batch_desc = {\n-\t\t\t\t.op_flags = (idxd_op_batch << IDXD_CMD_OP_SHIFT) |\n-\t\t\t\t\tIDXD_FLAG_COMPLETION_ADDR_VALID |\n-\t\t\t\t\tIDXD_FLAG_REQUEST_COMPLETION,\n-\t\t\t\t.desc_addr = __desc_idx_to_iova(idxd, idxd->batch_start),\n-\t\t\t\t.completion = __desc_idx_to_iova(idxd, comp_idx),\n-\t\t\t\t.size = idxd->batch_size,\n-\t\t};\n-\n-\t\t__idxd_movdir64b(idxd->portal, &batch_desc);\n-\t} else {\n-\t\t/* special case batch size of 1, as not allowed by HW */\n-\t\t/* comp_idx == batch_start */\n-\t\tstruct rte_idxd_hw_desc *desc = &idxd->desc_ring[comp_idx];\n-\t\tdesc->op_flags |= IDXD_FLAG_COMPLETION_ADDR_VALID |\n-\t\t\t\tIDXD_FLAG_REQUEST_COMPLETION;\n-\t\tdesc->completion = __desc_idx_to_iova(idxd, comp_idx);\n-\n-\t\t__idxd_movdir64b(idxd->portal, desc);\n-\t}\n+\tif (idxd->batch_size == 1)\n+\t\t/* use a fence as a null descriptor, so batch_size >= 2 */\n+\t\tif (__idxd_fence(dev_id) != 1)\n+\t\t\treturn -1;\n+\n+\t/* write completion beyond last desc in the batch */\n+\tuint16_t comp_idx = (idxd->batch_start + idxd->batch_size) & idxd->desc_ring_mask;\n+\t*((uint64_t *)&idxd->desc_ring[comp_idx]) = 0; /* zero start of desc */\n+\tidxd->hdl_ring_flags[comp_idx] = RTE_IDXD_HDL_INVALID;\n+\n+\tconst struct rte_idxd_hw_desc batch_desc = {\n+\t\t\t.op_flags = (idxd_op_batch << IDXD_CMD_OP_SHIFT) |\n+\t\t\t\tIDXD_FLAG_COMPLETION_ADDR_VALID |\n+\t\t\t\tIDXD_FLAG_REQUEST_COMPLETION,\n+\t\t\t.desc_addr = __desc_idx_to_iova(idxd, idxd->batch_start),\n+\t\t\t.completion = __desc_idx_to_iova(idxd, comp_idx),\n+\t\t\t.size = idxd->batch_size,\n+\t};\n \n+\t_mm_sfence(); /* fence before writing desc to device */\n+\t__idxd_movdir64b(idxd->portal, &batch_desc);\n \tidxd->xstats.started += idxd->batch_size;\n \n-\tidxd->batch_start += idxd->batch_size;\n+\tidxd->batch_start += idxd->batch_size + 1;\n \tidxd->batch_start &= idxd->desc_ring_mask;\n \tidxd->batch_size = 0;\n \n@@ -280,7 +290,7 @@ __idxd_perform_ops(int dev_id)\n }\n \n static __rte_always_inline int\n-__idxd_completed_ops(int dev_id, uint8_t max_ops,\n+__idxd_completed_ops(int dev_id, uint8_t max_ops, uint32_t *status, uint8_t *num_unsuccessful,\n \t\tuintptr_t *src_hdls, uintptr_t *dst_hdls)\n {\n \tstruct rte_idxd_rawdev *idxd =\n@@ -291,8 +301,37 @@ __idxd_completed_ops(int dev_id, uint8_t max_ops,\n \t\tuint16_t idx_to_chk = idxd->batch_idx_ring[idxd->batch_idx_read];\n \t\tvolatile struct rte_idxd_completion *comp_to_chk =\n \t\t\t\t(struct rte_idxd_completion *)&idxd->desc_ring[idx_to_chk];\n-\t\tif (comp_to_chk->status == 0)\n+\t\tuint8_t status = comp_to_chk->status;\n+\t\tif (status == 0)\n \t\t\tbreak;\n+\t\tcomp_to_chk->status = 0;\n+\t\tif (unlikely(status > 1)) {\n+\t\t\t/* error occurred somewhere in batch, start where last checked */\n+\t\t\tuint16_t desc_count = comp_to_chk->completed_size;\n+\t\t\tuint16_t batch_start = idxd->hdls_avail;\n+\t\t\tuint16_t batch_end = idx_to_chk;\n+\n+\t\t\tif (batch_start > batch_end)\n+\t\t\t\tbatch_end += idxd->desc_ring_mask + 1;\n+\t\t\t/* go through each batch entry and see status */\n+\t\t\tfor (n = 0; n < desc_count; n++) {\n+\t\t\t\tuint16_t idx = (batch_start + n) & idxd->desc_ring_mask;\n+\t\t\t\tvolatile struct rte_idxd_completion *comp =\n+\t\t\t\t\t(struct rte_idxd_completion *)&idxd->desc_ring[idx];\n+\t\t\t\tif (comp->status != 0 &&\n+\t\t\t\t\t\tidxd->hdl_ring_flags[idx] == RTE_IDXD_HDL_NORMAL) {\n+\t\t\t\t\tidxd->hdl_ring_flags[idx] = RTE_IDXD_HDL_OP_FAILED;\n+\t\t\t\t\tidxd->hdl_ring_flags[idx] |= (comp->status << 8);\n+\t\t\t\t\tcomp->status = 0; /* clear error for next time */\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\t/* if batch is incomplete, mark rest as skipped */\n+\t\t\tfor ( ; n < batch_end - batch_start; n++) {\n+\t\t\t\tuint16_t idx = (batch_start + n) & idxd->desc_ring_mask;\n+\t\t\t\tif (idxd->hdl_ring_flags[idx] == RTE_IDXD_HDL_NORMAL)\n+\t\t\t\t\tidxd->hdl_ring_flags[idx] = RTE_IDXD_HDL_OP_SKIPPED;\n+\t\t\t}\n+\t\t}\n \t\t/* avail points to one after the last one written */\n \t\tidxd->hdls_avail = (idx_to_chk + 1) & idxd->desc_ring_mask;\n \t\tidxd->batch_idx_read++;\n@@ -300,7 +339,7 @@ __idxd_completed_ops(int dev_id, uint8_t max_ops,\n \t\t\tidxd->batch_idx_read = 0;\n \t}\n \n-\tif (idxd->cfg.hdls_disable) {\n+\tif (idxd->cfg.hdls_disable && status == NULL) {\n \t\tn = (idxd->hdls_avail < idxd->hdls_read) ?\n \t\t\t\t(idxd->hdls_avail + idxd->desc_ring_mask + 1 - idxd->hdls_read) :\n \t\t\t\t(idxd->hdls_avail - idxd->hdls_read);\n@@ -308,10 +347,36 @@ __idxd_completed_ops(int dev_id, uint8_t max_ops,\n \t\tgoto out;\n \t}\n \n-\tfor (n = 0, h_idx = idxd->hdls_read;\n-\t\t\tn < max_ops && h_idx != idxd->hdls_avail; n++) {\n-\t\tsrc_hdls[n] = idxd->hdl_ring[h_idx].src;\n-\t\tdst_hdls[n] = idxd->hdl_ring[h_idx].dst;\n+\tn = 0;\n+\th_idx = idxd->hdls_read;\n+\twhile (h_idx != idxd->hdls_avail) {\n+\t\tuint16_t flag = idxd->hdl_ring_flags[h_idx];\n+\t\tif (flag != RTE_IDXD_HDL_INVALID) {\n+\t\t\tif (!idxd->cfg.hdls_disable) {\n+\t\t\t\tsrc_hdls[n] = idxd->hdl_ring[h_idx].src;\n+\t\t\t\tdst_hdls[n] = idxd->hdl_ring[h_idx].dst;\n+\t\t\t}\n+\t\t\tif (unlikely(flag != RTE_IDXD_HDL_NORMAL)) {\n+\t\t\t\tif (status != NULL)\n+\t\t\t\t\tstatus[n] = flag == RTE_IDXD_HDL_OP_SKIPPED ?\n+\t\t\t\t\t\t\tRTE_IOAT_OP_SKIPPED :\n+\t\t\t\t\t\t\t/* failure case, return err code */\n+\t\t\t\t\t\t\tidxd->hdl_ring_flags[h_idx] >> 8;\n+\t\t\t\tif (num_unsuccessful != NULL)\n+\t\t\t\t\t*num_unsuccessful += 1;\n+\t\t\t}\n+\t\t\tn++;\n+\t\t}\n+\t\tidxd->hdl_ring_flags[h_idx] = RTE_IDXD_HDL_NORMAL;\n+\t\tif (++h_idx > idxd->desc_ring_mask)\n+\t\t\th_idx = 0;\n+\t\tif (n >= max_ops)\n+\t\t\tbreak;\n+\t}\n+\n+\t/* skip over any remaining blank elements, e.g. batch completion */\n+\twhile (idxd->hdl_ring_flags[h_idx] == RTE_IDXD_HDL_INVALID && h_idx != idxd->hdls_avail) {\n+\t\tidxd->hdl_ring_flags[h_idx] = RTE_IDXD_HDL_NORMAL;\n \t\tif (++h_idx > idxd->desc_ring_mask)\n \t\t\th_idx = 0;\n \t}\ndiff --git a/drivers/raw/ioat/rte_ioat_rawdev.h b/drivers/raw/ioat/rte_ioat_rawdev.h\nindex e5a22a0799..6cc1560a64 100644\n--- a/drivers/raw/ioat/rte_ioat_rawdev.h\n+++ b/drivers/raw/ioat/rte_ioat_rawdev.h\n@@ -35,6 +35,10 @@ extern \"C\" {\n struct rte_ioat_rawdev_config {\n \tunsigned short ring_size; /**< size of job submission descriptor ring */\n \tbool hdls_disable;    /**< if set, ignore user-supplied handle params */\n+\t/** set \"no_prefetch_completions\", if polling completions on separate core\n+\t * from the core submitting the jobs\n+\t */\n+\tbool no_prefetch_completions;\n };\n \n /**\n@@ -131,40 +135,73 @@ static inline int\n __rte_experimental\n rte_ioat_perform_ops(int dev_id);\n \n+/*\n+ *  Status codes for operations.\n+ */\n+#define RTE_IOAT_OP_SUCCESS 0  /**< Operation completed successfully */\n+#define RTE_IOAT_OP_SKIPPED 1  /**< Operation was not attempted (Earlier fenced op failed) */\n+/* Values >1 indicate a failure condition */\n+/* Error codes taken from Intel(R) Data Streaming Accelerator Architecture\n+ * Specification, section 5.7\n+ */\n+#define RTE_IOAT_OP_ADDRESS_ERR 0x03  /**< Page fault or invalid address */\n+#define RTE_IOAT_OP_INVALID_LEN 0x13  /**< Invalid/too big length field passed */\n+#define RTE_IOAT_OP_OVERLAPPING_BUFS 0x16 /**< Overlapping buffers error */\n+\n+\n /**\n  * Returns details of operations that have been completed\n  *\n+ * The status of each operation is returned in the status array parameter.\n  * If the hdls_disable option was not set when the device was configured,\n  * the function will return to the caller the user-provided \"handles\" for\n  * the copy operations which have been completed by the hardware, and not\n  * already returned by a previous call to this API.\n  * If the hdls_disable option for the device was set on configure, the\n- * max_copies, src_hdls and dst_hdls parameters will be ignored, and the\n+ * src_hdls and dst_hdls parameters will be ignored, and the\n  * function returns the number of newly-completed operations.\n+ * If status is also NULL, then max_copies parameter is also ignored and the\n+ * function returns a count of the number of newly-completed operations.\n  *\n  * @param dev_id\n  *   The rawdev device id of the ioat instance\n  * @param max_copies\n- *   The number of entries which can fit in the src_hdls and dst_hdls\n+ *   The number of entries which can fit in the status, src_hdls and dst_hdls\n  *   arrays, i.e. max number of completed operations to report.\n  *   NOTE: If hdls_disable configuration option for the device is set, this\n- *   parameter is ignored.\n+ *   parameter applies only to the \"status\" array if specified\n+ * @param status\n+ *   Array to hold the status of each completed operation. Array should be\n+ *   set to zeros on input, as the driver will only write error status values.\n+ *   A value of 1 implies an operation was not attempted, and any other non-zero\n+ *   value indicates operation failure.\n+ *   Parameter may be NULL if no status value checking is required.\n+ * @param num_unsuccessful\n+ *   Returns the number of elements in status where the value is non-zero,\n+ *   i.e. the operation either failed or was not attempted due to an earlier\n+ *   failure. If this value is returned as zero (the expected case), the\n+ *   status array will not have been modified by the function and need not be\n+ *   checked by software\n  * @param src_hdls\n  *   Array to hold the source handle parameters of the completed ops.\n  *   NOTE: If hdls_disable configuration option for the device is set, this\n- *   parameter is ignored.\n+ *   parameter is ignored, and may be NULL\n  * @param dst_hdls\n  *   Array to hold the destination handle parameters of the completed ops.\n  *   NOTE: If hdls_disable configuration option for the device is set, this\n- *   parameter is ignored.\n+ *   parameter is ignored, and may be NULL\n  * @return\n- *   -1 on error, with rte_errno set appropriately.\n- *   Otherwise number of completed operations i.e. number of entries written\n- *   to the src_hdls and dst_hdls array parameters.\n+ *   -1 on device error, with rte_errno set appropriately and parameters\n+ *   unmodified.\n+ *   Otherwise number of returned operations i.e. number of valid entries\n+ *   in the status, src_hdls and dst_hdls array parameters. If status is NULL,\n+ *   and the hdls_disable config option is set, this value may be greater than\n+ *   max_copies parameter.\n  */\n static inline int\n __rte_experimental\n rte_ioat_completed_ops(int dev_id, uint8_t max_copies,\n+\t\tuint32_t *status, uint8_t *num_unsuccessful,\n \t\tuintptr_t *src_hdls, uintptr_t *dst_hdls);\n \n /* include the implementation details from a separate file */\ndiff --git a/drivers/raw/ioat/rte_ioat_rawdev_fns.h b/drivers/raw/ioat/rte_ioat_rawdev_fns.h\nindex 1eff75ec0a..6049e3bd8b 100644\n--- a/drivers/raw/ioat/rte_ioat_rawdev_fns.h\n+++ b/drivers/raw/ioat/rte_ioat_rawdev_fns.h\n@@ -345,16 +345,22 @@ rte_ioat_perform_ops(int dev_id)\n \n static inline int\n rte_ioat_completed_ops(int dev_id, uint8_t max_copies,\n+\t\tuint32_t *status, uint8_t *num_unsuccessful,\n \t\tuintptr_t *src_hdls, uintptr_t *dst_hdls)\n {\n \tenum rte_ioat_dev_type *type =\n \t\t\t(enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;\n+\tuint8_t tmp; /* used so functions don't need to check for null parameter */\n+\n+\tif (num_unsuccessful == NULL)\n+\t\tnum_unsuccessful = &tmp;\n+\n+\t*num_unsuccessful = 0;\n \tif (*type == RTE_IDXD_DEV)\n-\t\treturn __idxd_completed_ops(dev_id, max_copies,\n+\t\treturn __idxd_completed_ops(dev_id, max_copies, status, num_unsuccessful,\n \t\t\t\tsrc_hdls, dst_hdls);\n \telse\n-\t\treturn __ioat_completed_ops(dev_id,  max_copies,\n-\t\t\t\tsrc_hdls, dst_hdls);\n+\t\treturn __ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);\n }\n \n static inline void\n@@ -366,7 +372,8 @@ __rte_deprecated_msg(\"use rte_ioat_completed_ops() instead\")\n rte_ioat_completed_copies(int dev_id, uint8_t max_copies,\n \t\tuintptr_t *src_hdls, uintptr_t *dst_hdls)\n {\n-\treturn rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);\n+\treturn rte_ioat_completed_ops(dev_id, max_copies, NULL, NULL,\n+\t\t\tsrc_hdls, dst_hdls);\n }\n \n #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */\ndiff --git a/examples/ioat/ioatfwd.c b/examples/ioat/ioatfwd.c\nindex 845301a6db..2e377e2d4b 100644\n--- a/examples/ioat/ioatfwd.c\n+++ b/examples/ioat/ioatfwd.c\n@@ -447,12 +447,15 @@ ioat_tx_port(struct rxtx_port_config *tx_config)\n \n \tfor (i = 0; i < tx_config->nb_queues; i++) {\n \t\tif (copy_mode == COPY_MODE_IOAT_NUM) {\n-\t\t\t/* Deque the mbufs from IOAT device. */\n+\t\t\t/* Dequeue the mbufs from IOAT device. Since all memory\n+\t\t\t * is DPDK pinned memory and therefore all addresses should\n+\t\t\t * be valid, we don't check for copy errors\n+\t\t\t */\n \t\t\tnb_dq = rte_ioat_completed_ops(\n-\t\t\t\ttx_config->ioat_ids[i], MAX_PKT_BURST,\n+\t\t\t\ttx_config->ioat_ids[i], MAX_PKT_BURST, NULL, NULL,\n \t\t\t\t(void *)mbufs_src, (void *)mbufs_dst);\n \t\t} else {\n-\t\t\t/* Deque the mbufs from rx_to_tx_ring. */\n+\t\t\t/* Dequeue the mbufs from rx_to_tx_ring. */\n \t\t\tnb_dq = rte_ring_dequeue_burst(\n \t\t\t\ttx_config->rx_to_tx_ring, (void *)mbufs_dst,\n \t\t\t\tMAX_PKT_BURST, NULL);\n@@ -725,7 +728,10 @@ check_link_status(uint32_t port_mask)\n static void\n configure_rawdev_queue(uint32_t dev_id)\n {\n-\tstruct rte_ioat_rawdev_config dev_config = { .ring_size = ring_size };\n+\tstruct rte_ioat_rawdev_config dev_config = {\n+\t\t\t.ring_size = ring_size,\n+\t\t\t.no_prefetch_completions = (cfg.nb_lcores > 1),\n+\t};\n \tstruct rte_rawdev_info info = { .dev_private = &dev_config };\n \n \tif (rte_rawdev_configure(dev_id, &info, sizeof(dev_config)) != 0) {\ndiff --git a/examples/vhost/ioat.c b/examples/vhost/ioat.c\nindex 60b73be936..efdd3f6f76 100644\n--- a/examples/vhost/ioat.c\n+++ b/examples/vhost/ioat.c\n@@ -183,7 +183,7 @@ ioat_check_completed_copies_cb(int vid, uint16_t queue_id,\n \n \t\tuint16_t dev_id = dma_bind[vid].dmas[queue_id * 2\n \t\t\t\t+ VIRTIO_RXQ].dev_id;\n-\t\tn_seg = rte_ioat_completed_ops(dev_id, 255, dump, dump);\n+\t\tn_seg = rte_ioat_completed_ops(dev_id, 255, NULL, NULL, dump, dump);\n \t\tif (n_seg < 0) {\n \t\t\tRTE_LOG(ERR,\n \t\t\t\tVHOST_DATA,\n",
    "prefixes": [
        "v5",
        "12/12"
    ]
}