Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 107826,
    "url": "http://patchwork.dpdk.org/api/patches/107826/?format=api",
    "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20220218171527.56719-9-kai.ji@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20220218171527.56719-9-kai.ji@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20220218171527.56719-9-kai.ji@intel.com",
    "date": "2022-02-18T17:15:26",
    "name": "[v9,8/9] crypto/qat: unify raw data path functions",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "aef3d1f6a0e49ebd55d866b64984ebe17b322e0b",
    "submitter": {
        "id": 2202,
        "url": "http://patchwork.dpdk.org/api/people/2202/?format=api",
        "name": "Ji, Kai",
        "email": "kai.ji@intel.com"
    },
    "delegate": {
        "id": 6690,
        "url": "http://patchwork.dpdk.org/api/users/6690/?format=api",
        "username": "akhil",
        "first_name": "akhil",
        "last_name": "goyal",
        "email": "gakhil@marvell.com"
    },
    "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20220218171527.56719-9-kai.ji@intel.com/mbox/",
    "series": [
        {
            "id": 21741,
            "url": "http://patchwork.dpdk.org/api/series/21741/?format=api",
            "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=21741",
            "date": "2022-02-18T17:15:18",
            "name": "drivers/qat: QAT symmetric crypto datapatch rework",
            "version": 9,
            "mbox": "http://patchwork.dpdk.org/series/21741/mbox/"
        }
    ],
    "comments": "http://patchwork.dpdk.org/api/patches/107826/comments/",
    "check": "success",
    "checks": "http://patchwork.dpdk.org/api/patches/107826/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id AADE6A0032;\n\tFri, 18 Feb 2022 18:16:50 +0100 (CET)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 585B741142;\n\tFri, 18 Feb 2022 18:16:04 +0100 (CET)",
            "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by mails.dpdk.org (Postfix) with ESMTP id 58FCB4113F\n for <dev@dpdk.org>; Fri, 18 Feb 2022 18:15:50 +0100 (CET)",
            "from orsmga005.jf.intel.com ([10.7.209.41])\n by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 18 Feb 2022 09:15:44 -0800",
            "from silpixa00400465.ir.intel.com ([10.55.128.22])\n by orsmga005.jf.intel.com with ESMTP; 18 Feb 2022 09:15:42 -0800"
        ],
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1645204550; x=1676740550;\n h=from:to:cc:subject:date:message-id:in-reply-to: references;\n bh=jACEdLXKaBiXc7nsT4I9xkWJOzJwfIeFYOx/c82dgtk=;\n b=W7Cv5P72AgqEe/eyD8kuL8e/b/MZKMsTO7pr+jeEAunDWVMKmSpN+kD6\n ieqXS+96Eik1cNKQ5c1KEaRyqtU3qL9ASJW+/TgW8/kEbExRsPftj9Ybr\n FmXZbsfRBOJzBznhim4xQautWD+f/b4jwEjp3HM92Kzd8UlabtXJmKRsJ\n otJMR0sEFBWjLUoe9gkqa1yPD02VxL5CEp+gLlKQFgn/+jaGQLXNDz6Gh\n J8o8hYau40+HWGPdp24Io+c1XQQcl3KMIieTtntyFwFWPIK28gjnLFFCN\n W5br17q24soijouvLzCRS4e87bZ9y6182lYuNi56JA2nnv0aeVXeLBDEz A==;",
        "X-IronPort-AV": [
            "E=McAfee;i=\"6200,9189,10262\"; a=\"238571908\"",
            "E=Sophos;i=\"5.88,379,1635231600\"; d=\"scan'208\";a=\"238571908\"",
            "E=Sophos;i=\"5.88,379,1635231600\"; d=\"scan'208\";a=\"705446305\""
        ],
        "X-ExtLoop1": "1",
        "From": "Kai Ji <kai.ji@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "gakhil@marvell.com,\n\troy.fan.zhang@intel.com,\n\tKai Ji <kai.ji@intel.com>",
        "Subject": "[dpdk-dev v9 8/9] crypto/qat: unify raw data path functions",
        "Date": "Sat, 19 Feb 2022 01:15:26 +0800",
        "Message-Id": "<20220218171527.56719-9-kai.ji@intel.com>",
        "X-Mailer": "git-send-email 2.17.1",
        "In-Reply-To": "<20220218171527.56719-1-kai.ji@intel.com>",
        "References": "<20220217162909.22713-1-kai.ji@intel.com>\n <20220218171527.56719-1-kai.ji@intel.com>",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "This patch unifies QAT's raw dp api implementations\nto the same enqueue/dequeue methods used in crypto operations.\nThe specific functions for different QAT generation are updated\nrespectively. The qat_sym_hw_dp.c is removed as no longer required.\n\nSigned-off-by: Kai Ji <kai.ji@intel.com>\n---\n drivers/common/qat/meson.build               |   2 +-\n drivers/compress/qat/qat_comp_pmd.c          |  12 +-\n drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +\n drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++\n drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++\n drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++\n drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 ++++++++++++\n drivers/crypto/qat/qat_crypto.h              |   3 +\n drivers/crypto/qat/qat_sym.c                 |  56 +-\n drivers/crypto/qat/qat_sym_hw_dp.c           | 986 -------------------\n 10 files changed, 1137 insertions(+), 994 deletions(-)\n delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c",
    "diff": "diff --git a/drivers/common/qat/meson.build b/drivers/common/qat/meson.build\nindex f687f5c9d8..b7027f3164 100644\n--- a/drivers/common/qat/meson.build\n+++ b/drivers/common/qat/meson.build\n@@ -74,7 +74,7 @@ endif\n \n if qat_crypto\n     foreach f: ['qat_sym.c', 'qat_sym_session.c',\n-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',\n+            'qat_asym.c', 'qat_crypto.c',\n             'dev/qat_sym_pmd_gen1.c',\n             'dev/qat_asym_pmd_gen1.c',\n             'dev/qat_crypto_pmd_gen2.c',\ndiff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c\nindex 8e497e7a09..dc8db84a68 100644\n--- a/drivers/compress/qat/qat_comp_pmd.c\n+++ b/drivers/compress/qat/qat_comp_pmd.c\n@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {\n \t.private_xform_free\t= qat_comp_private_xform_free\n };\n \n+static uint16_t\n+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)\n+{\n+\treturn qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,\n+\t\t\t\tnb_ops);\n+}\n+\n static uint16_t\n qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,\n \t\t\t\t   uint16_t nb_ops)\n {\n-\tuint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);\n+\tuint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);\n \tstruct qat_qp *tmp_qp = (struct qat_qp *)qp;\n \n \tif (ret) {\n@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,\n \n \t\t} else {\n \t\t\ttmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =\n-\t\t\t\t\t(compressdev_dequeue_pkt_burst_t)\n-\t\t\t\t\tqat_dequeue_op_burst;\n+\t\t\t\t\tqat_comp_dequeue_burst;\n \t\t}\n \t}\n \treturn ret;\ndiff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c b/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c\nindex 64e6ae66ec..0c64c1e43f 100644\n--- a/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c\n+++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c\n@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)\n \t\t\tqat_sym_crypto_cap_get_gen2;\n \tqat_sym_gen_dev_ops[QAT_GEN2].set_session =\n \t\t\tqat_sym_crypto_set_session_gen2;\n+\tqat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =\n+\t\t\tqat_sym_configure_raw_dp_ctx_gen1;\n \tqat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =\n \t\t\tqat_sym_crypto_feature_flags_get_gen1;\n \ndiff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c b/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c\nindex db864d973a..ffa093a7a3 100644\n--- a/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c\n+++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c\n@@ -394,6 +394,218 @@ qat_sym_crypto_set_session_gen3(void *cdev __rte_unused, void *session)\n \treturn ret;\n }\n \n+static int\n+qat_sym_dp_enqueue_single_aead_gen3(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_aead_job_gen3(ctx, req, iv, digest, aad, ofs,\n+\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,\n+\t\t\tNULL, aad, digest);\n+#endif\n+\treturn 0;\n+}\n+\n+static uint32_t\n+qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\n+\t\tenqueue_one_aead_job_gen3(ctx, req, &vec->iv[i],\n+\t\t\t\t&vec->digest[i], &vec->aad[i], ofs,\n+\t\t\t\t(uint32_t)data_len);\n+\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, &vec->iv[i], NULL,\n+\t\t\t\t&vec->aad[i], &vec->digest[i]);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+static int\n+qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv __rte_unused,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *auth_iv,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_auth_job_gen3(ctx, cookie, req, digest, auth_iv, ofs,\n+\t\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+\treturn 0;\n+}\n+\n+static uint32_t\n+qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\tenqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],\n+\t\t\t&vec->auth_iv[i], ofs, (uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+static int\n+qat_sym_configure_raw_dp_ctx_gen3(void *_raw_dp_ctx, void *_ctx)\n+{\n+\tstruct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;\n+\tstruct qat_sym_session *ctx = _ctx;\n+\tint ret;\n+\n+\tret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tif (ctx->is_single_pass) {\n+\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen3;\n+\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen3;\n+\t} else if (ctx->is_single_pass_gmac) {\n+\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen3;\n+\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen3;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+\n RTE_INIT(qat_sym_crypto_gen3_init)\n {\n \tqat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;\n@@ -403,6 +615,8 @@ RTE_INIT(qat_sym_crypto_gen3_init)\n \t\t\tqat_sym_crypto_feature_flags_get_gen1;\n \tqat_sym_gen_dev_ops[QAT_GEN3].set_session =\n \t\t\tqat_sym_crypto_set_session_gen3;\n+\tqat_sym_gen_dev_ops[QAT_GEN3].set_raw_dp_ctx =\n+\t\t\tqat_sym_configure_raw_dp_ctx_gen3;\n #ifdef RTE_LIB_SECURITY\n \tqat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx =\n \t\t\tqat_sym_create_security_gen1;\ndiff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c b/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c\nindex 7642a87d55..f803bc1459 100644\n--- a/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c\n+++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c\n@@ -223,6 +223,126 @@ qat_sym_crypto_set_session_gen4(void *cdev, void *session)\n \treturn ret;\n }\n \n+static int\n+qat_sym_dp_enqueue_single_aead_gen4(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_aead_job_gen4(ctx, req, iv, digest, aad, ofs,\n+\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,\n+\t\t\tNULL, aad, digest);\n+#endif\n+\treturn 0;\n+}\n+\n+static uint32_t\n+qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\n+\t\tenqueue_one_aead_job_gen4(ctx, req, &vec->iv[i],\n+\t\t\t\t&vec->digest[i], &vec->aad[i], ofs,\n+\t\t\t\t(uint32_t)data_len);\n+\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, &vec->iv[i], NULL,\n+\t\t\t\t&vec->aad[i], &vec->digest[i]);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+static int\n+qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx)\n+{\n+\tstruct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;\n+\tstruct qat_sym_session *ctx = _ctx;\n+\tint ret;\n+\n+\tret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tif (ctx->is_single_pass && ctx->is_ucs) {\n+\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;\n+\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen4;\n+\t}\n+\n+\treturn 0;\n+}\n+\n RTE_INIT(qat_sym_crypto_gen4_init)\n {\n \tqat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;\n@@ -230,6 +350,8 @@ RTE_INIT(qat_sym_crypto_gen4_init)\n \t\t\tqat_sym_crypto_cap_get_gen4;\n \tqat_sym_gen_dev_ops[QAT_GEN4].set_session =\n \t\t\tqat_sym_crypto_set_session_gen4;\n+\tqat_sym_gen_dev_ops[QAT_GEN4].set_raw_dp_ctx =\n+\t\t\tqat_sym_configure_raw_dp_ctx_gen4;\n \tqat_sym_gen_dev_ops[QAT_GEN4].get_feature_flags =\n \t\t\tqat_sym_crypto_feature_flags_get_gen1;\n #ifdef RTE_LIB_SECURITY\ndiff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h\nindex 96cdb97a26..50a9c5ad5b 100644\n--- a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h\n+++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h\n@@ -839,6 +839,84 @@ int\n qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,\n \t\tuint8_t *out_msg, void *op_cookie);\n \n+/* -----------------GEN 1 sym crypto raw data path APIs ---------------- */\n+int\n+qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest __rte_unused,\n+\tstruct rte_crypto_va_iova_ptr *aad __rte_unused,\n+\tvoid *user_data);\n+\n+uint32_t\n+qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status);\n+\n+int\n+qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv __rte_unused,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *auth_iv,\n+\tvoid *user_data);\n+\n+uint32_t\n+qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status);\n+\n+int\n+qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *cipher_iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *auth_iv,\n+\tvoid *user_data);\n+\n+uint32_t\n+qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status);\n+\n+int\n+qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tvoid *user_data);\n+\n+uint32_t\n+qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status);\n+\n+void *\n+qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tint *dequeue_status, enum rte_crypto_op_status *op_status);\n+\n+uint32_t\n+qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,\n+\trte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,\n+\tuint32_t max_nb_to_dequeue,\n+\trte_cryptodev_raw_post_dequeue_t post_dequeue,\n+\tvoid **out_user_data, uint8_t is_user_data_array,\n+\tuint32_t *n_success_jobs, int *return_status);\n+\n+int\n+qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);\n+\n+int\n+qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);\n+\n+int\n+qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);\n+\n /* -----------------GENx control path APIs ---------------- */\n uint64_t\n qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);\ndiff --git a/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c b/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c\nindex c58a628915..fee6507512 100644\n--- a/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c\n+++ b/drivers/crypto/qat/dev/qat_sym_pmd_gen1.c\n@@ -146,6 +146,10 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {\n \t.sym_session_get_size\t= qat_sym_session_get_private_size,\n \t.sym_session_configure\t= qat_sym_session_configure,\n \t.sym_session_clear\t= qat_sym_session_clear,\n+\n+\t/* Raw data-path API related operations */\n+\t.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,\n+\t.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,\n };\n \n static struct qat_capabilities_info\n@@ -448,6 +452,656 @@ qat_sym_create_security_gen1(void *cryptodev)\n }\n \n #endif\n+int\n+qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest __rte_unused,\n+\tstruct rte_crypto_va_iova_ptr *aad __rte_unused,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_cipher_job_gen1(ctx, req, iv, ofs, (uint32_t)data_len);\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, &iv,\n+\t\t\tNULL, NULL, NULL);\n+#endif\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+\treturn 0;\n+}\n+\n+uint32_t\n+qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i],\n+\t\t\t\tcookie, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\tenqueue_one_cipher_job_gen1(ctx, req, &vec->iv[i], ofs,\n+\t\t\t(uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, &vec->iv[i],\n+\t\t\t\tNULL, NULL, NULL);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+int\n+qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv __rte_unused,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *auth_iv,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,\n+\t\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, NULL,\n+\t\t\tauth_iv, NULL, digest);\n+#endif\n+\treturn 0;\n+}\n+\n+uint32_t\n+qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\t\tenqueue_one_auth_job_gen1(ctx, req, &vec->digest[i],\n+\t\t\t&vec->auth_iv[i], ofs, (uint32_t)data_len);\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, NULL, &vec->auth_iv[i],\n+\t\t\t\tNULL, &vec->digest[i]);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+int\n+qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *cipher_iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *auth_iv,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tif (unlikely(enqueue_one_chain_job_gen1(ctx, req, data, n_data_vecs,\n+\t\t\tNULL, 0, cipher_iv, digest, auth_iv, ofs,\n+\t\t\t(uint32_t)data_len)))\n+\t\treturn -1;\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, cipher_iv,\n+\t\t\tauth_iv, NULL, digest);\n+#endif\n+\treturn 0;\n+}\n+\n+uint32_t\n+qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\n+\t\tif (unlikely(enqueue_one_chain_job_gen1(ctx, req,\n+\t\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num,\n+\t\t\t\tNULL, 0,\n+\t\t\t\t&vec->iv[i], &vec->digest[i],\n+\t\t\t\t&vec->auth_iv[i], ofs, (uint32_t)data_len)))\n+\t\t\tbreak;\n+\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, &vec->iv[i],\n+\t\t\t\t&vec->auth_iv[i],\n+\t\t\t\tNULL, &vec->digest[i]);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+int\n+qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n+\tunion rte_crypto_sym_ofs ofs,\n+\tstruct rte_crypto_va_iova_ptr *iv,\n+\tstruct rte_crypto_va_iova_ptr *digest,\n+\tstruct rte_crypto_va_iova_ptr *aad,\n+\tvoid *user_data)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_op_cookie *cookie;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\n+\tint32_t data_len;\n+\tuint32_t tail = dp_ctx->tail;\n+\n+\treq = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\tcookie = qp->op_cookies[tail >> tx_queue->trailz];\n+\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n+\tdata_len = qat_sym_build_req_set_data(req, user_data, cookie,\n+\t\t\tdata, n_data_vecs, NULL, 0);\n+\tif (unlikely(data_len < 0))\n+\t\treturn -1;\n+\n+\tenqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs,\n+\t\t(uint32_t)data_len);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue++;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\tqat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,\n+\t\t\tNULL, aad, digest);\n+#endif\n+\treturn 0;\n+}\n+\n+uint32_t\n+qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n+\tvoid *user_data[], int *status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_session *ctx = dp_ctx->session;\n+\tuint32_t i, n;\n+\tuint32_t tail;\n+\tstruct icp_qat_fw_la_bulk_req *req;\n+\tint32_t data_len;\n+\n+\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n+\tif (unlikely(n == 0)) {\n+\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n+\t\t*status = 0;\n+\t\treturn 0;\n+\t}\n+\n+\ttail = dp_ctx->tail;\n+\n+\tfor (i = 0; i < n; i++) {\n+\t\tstruct qat_sym_op_cookie *cookie =\n+\t\t\tqp->op_cookies[tail >> tx_queue->trailz];\n+\n+\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n+\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n+\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n+\n+\t\tdata_len = qat_sym_build_req_set_data(req, user_data[i], cookie,\n+\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);\n+\t\tif (unlikely(data_len < 0))\n+\t\t\tbreak;\n+\n+\t\tenqueue_one_aead_job_gen1(ctx, req, &vec->iv[i],\n+\t\t\t\t&vec->digest[i], &vec->aad[i], ofs,\n+\t\t\t\t(uint32_t)data_len);\n+\n+\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n+\n+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG\n+\t\tqat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,\n+\t\t\t\tvec->src_sgl[i].num, &vec->iv[i], NULL,\n+\t\t\t\t&vec->aad[i], &vec->digest[i]);\n+#endif\n+\t}\n+\n+\tif (unlikely(i < n))\n+\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n+\n+\tdp_ctx->tail = tail;\n+\tdp_ctx->cached_enqueue += i;\n+\t*status = 0;\n+\treturn i;\n+}\n+\n+\n+uint32_t\n+qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,\n+\trte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,\n+\tuint32_t max_nb_to_dequeue,\n+\trte_cryptodev_raw_post_dequeue_t post_dequeue,\n+\tvoid **out_user_data, uint8_t is_user_data_array,\n+\tuint32_t *n_success_jobs, int *return_status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\tstruct icp_qat_fw_comn_resp *resp;\n+\tvoid *resp_opaque;\n+\tuint32_t i, n, inflight;\n+\tuint32_t head;\n+\tuint8_t status;\n+\n+\t*n_success_jobs = 0;\n+\t*return_status = 0;\n+\thead = dp_ctx->head;\n+\n+\tinflight = qp->enqueued - qp->dequeued;\n+\tif (unlikely(inflight == 0))\n+\t\treturn 0;\n+\n+\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n+\t\t\thead);\n+\t/* no operation ready */\n+\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\treturn 0;\n+\n+\tresp_opaque = (void *)(uintptr_t)resp->opaque_data;\n+\t/* get the dequeue count */\n+\tif (get_dequeue_count) {\n+\t\tn = get_dequeue_count(resp_opaque);\n+\t\tif (unlikely(n == 0))\n+\t\t\treturn 0;\n+\t} else {\n+\t\tif (unlikely(max_nb_to_dequeue == 0))\n+\t\t\treturn 0;\n+\t\tn = max_nb_to_dequeue;\n+\t}\n+\n+\tout_user_data[0] = resp_opaque;\n+\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\tpost_dequeue(resp_opaque, 0, status);\n+\t*n_success_jobs += status;\n+\n+\thead = (head + rx_queue->msg_size) & rx_queue->modulo_mask;\n+\n+\t/* we already finished dequeue when n == 1 */\n+\tif (unlikely(n == 1)) {\n+\t\ti = 1;\n+\t\tgoto end_deq;\n+\t}\n+\n+\tif (is_user_data_array) {\n+\t\tfor (i = 1; i < n; i++) {\n+\t\t\tresp = (struct icp_qat_fw_comn_resp *)(\n+\t\t\t\t(uint8_t *)rx_queue->base_addr + head);\n+\t\t\tif (unlikely(*(uint32_t *)resp ==\n+\t\t\t\t\tADF_RING_EMPTY_SIG))\n+\t\t\t\tgoto end_deq;\n+\t\t\tout_user_data[i] = (void *)(uintptr_t)resp->opaque_data;\n+\t\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\t\t\t*n_success_jobs += status;\n+\t\t\tpost_dequeue(out_user_data[i], i, status);\n+\t\t\thead = (head + rx_queue->msg_size) &\n+\t\t\t\t\trx_queue->modulo_mask;\n+\t\t}\n+\n+\t\tgoto end_deq;\n+\t}\n+\n+\t/* opaque is not array */\n+\tfor (i = 1; i < n; i++) {\n+\t\tresp = (struct icp_qat_fw_comn_resp *)(\n+\t\t\t(uint8_t *)rx_queue->base_addr + head);\n+\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n+\t\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\t\tgoto end_deq;\n+\t\thead = (head + rx_queue->msg_size) &\n+\t\t\t\trx_queue->modulo_mask;\n+\t\tpost_dequeue(resp_opaque, i, status);\n+\t\t*n_success_jobs += status;\n+\t}\n+\n+end_deq:\n+\tdp_ctx->head = head;\n+\tdp_ctx->cached_dequeue += i;\n+\treturn i;\n+}\n+\n+void *\n+qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,\n+\tint *dequeue_status, enum rte_crypto_op_status *op_status)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\tregister struct icp_qat_fw_comn_resp *resp;\n+\n+\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n+\t\t\tdp_ctx->head);\n+\n+\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n+\t\treturn NULL;\n+\n+\tdp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &\n+\t\t\trx_queue->modulo_mask;\n+\tdp_ctx->cached_dequeue++;\n+\n+\t*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?\n+\t\t\tRTE_CRYPTO_OP_STATUS_SUCCESS :\n+\t\t\tRTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n+\t*dequeue_status = 0;\n+\treturn (void *)(uintptr_t)resp->opaque_data;\n+}\n+\n+int\n+qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_queue *tx_queue = &qp->tx_q;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\n+\tif (unlikely(dp_ctx->cached_enqueue != n))\n+\t\treturn -1;\n+\n+\tqp->enqueued += n;\n+\tqp->stats.enqueued_count += n;\n+\n+\ttx_queue->tail = dp_ctx->tail;\n+\n+\tWRITE_CSR_RING_TAIL(qp->mmap_bar_addr,\n+\t\t\ttx_queue->hw_bundle_number,\n+\t\t\ttx_queue->hw_queue_number, tx_queue->tail);\n+\ttx_queue->csr_tail = tx_queue->tail;\n+\tdp_ctx->cached_enqueue = 0;\n+\n+\treturn 0;\n+}\n+\n+int\n+qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)\n+{\n+\tstruct qat_qp *qp = qp_data;\n+\tstruct qat_queue *rx_queue = &qp->rx_q;\n+\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n+\n+\tif (unlikely(dp_ctx->cached_dequeue != n))\n+\t\treturn -1;\n+\n+\trx_queue->head = dp_ctx->head;\n+\trx_queue->nb_processed_responses += n;\n+\tqp->dequeued += n;\n+\tqp->stats.dequeued_count += n;\n+\tif (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {\n+\t\tuint32_t old_head, new_head;\n+\t\tuint32_t max_head;\n+\n+\t\told_head = rx_queue->csr_head;\n+\t\tnew_head = rx_queue->head;\n+\t\tmax_head = qp->nb_descriptors * rx_queue->msg_size;\n+\n+\t\t/* write out free descriptors */\n+\t\tvoid *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;\n+\n+\t\tif (new_head < old_head) {\n+\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,\n+\t\t\t\t\tmax_head - old_head);\n+\t\t\tmemset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,\n+\t\t\t\t\tnew_head);\n+\t\t} else {\n+\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -\n+\t\t\t\t\told_head);\n+\t\t}\n+\t\trx_queue->nb_processed_responses = 0;\n+\t\trx_queue->csr_head = new_head;\n+\n+\t\t/* write current head to CSR */\n+\t\tWRITE_CSR_RING_HEAD(qp->mmap_bar_addr,\n+\t\t\trx_queue->hw_bundle_number, rx_queue->hw_queue_number,\n+\t\t\tnew_head);\n+\t}\n+\n+\tdp_ctx->cached_dequeue = 0;\n+\treturn 0;\n+}\n+\n+int\n+qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx)\n+{\n+\tstruct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;\n+\tstruct qat_sym_session *ctx = _ctx;\n+\n+\traw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen1;\n+\traw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1;\n+\traw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1;\n+\traw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen1;\n+\n+\tif ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||\n+\t\t\tctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&\n+\t\t\t!ctx->is_gmac) {\n+\t\t/* AES-GCM or AES-CCM */\n+\t\tif (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||\n+\t\t\tctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||\n+\t\t\t(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128\n+\t\t\t&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE\n+\t\t\t&& ctx->qat_hash_alg ==\n+\t\t\t\t\tICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_aead_jobs_gen1;\n+\t\t\traw_dp_ctx->enqueue =\n+\t\t\t\t\tqat_sym_dp_enqueue_single_aead_gen1;\n+\t\t} else {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_chain_jobs_gen1;\n+\t\t\traw_dp_ctx->enqueue =\n+\t\t\t\t\tqat_sym_dp_enqueue_single_chain_gen1;\n+\t\t}\n+\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {\n+\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1;\n+\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1;\n+\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {\n+\t\tif (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||\n+\t\t\tctx->qat_cipher_alg ==\n+\t\t\t\tICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_aead_jobs_gen1;\n+\t\t\traw_dp_ctx->enqueue =\n+\t\t\t\t\tqat_sym_dp_enqueue_single_aead_gen1;\n+\t\t} else {\n+\t\t\traw_dp_ctx->enqueue_burst =\n+\t\t\t\t\tqat_sym_dp_enqueue_cipher_jobs_gen1;\n+\t\t\traw_dp_ctx->enqueue =\n+\t\t\t\t\tqat_sym_dp_enqueue_single_cipher_gen1;\n+\t\t}\n+\t} else\n+\t\treturn -1;\n+\n+\treturn 0;\n+}\n+\n int\n qat_sym_crypto_set_session_gen1(void *cryptodev __rte_unused, void *session)\n {\n@@ -518,6 +1172,8 @@ RTE_INIT(qat_sym_crypto_gen1_init)\n \t\t\tqat_sym_crypto_cap_get_gen1;\n \tqat_sym_gen_dev_ops[QAT_GEN1].set_session =\n \t\t\tqat_sym_crypto_set_session_gen1;\n+\tqat_sym_gen_dev_ops[QAT_GEN1].set_raw_dp_ctx =\n+\t\t\tqat_sym_configure_raw_dp_ctx_gen1;\n \tqat_sym_gen_dev_ops[QAT_GEN1].get_feature_flags =\n \t\t\tqat_sym_crypto_feature_flags_get_gen1;\n #ifdef RTE_LIB_SECURITY\ndiff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h\nindex c01266f81c..cf386d0ed0 100644\n--- a/drivers/crypto/qat/qat_crypto.h\n+++ b/drivers/crypto/qat/qat_crypto.h\n@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);\n \n typedef int (*set_session_t)(void *cryptodev, void *session);\n \n+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);\n+\n struct qat_crypto_gen_dev_ops {\n \tget_feature_flags_t get_feature_flags;\n \tget_capabilities_info_t get_capabilities;\n \tstruct rte_cryptodev_ops *cryptodev_ops;\n \tset_session_t set_session;\n+\tset_raw_dp_ctx_t set_raw_dp_ctx;\n #ifdef RTE_LIB_SECURITY\n \tcreate_security_ctx_t create_security_ctx;\n #endif\ndiff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c\nindex 692e1f8f0a..1ccffad5ab 100644\n--- a/drivers/crypto/qat/qat_sym.c\n+++ b/drivers/crypto/qat/qat_sym.c\n@@ -89,7 +89,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,\n \t\t\t\t\t(void *)cdev, (void *)sess);\n \t\t\t\tif (ret < 0) {\n \t\t\t\t\top->status =\n-\t\t\t\t\tRTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\t\t\t\t\tRTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n \t\t\t\t\treturn -EINVAL;\n \t\t\t\t}\n \t\t\t}\n@@ -144,7 +144,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,\n \t\t\t\t\t(void *)cdev, (void *)sess);\n \t\t\t\tif (ret < 0) {\n \t\t\t\t\top->status =\n-\t\t\t\t\tRTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n+\t\t\t\t\t\tRTE_CRYPTO_OP_STATUS_INVALID_SESSION;\n \t\t\t\t\treturn -EINVAL;\n \t\t\t\t}\n \t\t\t}\n@@ -292,8 +292,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,\n \t\tif (internals->capa_mz == NULL) {\n \t\t\tQAT_LOG(DEBUG,\n \t\t\t\t\"Error allocating memzone for capabilities, \"\n-\t\t\t\t\"destroying PMD for %s\",\n-\t\t\t\tname);\n+\t\t\t\t\"destroying PMD for %s\", name);\n \t\t\tret = -EFAULT;\n \t\t\tgoto error;\n \t\t}\n@@ -355,6 +354,55 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)\n \treturn 0;\n }\n \n+int\n+qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n+\tstruct rte_crypto_raw_dp_ctx *raw_dp_ctx,\n+\tenum rte_crypto_op_sess_type sess_type,\n+\tunion rte_cryptodev_session_ctx session_ctx, uint8_t is_update)\n+{\n+\tstruct qat_cryptodev_private *internals = dev->data->dev_private;\n+\tenum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;\n+\tstruct qat_crypto_gen_dev_ops *gen_dev_ops =\n+\t\t\t&qat_sym_gen_dev_ops[qat_dev_gen];\n+\tstruct qat_qp *qp;\n+\tstruct qat_sym_session *ctx;\n+\tstruct qat_sym_dp_ctx *dp_ctx;\n+\n+\tif (!gen_dev_ops->set_raw_dp_ctx) {\n+\t\tQAT_LOG(ERR, \"Device GEN %u does not support raw data path\",\n+\t\t\t\tqat_dev_gen);\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tqp = dev->data->queue_pairs[qp_id];\n+\tdp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;\n+\n+\tif (!is_update) {\n+\t\tmemset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +\n+\t\t\t\tsizeof(struct qat_sym_dp_ctx));\n+\t\traw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];\n+\t\tdp_ctx->tail = qp->tx_q.tail;\n+\t\tdp_ctx->head = qp->rx_q.head;\n+\t\tdp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;\n+\t}\n+\n+\tif (sess_type != RTE_CRYPTO_OP_WITH_SESSION)\n+\t\treturn -EINVAL;\n+\n+\tctx = (struct qat_sym_session *)get_sym_session_private_data(\n+\t\t\tsession_ctx.crypto_sess, qat_sym_driver_id);\n+\n+\tdp_ctx->session = ctx;\n+\n+\treturn gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);\n+}\n+\n+int\n+qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)\n+{\n+\treturn sizeof(struct qat_sym_dp_ctx);\n+}\n+\n static struct cryptodev_driver qat_crypto_drv;\n RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,\n \t\tcryptodev_qat_sym_driver,\ndiff --git a/drivers/crypto/qat/qat_sym_hw_dp.c b/drivers/crypto/qat/qat_sym_hw_dp.c\ndeleted file mode 100644\nindex 5322faff44..0000000000\n--- a/drivers/crypto/qat/qat_sym_hw_dp.c\n+++ /dev/null\n@@ -1,986 +0,0 @@\n-/* SPDX-License-Identifier: BSD-3-Clause\n- * Copyright(c) 2022 Intel Corporation\n- */\n-\n-#include <cryptodev_pmd.h>\n-\n-#include \"adf_transport_access_macros.h\"\n-#include \"icp_qat_fw.h\"\n-#include \"icp_qat_fw_la.h\"\n-\n-#include \"qat_sym.h\"\n-#include \"qat_sym_session.h\"\n-#include \"qat_qp.h\"\n-\n-static __rte_always_inline int32_t\n-qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,\n-\t\tstruct rte_crypto_vec *data, uint16_t n_data_vecs)\n-{\n-\tstruct qat_queue *tx_queue;\n-\tstruct qat_sym_op_cookie *cookie;\n-\tstruct qat_sgl *list;\n-\tuint32_t i;\n-\tuint32_t total_len;\n-\n-\tif (likely(n_data_vecs == 1)) {\n-\t\treq->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =\n-\t\t\tdata[0].iova;\n-\t\treq->comn_mid.src_length = req->comn_mid.dst_length =\n-\t\t\tdata[0].len;\n-\t\treturn data[0].len;\n-\t}\n-\n-\tif (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)\n-\t\treturn -1;\n-\n-\ttotal_len = 0;\n-\ttx_queue = &qp->tx_q;\n-\n-\tICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,\n-\t\t\tQAT_COMN_PTR_TYPE_SGL);\n-\tcookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];\n-\tlist = (struct qat_sgl *)&cookie->qat_sgl_src;\n-\n-\tfor (i = 0; i < n_data_vecs; i++) {\n-\t\tlist->buffers[i].len = data[i].len;\n-\t\tlist->buffers[i].resrvd = 0;\n-\t\tlist->buffers[i].addr = data[i].iova;\n-\t\tif (total_len + data[i].len > UINT32_MAX) {\n-\t\t\tQAT_DP_LOG(ERR, \"Message too long\");\n-\t\t\treturn -1;\n-\t\t}\n-\t\ttotal_len += data[i].len;\n-\t}\n-\n-\tlist->num_bufs = i;\n-\treq->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =\n-\t\t\tcookie->qat_sgl_src_phys_addr;\n-\treq->comn_mid.src_length = req->comn_mid.dst_length = 0;\n-\treturn total_len;\n-}\n-\n-static __rte_always_inline void\n-set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,\n-\t\tstruct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,\n-\t\tstruct icp_qat_fw_la_bulk_req *qat_req)\n-{\n-\t/* copy IV into request if it fits */\n-\tif (iv_len <= sizeof(cipher_param->u.cipher_IV_array))\n-\t\trte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,\n-\t\t\t\tiv_len);\n-\telse {\n-\t\tICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(\n-\t\t\t\tqat_req->comn_hdr.serv_specif_flags,\n-\t\t\t\tICP_QAT_FW_CIPH_IV_64BIT_PTR);\n-\t\tcipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;\n-\t}\n-}\n-\n-#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \\\n-\t(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \\\n-\tICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))\n-\n-static __rte_always_inline void\n-qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)\n-{\n-\tuint32_t i;\n-\n-\tfor (i = 0; i < n; i++)\n-\t\tsta[i] = status;\n-}\n-\n-#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \\\n-\tRTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)\n-\n-static __rte_always_inline void\n-enqueue_one_cipher_job(struct qat_sym_session *ctx,\n-\tstruct icp_qat_fw_la_bulk_req *req,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n-{\n-\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n-\n-\tcipher_param = (void *)&req->serv_specif_rqpars;\n-\n-\t/* cipher IV */\n-\tset_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);\n-\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n-\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head -\n-\t\t\tofs.ofs.cipher.tail;\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_enqueue_single_cipher(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n-\tunion rte_crypto_sym_ofs ofs,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tstruct rte_crypto_va_iova_ptr *digest __rte_unused,\n-\tstruct rte_crypto_va_iova_ptr *aad __rte_unused,\n-\tvoid *user_data)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\tuint32_t tail = dp_ctx->tail;\n-\n-\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n-\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n-\tif (unlikely(data_len < 0))\n-\t\treturn -1;\n-\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n-\n-\tenqueue_one_cipher_job(ctx, req, iv, ofs, (uint32_t)data_len);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue++;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_enqueue_cipher_jobs(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n-\tvoid *user_data[], int *status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tuint32_t i, n;\n-\tuint32_t tail;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\n-\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n-\tif (unlikely(n == 0)) {\n-\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n-\t\t*status = 0;\n-\t\treturn 0;\n-\t}\n-\n-\ttail = dp_ctx->tail;\n-\n-\tfor (i = 0; i < n; i++) {\n-\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\n-\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req,\n-\t\t\tvec->src_sgl[i].vec,\n-\t\t\tvec->src_sgl[i].num);\n-\t\tif (unlikely(data_len < 0))\n-\t\t\tbreak;\n-\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n-\t\tenqueue_one_cipher_job(ctx, req, &vec->iv[i], ofs,\n-\t\t\t(uint32_t)data_len);\n-\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\t}\n-\n-\tif (unlikely(i < n))\n-\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue += i;\n-\t*status = 0;\n-\treturn i;\n-}\n-\n-static __rte_always_inline void\n-enqueue_one_auth_job(struct qat_sym_session *ctx,\n-\tstruct icp_qat_fw_la_bulk_req *req,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *auth_iv,\n-\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n-{\n-\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n-\tstruct icp_qat_fw_la_auth_req_params *auth_param;\n-\n-\tcipher_param = (void *)&req->serv_specif_rqpars;\n-\tauth_param = (void *)((uint8_t *)cipher_param +\n-\t\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n-\n-\tauth_param->auth_off = ofs.ofs.auth.head;\n-\tauth_param->auth_len = data_len - ofs.ofs.auth.head -\n-\t\t\tofs.ofs.auth.tail;\n-\tauth_param->auth_res_addr = digest->iova;\n-\n-\tswitch (ctx->qat_hash_alg) {\n-\tcase ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:\n-\tcase ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:\n-\tcase ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:\n-\t\tauth_param->u1.aad_adr = auth_iv->iova;\n-\t\tbreak;\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n-\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n-\t\t\treq->comn_hdr.serv_specif_flags,\n-\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n-\t\trte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,\n-\t\t\t\tctx->auth_iv.length);\n-\t\tbreak;\n-\tdefault:\n-\t\tbreak;\n-\t}\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_enqueue_single_auth(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n-\tunion rte_crypto_sym_ofs ofs,\n-\tstruct rte_crypto_va_iova_ptr *iv __rte_unused,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *auth_iv,\n-\tvoid *user_data)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\tuint32_t tail = dp_ctx->tail;\n-\n-\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n-\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n-\tif (unlikely(data_len < 0))\n-\t\treturn -1;\n-\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n-\n-\tenqueue_one_auth_job(ctx, req, digest, auth_iv, ofs,\n-\t\t\t(uint32_t)data_len);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue++;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_enqueue_auth_jobs(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n-\tvoid *user_data[], int *status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tuint32_t i, n;\n-\tuint32_t tail;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\n-\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n-\tif (unlikely(n == 0)) {\n-\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n-\t\t*status = 0;\n-\t\treturn 0;\n-\t}\n-\n-\ttail = dp_ctx->tail;\n-\n-\tfor (i = 0; i < n; i++) {\n-\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\n-\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req,\n-\t\t\tvec->src_sgl[i].vec,\n-\t\t\tvec->src_sgl[i].num);\n-\t\tif (unlikely(data_len < 0))\n-\t\t\tbreak;\n-\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n-\t\tenqueue_one_auth_job(ctx, req, &vec->digest[i],\n-\t\t\t&vec->auth_iv[i], ofs, (uint32_t)data_len);\n-\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\t}\n-\n-\tif (unlikely(i < n))\n-\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue += i;\n-\t*status = 0;\n-\treturn i;\n-}\n-\n-static __rte_always_inline int\n-enqueue_one_chain_job(struct qat_sym_session *ctx,\n-\tstruct icp_qat_fw_la_bulk_req *req,\n-\tstruct rte_crypto_vec *data,\n-\tuint16_t n_data_vecs,\n-\tstruct rte_crypto_va_iova_ptr *cipher_iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *auth_iv,\n-\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n-{\n-\tstruct icp_qat_fw_la_cipher_req_params *cipher_param;\n-\tstruct icp_qat_fw_la_auth_req_params *auth_param;\n-\trte_iova_t auth_iova_end;\n-\tint32_t cipher_len, auth_len;\n-\n-\tcipher_param = (void *)&req->serv_specif_rqpars;\n-\tauth_param = (void *)((uint8_t *)cipher_param +\n-\t\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n-\n-\tcipher_len = data_len - ofs.ofs.cipher.head -\n-\t\t\tofs.ofs.cipher.tail;\n-\tauth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;\n-\n-\tif (unlikely(cipher_len < 0 || auth_len < 0))\n-\t\treturn -1;\n-\n-\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n-\tcipher_param->cipher_length = cipher_len;\n-\tset_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);\n-\n-\tauth_param->auth_off = ofs.ofs.auth.head;\n-\tauth_param->auth_len = auth_len;\n-\tauth_param->auth_res_addr = digest->iova;\n-\n-\tswitch (ctx->qat_hash_alg) {\n-\tcase ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:\n-\tcase ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:\n-\tcase ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:\n-\t\tauth_param->u1.aad_adr = auth_iv->iova;\n-\t\tbreak;\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n-\t\tbreak;\n-\tdefault:\n-\t\tbreak;\n-\t}\n-\n-\tif (unlikely(n_data_vecs > 1)) {\n-\t\tint auth_end_get = 0, i = n_data_vecs - 1;\n-\t\tstruct rte_crypto_vec *cvec = &data[0];\n-\t\tuint32_t len;\n-\n-\t\tlen = data_len - ofs.ofs.auth.tail;\n-\n-\t\twhile (i >= 0 && len > 0) {\n-\t\t\tif (cvec->len >= len) {\n-\t\t\t\tauth_iova_end = cvec->iova + len;\n-\t\t\t\tlen = 0;\n-\t\t\t\tauth_end_get = 1;\n-\t\t\t\tbreak;\n-\t\t\t}\n-\t\t\tlen -= cvec->len;\n-\t\t\ti--;\n-\t\t\tcvec++;\n-\t\t}\n-\n-\t\tif (unlikely(auth_end_get == 0))\n-\t\t\treturn -1;\n-\t} else\n-\t\tauth_iova_end = data[0].iova + auth_param->auth_off +\n-\t\t\tauth_param->auth_len;\n-\n-\t/* Then check if digest-encrypted conditions are met */\n-\tif ((auth_param->auth_off + auth_param->auth_len <\n-\t\tcipher_param->cipher_offset +\n-\t\tcipher_param->cipher_length) &&\n-\t\t(digest->iova == auth_iova_end)) {\n-\t\t/* Handle partial digest encryption */\n-\t\tif (cipher_param->cipher_offset +\n-\t\t\t\tcipher_param->cipher_length <\n-\t\t\t\tauth_param->auth_off +\n-\t\t\t\tauth_param->auth_len +\n-\t\t\t\tctx->digest_length)\n-\t\t\treq->comn_mid.dst_length =\n-\t\t\t\treq->comn_mid.src_length =\n-\t\t\t\tauth_param->auth_off +\n-\t\t\t\tauth_param->auth_len +\n-\t\t\t\tctx->digest_length;\n-\t\tstruct icp_qat_fw_comn_req_hdr *header =\n-\t\t\t&req->comn_hdr;\n-\t\tICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(\n-\t\t\theader->serv_specif_flags,\n-\t\t\tICP_QAT_FW_LA_DIGEST_IN_BUFFER);\n-\t}\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_enqueue_single_chain(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n-\tunion rte_crypto_sym_ofs ofs,\n-\tstruct rte_crypto_va_iova_ptr *cipher_iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *auth_iv,\n-\tvoid *user_data)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\tuint32_t tail = dp_ctx->tail;\n-\n-\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n-\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n-\tif (unlikely(data_len < 0))\n-\t\treturn -1;\n-\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n-\n-\tif (unlikely(enqueue_one_chain_job(ctx, req, data, n_data_vecs,\n-\t\t\tcipher_iv, digest, auth_iv, ofs, (uint32_t)data_len)))\n-\t\treturn -1;\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue++;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n-\tvoid *user_data[], int *status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tuint32_t i, n;\n-\tuint32_t tail;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\n-\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n-\tif (unlikely(n == 0)) {\n-\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n-\t\t*status = 0;\n-\t\treturn 0;\n-\t}\n-\n-\ttail = dp_ctx->tail;\n-\n-\tfor (i = 0; i < n; i++) {\n-\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\n-\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req,\n-\t\t\tvec->src_sgl[i].vec,\n-\t\t\tvec->src_sgl[i].num);\n-\t\tif (unlikely(data_len < 0))\n-\t\t\tbreak;\n-\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n-\t\tif (unlikely(enqueue_one_chain_job(ctx, req,\n-\t\t\tvec->src_sgl[i].vec, vec->src_sgl[i].num,\n-\t\t\t&vec->iv[i], &vec->digest[i],\n-\t\t\t&vec->auth_iv[i], ofs, (uint32_t)data_len)))\n-\t\t\tbreak;\n-\n-\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\t}\n-\n-\tif (unlikely(i < n))\n-\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue += i;\n-\t*status = 0;\n-\treturn i;\n-}\n-\n-static __rte_always_inline void\n-enqueue_one_aead_job(struct qat_sym_session *ctx,\n-\tstruct icp_qat_fw_la_bulk_req *req,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *aad,\n-\tunion rte_crypto_sym_ofs ofs, uint32_t data_len)\n-{\n-\tstruct icp_qat_fw_la_cipher_req_params *cipher_param =\n-\t\t(void *)&req->serv_specif_rqpars;\n-\tstruct icp_qat_fw_la_auth_req_params *auth_param =\n-\t\t(void *)((uint8_t *)&req->serv_specif_rqpars +\n-\t\tICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);\n-\tuint8_t *aad_data;\n-\tuint8_t aad_ccm_real_len;\n-\tuint8_t aad_len_field_sz;\n-\tuint32_t msg_len_be;\n-\trte_iova_t aad_iova = 0;\n-\tuint8_t q;\n-\n-\t/* CPM 1.7 uses single pass to treat AEAD as cipher operation */\n-\tif (ctx->is_single_pass) {\n-\t\tenqueue_one_cipher_job(ctx, req, iv, ofs, data_len);\n-\n-\t\tif (ctx->is_ucs) {\n-\t\t\t/* QAT GEN4 uses single pass to treat AEAD as cipher\n-\t\t\t * operation\n-\t\t\t */\n-\t\t\tstruct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =\n-\t\t\t\t(void *)&req->serv_specif_rqpars;\n-\t\t\tcipher_param_20->spc_aad_addr = aad->iova;\n-\t\t\tcipher_param_20->spc_auth_res_addr = digest->iova;\n-\t\t} else {\n-\t\t\tcipher_param->spc_aad_addr = aad->iova;\n-\t\t\tcipher_param->spc_auth_res_addr = digest->iova;\n-\t\t}\n-\n-\t\treturn;\n-\t}\n-\n-\tswitch (ctx->qat_hash_alg) {\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_128:\n-\tcase ICP_QAT_HW_AUTH_ALGO_GALOIS_64:\n-\t\tICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(\n-\t\t\treq->comn_hdr.serv_specif_flags,\n-\t\t\t\tICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);\n-\t\trte_memcpy(cipher_param->u.cipher_IV_array, iv->va,\n-\t\t\t\tctx->cipher_iv.length);\n-\t\taad_iova = aad->iova;\n-\t\tbreak;\n-\tcase ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:\n-\t\taad_data = aad->va;\n-\t\taad_iova = aad->iova;\n-\t\taad_ccm_real_len = 0;\n-\t\taad_len_field_sz = 0;\n-\t\tmsg_len_be = rte_bswap32((uint32_t)data_len -\n-\t\t\t\tofs.ofs.cipher.head);\n-\n-\t\tif (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {\n-\t\t\taad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;\n-\t\t\taad_ccm_real_len = ctx->aad_len -\n-\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN -\n-\t\t\t\tICP_QAT_HW_CCM_AAD_LEN_INFO;\n-\t\t} else {\n-\t\t\taad_data = iv->va;\n-\t\t\taad_iova = iv->iova;\n-\t\t}\n-\n-\t\tq = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;\n-\t\taad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(\n-\t\t\taad_len_field_sz, ctx->digest_length, q);\n-\t\tif (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {\n-\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET + (q -\n-\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),\n-\t\t\t\t(uint8_t *)&msg_len_be,\n-\t\t\t\tICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);\n-\t\t} else {\n-\t\t\tmemcpy(aad_data\t+ ctx->cipher_iv.length +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t\t(uint8_t *)&msg_len_be +\n-\t\t\t\t(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE\n-\t\t\t\t- q), q);\n-\t\t}\n-\n-\t\tif (aad_len_field_sz > 0) {\n-\t\t\t*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =\n-\t\t\t\trte_bswap16(aad_ccm_real_len);\n-\n-\t\t\tif ((aad_ccm_real_len + aad_len_field_sz)\n-\t\t\t\t% ICP_QAT_HW_CCM_AAD_B0_LEN) {\n-\t\t\t\tuint8_t pad_len = 0;\n-\t\t\t\tuint8_t pad_idx = 0;\n-\n-\t\t\t\tpad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -\n-\t\t\t\t\t((aad_ccm_real_len +\n-\t\t\t\t\taad_len_field_sz) %\n-\t\t\t\t\tICP_QAT_HW_CCM_AAD_B0_LEN);\n-\t\t\t\tpad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +\n-\t\t\t\t\taad_ccm_real_len +\n-\t\t\t\t\taad_len_field_sz;\n-\t\t\t\tmemset(&aad_data[pad_idx], 0, pad_len);\n-\t\t\t}\n-\t\t}\n-\n-\t\trte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)\n-\t\t\t+ ICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t(uint8_t *)iv->va +\n-\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);\n-\t\t*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =\n-\t\t\tq - ICP_QAT_HW_CCM_NONCE_OFFSET;\n-\n-\t\trte_memcpy((uint8_t *)aad->va +\n-\t\t\t\tICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\t(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,\n-\t\t\tctx->cipher_iv.length);\n-\t\tbreak;\n-\tdefault:\n-\t\tbreak;\n-\t}\n-\n-\tcipher_param->cipher_offset = ofs.ofs.cipher.head;\n-\tcipher_param->cipher_length = data_len - ofs.ofs.cipher.head -\n-\t\t\tofs.ofs.cipher.tail;\n-\tauth_param->auth_off = ofs.ofs.cipher.head;\n-\tauth_param->auth_len = cipher_param->cipher_length;\n-\tauth_param->auth_res_addr = digest->iova;\n-\tauth_param->u1.aad_adr = aad_iova;\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_vec *data, uint16_t n_data_vecs,\n-\tunion rte_crypto_sym_ofs ofs,\n-\tstruct rte_crypto_va_iova_ptr *iv,\n-\tstruct rte_crypto_va_iova_ptr *digest,\n-\tstruct rte_crypto_va_iova_ptr *aad,\n-\tvoid *user_data)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\tuint32_t tail = dp_ctx->tail;\n-\n-\treq = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\trte_prefetch0((uint8_t *)tx_queue->base_addr + tail);\n-\tdata_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);\n-\tif (unlikely(data_len < 0))\n-\t\treturn -1;\n-\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;\n-\n-\tenqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,\n-\t\t(uint32_t)data_len);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue++;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,\n-\tstruct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,\n-\tvoid *user_data[], int *status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_session *ctx = dp_ctx->session;\n-\tuint32_t i, n;\n-\tuint32_t tail;\n-\tstruct icp_qat_fw_la_bulk_req *req;\n-\tint32_t data_len;\n-\n-\tn = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);\n-\tif (unlikely(n == 0)) {\n-\t\tqat_sym_dp_fill_vec_status(vec->status, -1, vec->num);\n-\t\t*status = 0;\n-\t\treturn 0;\n-\t}\n-\n-\ttail = dp_ctx->tail;\n-\n-\tfor (i = 0; i < n; i++) {\n-\t\treq  = (struct icp_qat_fw_la_bulk_req *)(\n-\t\t\t(uint8_t *)tx_queue->base_addr + tail);\n-\t\trte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));\n-\n-\t\tdata_len = qat_sym_dp_parse_data_vec(qp, req,\n-\t\t\tvec->src_sgl[i].vec,\n-\t\t\tvec->src_sgl[i].num);\n-\t\tif (unlikely(data_len < 0))\n-\t\t\tbreak;\n-\t\treq->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];\n-\t\tenqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],\n-\t\t\t&vec->aad[i], ofs, (uint32_t)data_len);\n-\t\ttail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;\n-\t}\n-\n-\tif (unlikely(i < n))\n-\t\tqat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);\n-\n-\tdp_ctx->tail = tail;\n-\tdp_ctx->cached_enqueue += i;\n-\t*status = 0;\n-\treturn i;\n-}\n-\n-static __rte_always_inline uint32_t\n-qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,\n-\trte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,\n-\tuint32_t max_nb_to_dequeue,\n-\trte_cryptodev_raw_post_dequeue_t post_dequeue,\n-\tvoid **out_user_data, uint8_t is_user_data_array,\n-\tuint32_t *n_success_jobs, int *return_status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *rx_queue = &qp->rx_q;\n-\tstruct icp_qat_fw_comn_resp *resp;\n-\tvoid *resp_opaque;\n-\tuint32_t i, n, inflight;\n-\tuint32_t head;\n-\tuint8_t status;\n-\n-\t*n_success_jobs = 0;\n-\t*return_status = 0;\n-\thead = dp_ctx->head;\n-\n-\tinflight = qp->enqueued - qp->dequeued;\n-\tif (unlikely(inflight == 0))\n-\t\treturn 0;\n-\n-\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n-\t\t\thead);\n-\t/* no operation ready */\n-\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n-\t\treturn 0;\n-\n-\tresp_opaque = (void *)(uintptr_t)resp->opaque_data;\n-\t/* get the dequeue count */\n-\tif (get_dequeue_count) {\n-\t\tn = get_dequeue_count(resp_opaque);\n-\t\tif (unlikely(n == 0))\n-\t\t\treturn 0;\n-\t} else {\n-\t\tif (unlikely(max_nb_to_dequeue == 0))\n-\t\t\treturn 0;\n-\t\tn = max_nb_to_dequeue;\n-\t}\n-\n-\tout_user_data[0] = resp_opaque;\n-\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n-\tpost_dequeue(resp_opaque, 0, status);\n-\t*n_success_jobs += status;\n-\n-\thead = (head + rx_queue->msg_size) & rx_queue->modulo_mask;\n-\n-\t/* we already finished dequeue when n == 1 */\n-\tif (unlikely(n == 1)) {\n-\t\ti = 1;\n-\t\tgoto end_deq;\n-\t}\n-\n-\tif (is_user_data_array) {\n-\t\tfor (i = 1; i < n; i++) {\n-\t\t\tresp = (struct icp_qat_fw_comn_resp *)(\n-\t\t\t\t(uint8_t *)rx_queue->base_addr + head);\n-\t\t\tif (unlikely(*(uint32_t *)resp ==\n-\t\t\t\t\tADF_RING_EMPTY_SIG))\n-\t\t\t\tgoto end_deq;\n-\t\t\tout_user_data[i] = (void *)(uintptr_t)resp->opaque_data;\n-\t\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n-\t\t\t*n_success_jobs += status;\n-\t\t\tpost_dequeue(out_user_data[i], i, status);\n-\t\t\thead = (head + rx_queue->msg_size) &\n-\t\t\t\t\trx_queue->modulo_mask;\n-\t\t}\n-\n-\t\tgoto end_deq;\n-\t}\n-\n-\t/* opaque is not array */\n-\tfor (i = 1; i < n; i++) {\n-\t\tresp = (struct icp_qat_fw_comn_resp *)(\n-\t\t\t(uint8_t *)rx_queue->base_addr + head);\n-\t\tstatus = QAT_SYM_DP_IS_RESP_SUCCESS(resp);\n-\t\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n-\t\t\tgoto end_deq;\n-\t\thead = (head + rx_queue->msg_size) &\n-\t\t\t\trx_queue->modulo_mask;\n-\t\tpost_dequeue(resp_opaque, i, status);\n-\t\t*n_success_jobs += status;\n-\t}\n-\n-end_deq:\n-\tdp_ctx->head = head;\n-\tdp_ctx->cached_dequeue += i;\n-\treturn i;\n-}\n-\n-static __rte_always_inline void *\n-qat_sym_dp_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,\n-\t\tenum rte_crypto_op_status *op_status)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\tstruct qat_queue *rx_queue = &qp->rx_q;\n-\tregister struct icp_qat_fw_comn_resp *resp;\n-\n-\tresp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +\n-\t\t\tdp_ctx->head);\n-\n-\tif (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))\n-\t\treturn NULL;\n-\n-\tdp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &\n-\t\t\trx_queue->modulo_mask;\n-\tdp_ctx->cached_dequeue++;\n-\n-\t*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?\n-\t\t\tRTE_CRYPTO_OP_STATUS_SUCCESS :\n-\t\t\tRTE_CRYPTO_OP_STATUS_AUTH_FAILED;\n-\t*dequeue_status = 0;\n-\treturn (void *)(uintptr_t)resp->opaque_data;\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_kick_tail(void *qp_data, uint8_t *drv_ctx, uint32_t n)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_queue *tx_queue = &qp->tx_q;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\n-\tif (unlikely(dp_ctx->cached_enqueue != n))\n-\t\treturn -1;\n-\n-\tqp->enqueued += n;\n-\tqp->stats.enqueued_count += n;\n-\n-\ttx_queue->tail = dp_ctx->tail;\n-\n-\tWRITE_CSR_RING_TAIL(qp->mmap_bar_addr,\n-\t\t\ttx_queue->hw_bundle_number,\n-\t\t\ttx_queue->hw_queue_number, tx_queue->tail);\n-\ttx_queue->csr_tail = tx_queue->tail;\n-\tdp_ctx->cached_enqueue = 0;\n-\n-\treturn 0;\n-}\n-\n-static __rte_always_inline int\n-qat_sym_dp_update_head(void *qp_data, uint8_t *drv_ctx, uint32_t n)\n-{\n-\tstruct qat_qp *qp = qp_data;\n-\tstruct qat_queue *rx_queue = &qp->rx_q;\n-\tstruct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;\n-\n-\tif (unlikely(dp_ctx->cached_dequeue != n))\n-\t\treturn -1;\n-\n-\trx_queue->head = dp_ctx->head;\n-\trx_queue->nb_processed_responses += n;\n-\tqp->dequeued += n;\n-\tqp->stats.dequeued_count += n;\n-\tif (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {\n-\t\tuint32_t old_head, new_head;\n-\t\tuint32_t max_head;\n-\n-\t\told_head = rx_queue->csr_head;\n-\t\tnew_head = rx_queue->head;\n-\t\tmax_head = qp->nb_descriptors * rx_queue->msg_size;\n-\n-\t\t/* write out free descriptors */\n-\t\tvoid *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;\n-\n-\t\tif (new_head < old_head) {\n-\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,\n-\t\t\t\t\tmax_head - old_head);\n-\t\t\tmemset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,\n-\t\t\t\t\tnew_head);\n-\t\t} else {\n-\t\t\tmemset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -\n-\t\t\t\t\told_head);\n-\t\t}\n-\t\trx_queue->nb_processed_responses = 0;\n-\t\trx_queue->csr_head = new_head;\n-\n-\t\t/* write current head to CSR */\n-\t\tWRITE_CSR_RING_HEAD(qp->mmap_bar_addr,\n-\t\t\trx_queue->hw_bundle_number, rx_queue->hw_queue_number,\n-\t\t\tnew_head);\n-\t}\n-\n-\tdp_ctx->cached_dequeue = 0;\n-\treturn 0;\n-}\n-\n-int\n-qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,\n-\tstruct rte_crypto_raw_dp_ctx *raw_dp_ctx,\n-\tenum rte_crypto_op_sess_type sess_type,\n-\tunion rte_cryptodev_session_ctx session_ctx, uint8_t is_update)\n-{\n-\tstruct qat_qp *qp;\n-\tstruct qat_sym_session *ctx;\n-\tstruct qat_sym_dp_ctx *dp_ctx;\n-\n-\tqp = dev->data->queue_pairs[qp_id];\n-\tdp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;\n-\n-\tif (!is_update) {\n-\t\tmemset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +\n-\t\t\t\tsizeof(struct qat_sym_dp_ctx));\n-\t\traw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];\n-\t\tdp_ctx->tail = qp->tx_q.tail;\n-\t\tdp_ctx->head = qp->rx_q.head;\n-\t\tdp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;\n-\t}\n-\n-\tif (sess_type != RTE_CRYPTO_OP_WITH_SESSION)\n-\t\treturn -EINVAL;\n-\n-\tctx = (struct qat_sym_session *)get_sym_session_private_data(\n-\t\t\tsession_ctx.crypto_sess, qat_sym_driver_id);\n-\n-\tdp_ctx->session = ctx;\n-\n-\traw_dp_ctx->enqueue_done = qat_sym_dp_kick_tail;\n-\traw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst;\n-\traw_dp_ctx->dequeue = qat_sym_dp_dequeue;\n-\traw_dp_ctx->dequeue_done = qat_sym_dp_update_head;\n-\n-\tif ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||\n-\t\t\tctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&\n-\t\t\t!ctx->is_gmac) {\n-\t\t/* AES-GCM or AES-CCM */\n-\t\tif (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||\n-\t\t\tctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||\n-\t\t\t(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128\n-\t\t\t&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE\n-\t\t\t&& ctx->qat_hash_alg ==\n-\t\t\t\t\tICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {\n-\t\t\traw_dp_ctx->enqueue_burst =\n-\t\t\t\t\tqat_sym_dp_enqueue_aead_jobs;\n-\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;\n-\t\t} else {\n-\t\t\traw_dp_ctx->enqueue_burst =\n-\t\t\t\t\tqat_sym_dp_enqueue_chain_jobs;\n-\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;\n-\t\t}\n-\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {\n-\t\traw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;\n-\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;\n-\t} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {\n-\t\tif (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||\n-\t\t\tctx->qat_cipher_alg ==\n-\t\t\t\tICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {\n-\t\t\traw_dp_ctx->enqueue_burst =\n-\t\t\t\t\tqat_sym_dp_enqueue_aead_jobs;\n-\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;\n-\t\t} else {\n-\t\t\traw_dp_ctx->enqueue_burst =\n-\t\t\t\t\tqat_sym_dp_enqueue_cipher_jobs;\n-\t\t\traw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;\n-\t\t}\n-\t} else\n-\t\treturn -1;\n-\n-\treturn 0;\n-}\n-\n-int\n-qat_sym_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)\n-{\n-\treturn sizeof(struct qat_sym_dp_ctx);\n-}\n",
    "prefixes": [
        "v9",
        "8/9"
    ]
}