From patchwork Tue Dec 20 14:32:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tejasree Kondoj X-Patchwork-Id: 121083 X-Patchwork-Delegate: gakhil@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 43D10A0545; Tue, 20 Dec 2022 15:33:46 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 720DA410DE; Tue, 20 Dec 2022 15:33:07 +0100 (CET) Received: from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com [67.231.156.173]) by mails.dpdk.org (Postfix) with ESMTP id 3B05D42D40 for ; Tue, 20 Dec 2022 15:33:05 +0100 (CET) Received: from pps.filterd (m0045851.ppops.net [127.0.0.1]) by mx0b-0016f401.pphosted.com (8.17.1.19/8.17.1.19) with ESMTP id 2BKEJKMV010037 for ; Tue, 20 Dec 2022 06:33:04 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : in-reply-to : references : mime-version : content-transfer-encoding : content-type; s=pfpt0220; bh=2jlKhFo6ZMf6TGQybe0UjNwAYA79BsjUNwc02EOKR+8=; b=GOqhrK1r6Ot2LN8Z5VLVyn5Tv7ahpszDYt4ufRjIZ1FfSpJEBuGQOZEUVPddFy2WgDlZ mwytVGxh98givB2LOHmYAn/Lr4S1+vWo1k3Sw+LdmpQbulifGBdq9/3hkJpT9tQ4AuSJ BuUc6n69IDwPBqmKLZUyf3Bnm/8rIOtU1pLoMrqdWYRcrS0k3VmwFyhzuo5Arzp21279 ISTbX8/CYtAxbsdRysJikzyDYWfBXdnivaFuWVmDUGYtxHc4YSuHk5I7uuCvXXw0a+36 0esdzksAD21vhdEx2VZA3Z0SEayLVlfgd/OymL2o/yYj4RHyrDHF8hK0aruuAqPDm4gN kg== Received: from dc5-exch02.marvell.com ([199.233.59.182]) by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3mhe5rnb5k-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT) for ; Tue, 20 Dec 2022 06:33:03 -0800 Received: from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server (TLS) id 15.0.1497.42; Tue, 20 Dec 2022 06:33:01 -0800 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.42 via Frontend Transport; Tue, 20 Dec 2022 06:33:01 -0800 Received: from hyd1554.marvell.com (unknown [10.29.57.11]) by maili.marvell.com (Postfix) with ESMTP id A0B0C3F704C; Tue, 20 Dec 2022 06:32:58 -0800 (PST) From: Tejasree Kondoj To: Akhil Goyal CC: Archana Muniganti , Anoob Joseph , Vidya Sagar Velumuri , Gowrishankar Muthukrishnan , Volodymyr Fialko , Aakash Sasidharan , Subject: [PATCH 09/17] crypto/cnxk: add CN9K IPsec SG support Date: Tue, 20 Dec 2022 20:02:24 +0530 Message-ID: <20221220143232.2519650-10-ktejasree@marvell.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20221220143232.2519650-1-ktejasree@marvell.com> References: <20221220143232.2519650-1-ktejasree@marvell.com> MIME-Version: 1.0 X-Proofpoint-GUID: nYsS0xngQnS6tZsRe6Vi0Kop6VKFtWqK X-Proofpoint-ORIG-GUID: nYsS0xngQnS6tZsRe6Vi0Kop6VKFtWqK X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.205,Aquarius:18.0.923,Hydra:6.0.545,FMLib:17.11.122.1 definitions=2022-12-20_05,2022-12-20_01,2022-06-22_01 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Archana Muniganti Added IPsec SG support in CN9K Signed-off-by: Archana Muniganti Signed-off-by: Tejasree Kondoj --- drivers/common/cnxk/roc_api.h | 1 + drivers/common/cnxk/roc_cpt_sg.h | 37 +++ drivers/common/cnxk/roc_ie_on.h | 9 +- drivers/common/cnxk/roc_se.h | 28 -- drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 75 ++--- drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 171 ++++++++-- drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 25 +- drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 19 ++ drivers/crypto/cnxk/cnxk_se.h | 387 +++-------------------- drivers/crypto/cnxk/cnxk_sg.h | 273 ++++++++++++++++ 10 files changed, 594 insertions(+), 431 deletions(-) create mode 100644 drivers/common/cnxk/roc_cpt_sg.h create mode 100644 drivers/crypto/cnxk/cnxk_sg.h diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h index 072f16d77d..14a11321e0 100644 --- a/drivers/common/cnxk/roc_api.h +++ b/drivers/common/cnxk/roc_api.h @@ -86,6 +86,7 @@ /* CPT microcode */ #include "roc_ae.h" #include "roc_ae_fpm_tables.h" +#include "roc_cpt_sg.h" #include "roc_ie.h" #include "roc_ie_on.h" #include "roc_ie_ot.h" diff --git a/drivers/common/cnxk/roc_cpt_sg.h b/drivers/common/cnxk/roc_cpt_sg.h new file mode 100644 index 0000000000..8a97e1aa5b --- /dev/null +++ b/drivers/common/cnxk/roc_cpt_sg.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2022 Marvell. + */ + +#ifndef _ROC_CPT_SG_H_ +#define _ROC_CPT_SG_H_ + +#define ROC_DMA_MODE_SG (1 << 7) + +#define ROC_MAX_SG_IN_OUT_CNT 32 +#define ROC_MAX_SG_CNT (ROC_MAX_SG_IN_OUT_CNT / 2) + +#define ROC_SG_LIST_HDR_SIZE (8u) +#define ROC_SG_ENTRY_SIZE sizeof(struct roc_sglist_comp) + +struct roc_sglist_comp { + union { + uint64_t len; + struct { + uint16_t len[4]; + } s; + } u; + uint64_t ptr[4]; +}; + +struct roc_sg2list_comp { + union { + uint64_t len; + struct { + uint16_t len[3]; + uint16_t valid_segs; + } s; + } u; + uint64_t ptr[3]; +}; + +#endif /* _ROC_CPT_SG_H_ */ diff --git a/drivers/common/cnxk/roc_ie_on.h b/drivers/common/cnxk/roc_ie_on.h index 057ff95362..9933ffa148 100644 --- a/drivers/common/cnxk/roc_ie_on.h +++ b/drivers/common/cnxk/roc_ie_on.h @@ -25,10 +25,11 @@ enum roc_ie_on_ucc_ipsec { }; /* Helper macros */ -#define ROC_IE_ON_INB_RPTR_HDR 16 -#define ROC_IE_ON_MAX_IV_LEN 16 -#define ROC_IE_ON_PER_PKT_IV BIT(43) -#define ROC_IE_ON_INPLACE_BIT BIT(6) +#define ROC_IE_ON_OUTB_DPTR_HDR 16 +#define ROC_IE_ON_INB_RPTR_HDR 16 +#define ROC_IE_ON_MAX_IV_LEN 16 +#define ROC_IE_ON_PER_PKT_IV BIT(43) +#define ROC_IE_ON_INPLACE_BIT BIT(6) enum { ROC_IE_ON_SA_ENC_NULL = 0, diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h index c357c19c0b..a8f0f49479 100644 --- a/drivers/common/cnxk/roc_se.h +++ b/drivers/common/cnxk/roc_se.h @@ -27,13 +27,6 @@ #define ROC_SE_MAX_MAC_LEN 64 #define ROC_SE_OFF_CTRL_LEN 8 -#define ROC_SE_DMA_MODE (1 << 7) - -#define ROC_SE_MAX_SG_IN_OUT_CNT 32 -#define ROC_SE_MAX_SG_CNT (ROC_SE_MAX_SG_IN_OUT_CNT / 2) - -#define ROC_SE_SG_LIST_HDR_SIZE (8u) -#define ROC_SE_SG_ENTRY_SIZE sizeof(struct roc_se_sglist_comp) #define ROC_SE_ZS_EA 0x1 #define ROC_SE_ZS_IA 0x2 @@ -173,27 +166,6 @@ typedef enum { ROC_SE_PDCP_MAC_LEN_128_BIT = 0x3 } roc_se_pdcp_mac_len_type; -struct roc_se_sglist_comp { - union { - uint64_t len; - struct { - uint16_t len[4]; - } s; - } u; - uint64_t ptr[4]; -}; - -struct roc_se_sg2list_comp { - union { - uint64_t len; - struct { - uint16_t len[3]; - uint16_t valid_segs; - } s; - } u; - uint64_t ptr[3]; -}; - struct roc_se_enc_context { uint64_t iv_source : 1; uint64_t aes_key : 2; diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c index 04c004bc7a..cfe1e08dff 100644 --- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c +++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c @@ -18,13 +18,11 @@ #include "cnxk_se.h" static __rte_always_inline int __rte_hot -cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, - struct cpt_inflight_req *infl_req, - struct cpt_inst_s *inst) +cn9k_cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, + struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst) { struct rte_crypto_sym_op *sym_op = op->sym; struct cn9k_sec_session *sec_sess; - int ret; sec_sess = (struct cn9k_sec_session *)(op->sym->session); @@ -33,22 +31,10 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, return -ENOTSUP; } - if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) { - plt_dp_err("Scatter Gather mode is not supported"); - return -ENOTSUP; - } - if (sec_sess->is_outbound) - ret = process_outb_sa(op, sec_sess, inst); - else { - infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND; - process_inb_sa(op, sec_sess, inst); - if (unlikely(sec_sess->replay_win_sz)) - infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY; - ret = 0; - } - - return ret; + return process_outb_sa(&qp->meta_info, op, sec_sess, inst, infl_req); + else + return process_inb_sa(&qp->meta_info, op, sec_sess, inst, infl_req); } static inline struct cnxk_se_sess * @@ -94,7 +80,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false); inst->w7.u64 = sess->cpt_inst_w7; } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) - ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst); + ret = cn9k_cpt_sec_inst_fill(qp, op, infl_req, inst); else { sess = cn9k_cpt_sym_temp_sess_create(qp, op); if (unlikely(sess == NULL)) { @@ -522,45 +508,60 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop, { struct rte_crypto_sym_op *sym_op = cop->sym; struct rte_mbuf *m = sym_op->m_src; + struct roc_ie_on_inb_hdr *hdr; struct cn9k_sec_session *priv; struct rte_ipv6_hdr *ip6; struct rte_ipv4_hdr *ip; uint16_t m_len = 0; - char *data; if (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) { - data = rte_pktmbuf_mtod(m, char *); - if (unlikely(infl_req->op_flags & - CPT_OP_FLAGS_IPSEC_INB_REPLAY)) { + hdr = (struct roc_ie_on_inb_hdr *)rte_pktmbuf_mtod(m, char *); + + if (likely(m->next == NULL)) { + ip = PLT_PTR_ADD(hdr, ROC_IE_ON_INB_RPTR_HDR); + } else { + ip = (struct rte_ipv4_hdr *)hdr; + hdr = infl_req->mdata; + } + + if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_IPSEC_INB_REPLAY)) { int ret; priv = (struct cn9k_sec_session *)(sym_op->session); - ret = ipsec_antireplay_check(priv, priv->replay_win_sz, - (struct roc_ie_on_inb_hdr *)data); + ret = ipsec_antireplay_check(priv, priv->replay_win_sz, hdr); if (unlikely(ret)) { cop->status = RTE_CRYPTO_OP_STATUS_ERROR; return; } } - ip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR); - - if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == - IPVERSION) { + if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == IPVERSION) { m_len = rte_be_to_cpu_16(ip->total_length); } else { - PLT_ASSERT(((ip->version_ihl & 0xf0) >> - RTE_IPV4_IHL_MULTIPLIER) == 6); + PLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6); ip6 = (struct rte_ipv6_hdr *)ip; - m_len = rte_be_to_cpu_16(ip6->payload_len) + - sizeof(struct rte_ipv6_hdr); + m_len = rte_be_to_cpu_16(ip6->payload_len) + sizeof(struct rte_ipv6_hdr); } - m->data_len = m_len; - m->pkt_len = m_len; - m->data_off += ROC_IE_ON_INB_RPTR_HDR; + if (likely(m->next == NULL)) { + m->data_len = m_len; + m->pkt_len = m_len; + + m->data_off += ROC_IE_ON_INB_RPTR_HDR; + } else { + struct rte_mbuf *temp = m; + uint8_t m_len_s = m_len; + + while (m_len_s - temp->data_len > 0) { + m_len_s -= temp->data_len; + temp = temp->next; + } + + temp->data_len = m_len_s; + m->pkt_len = m_len; + } } } diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h index f1298017ce..3d9c851f10 100644 --- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h +++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h @@ -10,7 +10,9 @@ #include #include "cn9k_ipsec.h" +#include "cnxk_cryptodev_ops.h" #include "cnxk_security_ar.h" +#include "cnxk_sg.h" static __rte_always_inline int32_t ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_mbuf *m_src) @@ -58,7 +60,9 @@ ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_m } static __rte_always_inline int -process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst) +process_outb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop, + struct cn9k_sec_session *sess, struct cpt_inst_s *inst, + struct cpt_inflight_req *infl_req) { const unsigned int hdr_len = sess->custom_hdr_len; struct rte_crypto_sym_op *sym_op = cop->sym; @@ -74,24 +78,90 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct rlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src); extend_tail = rlen - dlen; - if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) { - plt_dp_err("Not enough tail room (required: %d, available: %d)", - extend_tail, rte_pktmbuf_tailroom(m_src)); - return -ENOMEM; - } + pkt_len += extend_tail; - if (unlikely(hdr_len > data_off)) { - plt_dp_err("Not enough head room (required: %d, available: %d)", - hdr_len, rte_pktmbuf_headroom(m_src)); - return -ENOMEM; - } + if (likely(m_src->next == NULL)) { + if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) { + plt_dp_err("Not enough tail room (required: %d, available: %d)", + extend_tail, rte_pktmbuf_tailroom(m_src)); + return -ENOMEM; + } - pkt_len += extend_tail; + if (unlikely(hdr_len > data_off)) { + plt_dp_err("Not enough head room (required: %d, available: %d)", hdr_len, + rte_pktmbuf_headroom(m_src)); + return -ENOMEM; + } - m_src->data_len = pkt_len; - m_src->pkt_len = pkt_len; + m_src->data_len = pkt_len; + + hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len); + + inst->dptr = PLT_U64_CAST(hdr); + inst->w4.u64 = sess->inst.w4 | dlen; + } else { + struct roc_sglist_comp *scatter_comp, *gather_comp; + uint32_t g_size_bytes, s_size_bytes; + struct rte_mbuf *last_seg; + uint8_t *in_buffer; + void *m_data; + int i; + + last_seg = rte_pktmbuf_lastseg(m_src); + + if (unlikely(extend_tail > rte_pktmbuf_tailroom(last_seg))) { + plt_dp_err("Not enough tail room (required: %d, available: %d)", + extend_tail, rte_pktmbuf_tailroom(last_seg)); + return -ENOMEM; + } + + m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req); + if (unlikely(m_data == NULL)) { + plt_dp_err("Error allocating meta buffer for request"); + return -ENOMEM; + } + + hdr = m_data; + + m_data = (uint8_t *)m_data + hdr_len; + in_buffer = m_data; + + ((uint16_t *)in_buffer)[0] = 0; + ((uint16_t *)in_buffer)[1] = 0; + + /* + * Input Gather List + */ + i = 0; + gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8); + + i = fill_sg_comp(gather_comp, i, (uint64_t)hdr, hdr_len); + i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src); + ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); + + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); + + /* + * output Scatter List + */ + last_seg->data_len += extend_tail; + + i = 0; + scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); + + i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len); + i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src); + ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); - hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len); + s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); + + dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; + + inst->dptr = (uint64_t)in_buffer; + + inst->w4.u64 = sess->inst.w4 | dlen; + inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG; + } #ifdef LA_IPSEC_DEBUG if (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) { @@ -101,6 +171,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct } #endif + m_src->pkt_len = pkt_len; esn = ++sess->esn; /* Set ESN seq hi */ @@ -114,22 +185,80 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct hdr->ip_id = seq_lo; /* Prepare CPT instruction */ - inst->w4.u64 = sess->inst.w4 | dlen; - inst->dptr = PLT_U64_CAST(hdr); inst->w7.u64 = sess->inst.w7; return 0; } -static __rte_always_inline void -process_inb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst) +static __rte_always_inline int +process_inb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req) { + const unsigned int hdr_len = ROC_IE_ON_INB_RPTR_HDR; struct rte_crypto_sym_op *sym_op = cop->sym; struct rte_mbuf *m_src = sym_op->m_src; + struct roc_ie_on_inb_hdr *hdr; + uint32_t dlen; + + infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND; + if (likely(m_src->next == NULL)) { + dlen = rte_pktmbuf_pkt_len(m_src); + inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t); + inst->w4.u64 = sess->inst.w4 | dlen; + } else { + struct roc_sglist_comp *scatter_comp, *gather_comp; + uint32_t g_size_bytes, s_size_bytes; + uint8_t *in_buffer; + void *m_data; + int i; + + m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req); + if (unlikely(m_data == NULL)) { + plt_dp_err("Error allocating meta buffer for request"); + return -ENOMEM; + } + + hdr = m_data; + + m_data = (uint8_t *)m_data + hdr_len; + in_buffer = m_data; + + ((uint16_t *)in_buffer)[0] = 0; + ((uint16_t *)in_buffer)[1] = 0; + + /* + * Input Gather List + */ + i = 0; + gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8); + i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src); + ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); + + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); + + /* + * Output Scatter List + */ + i = 0; + scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); + i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len); + i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src); + ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); + + s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); + + dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; + + inst->dptr = (uint64_t)in_buffer; + inst->w4.u64 = sess->inst.w4 | dlen; + inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG; + } /* Prepare CPT instruction */ - inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src); - inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t); inst->w7.u64 = sess->inst.w7; + + if (unlikely(sess->replay_win_sz)) + infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY; + + return 0; } #endif /* __CN9K_IPSEC_LA_OPS_H__ */ diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c index a9c42205e6..eb2ed0d103 100644 --- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c +++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c @@ -30,10 +30,22 @@ cnxk_cpt_get_mlen(void) /* For PDCP_CHAIN passthrough alignment */ len += 8; len += ROC_SE_OFF_CTRL_LEN + ROC_CPT_AES_CBC_IV_LEN; - len += RTE_ALIGN_CEIL( - (ROC_SE_SG_LIST_HDR_SIZE + - (RTE_ALIGN_CEIL(ROC_SE_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SE_SG_ENTRY_SIZE), - 8); + len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE + + (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE), + 8); + + return len; +} + +static int +cnxk_cpt_sec_get_mlen(void) +{ + uint32_t len; + + len = ROC_IE_ON_OUTB_DPTR_HDR + ROC_IE_ON_MAX_IV_LEN; + len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE + + (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE), + 8); return len; } @@ -196,6 +208,11 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev, mlen = cnxk_cpt_get_mlen(); } + if (dev->feature_flags & RTE_CRYPTODEV_FF_SECURITY) { + /* Get meta len for security operations */ + mlen = cnxk_cpt_sec_get_mlen(); + } + if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) { /* Get meta len required for asymmetric operations */ diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h index 13c90444d6..5153d334ba 100644 --- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h +++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h @@ -158,4 +158,23 @@ pending_queue_free_cnt(uint64_t head, uint64_t tail, const uint64_t mask) return mask - pending_queue_infl_cnt(head, tail, mask); } +static __rte_always_inline void * +alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len, struct rte_mempool *cpt_meta_pool, + struct cpt_inflight_req *infl_req) +{ + uint8_t *mdata; + + if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) + return NULL; + + if (likely(buf)) { + buf->vaddr = mdata; + buf->size = len; + } + + infl_req->mdata = mdata; + infl_req->op_flags |= CPT_OP_FLAGS_METABUF; + + return mdata; +} #endif /* _CNXK_CRYPTODEV_OPS_H_ */ diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h index 2944d3c287..88049ac431 100644 --- a/drivers/crypto/cnxk/cnxk_se.h +++ b/drivers/crypto/cnxk/cnxk_se.h @@ -8,11 +8,12 @@ #include "cnxk_cryptodev.h" #include "cnxk_cryptodev_ops.h" +#include "cnxk_sg.h" #define SRC_IOV_SIZE \ - (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT)) + (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT)) #define DST_IOV_SIZE \ - (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT)) + (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT)) enum cpt_dp_thread_type { CPT_DP_THREAD_TYPE_FC_CHAIN = 0x1, @@ -193,272 +194,14 @@ cpt_fc_salt_update(struct roc_se_ctx *se_ctx, uint8_t *salt) memcpy(fctx->enc.encr_iv, salt, 4); } -static __rte_always_inline uint32_t -fill_sg_comp(struct roc_se_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, - uint32_t size) -{ - struct roc_se_sglist_comp *to = &list[i >> 2]; - - to->u.s.len[i % 4] = rte_cpu_to_be_16(size); - to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr); - i++; - return i; -} - -static __rte_always_inline uint32_t -fill_sg_comp_from_buf(struct roc_se_sglist_comp *list, uint32_t i, - struct roc_se_buf_ptr *from) -{ - struct roc_se_sglist_comp *to = &list[i >> 2]; - - to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size); - to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr); - i++; - return i; -} - -static __rte_always_inline uint32_t -fill_sg_comp_from_buf_min(struct roc_se_sglist_comp *list, uint32_t i, - struct roc_se_buf_ptr *from, uint32_t *psize) -{ - struct roc_se_sglist_comp *to = &list[i >> 2]; - uint32_t size = *psize; - uint32_t e_len; - - e_len = (size > from->size) ? from->size : size; - to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); - to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr); - *psize -= e_len; - i++; - return i; -} - -/* - * This fills the MC expected SGIO list - * from IOV given by user. - */ -static __rte_always_inline uint32_t -fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i, - struct roc_se_iov_ptr *from, uint32_t from_offset, - uint32_t *psize, struct roc_se_buf_ptr *extra_buf, - uint32_t extra_offset) -{ - int32_t j; - uint32_t extra_len = extra_buf ? extra_buf->size : 0; - uint32_t size = *psize; - - for (j = 0; j < from->buf_cnt; j++) { - struct roc_se_sglist_comp *to = &list[i >> 2]; - uint32_t buf_sz = from->bufs[j].size; - void *vaddr = from->bufs[j].vaddr; - uint64_t e_vaddr; - uint32_t e_len; - - if (unlikely(from_offset)) { - if (from_offset >= buf_sz) { - from_offset -= buf_sz; - continue; - } - e_vaddr = (uint64_t)vaddr + from_offset; - e_len = (size > (buf_sz - from_offset)) ? - (buf_sz - from_offset) : - size; - from_offset = 0; - } else { - e_vaddr = (uint64_t)vaddr; - e_len = (size > buf_sz) ? buf_sz : size; - } - - to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); - to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr); - - if (extra_len && (e_len >= extra_offset)) { - /* Break the data at given offset */ - uint32_t next_len = e_len - extra_offset; - uint64_t next_vaddr = e_vaddr + extra_offset; - - if (!extra_offset) { - i--; - } else { - e_len = extra_offset; - size -= e_len; - to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); - } - - extra_len = RTE_MIN(extra_len, size); - /* Insert extra data ptr */ - if (extra_len) { - i++; - to = &list[i >> 2]; - to->u.s.len[i % 4] = - rte_cpu_to_be_16(extra_len); - to->ptr[i % 4] = rte_cpu_to_be_64( - (uint64_t)extra_buf->vaddr); - size -= extra_len; - } - - next_len = RTE_MIN(next_len, size); - /* insert the rest of the data */ - if (next_len) { - i++; - to = &list[i >> 2]; - to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); - to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr); - size -= next_len; - } - extra_len = 0; - - } else { - size -= e_len; - } - if (extra_offset) - extra_offset -= size; - i++; - - if (unlikely(!size)) - break; - } - - *psize = size; - return (uint32_t)i; -} - -static __rte_always_inline uint32_t -fill_sg2_comp(struct roc_se_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size) -{ - struct roc_se_sg2list_comp *to = &list[i / 3]; - - to->u.s.len[i % 3] = (size); - to->ptr[i % 3] = (dma_addr); - to->u.s.valid_segs = (i % 3) + 1; - i++; - return i; -} - -static __rte_always_inline uint32_t -fill_sg2_comp_from_buf(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from) -{ - struct roc_se_sg2list_comp *to = &list[i / 3]; - - to->u.s.len[i % 3] = (from->size); - to->ptr[i % 3] = ((uint64_t)from->vaddr); - to->u.s.valid_segs = (i % 3) + 1; - i++; - return i; -} - -static __rte_always_inline uint32_t -fill_sg2_comp_from_buf_min(struct roc_se_sg2list_comp *list, uint32_t i, - struct roc_se_buf_ptr *from, uint32_t *psize) -{ - struct roc_se_sg2list_comp *to = &list[i / 3]; - uint32_t size = *psize; - uint32_t e_len; - - e_len = (size > from->size) ? from->size : size; - to->u.s.len[i % 3] = (e_len); - to->ptr[i % 3] = ((uint64_t)from->vaddr); - to->u.s.valid_segs = (i % 3) + 1; - *psize -= e_len; - i++; - return i; -} - -static __rte_always_inline uint32_t -fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from, - uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf, - uint32_t extra_offset) -{ - int32_t j; - uint32_t extra_len = extra_buf ? extra_buf->size : 0; - uint32_t size = *psize; - - rte_prefetch2(psize); - - for (j = 0; j < from->buf_cnt; j++) { - struct roc_se_sg2list_comp *to = &list[i / 3]; - uint32_t buf_sz = from->bufs[j].size; - void *vaddr = from->bufs[j].vaddr; - uint64_t e_vaddr; - uint32_t e_len; - - if (unlikely(from_offset)) { - if (from_offset >= buf_sz) { - from_offset -= buf_sz; - continue; - } - e_vaddr = (uint64_t)vaddr + from_offset; - e_len = (size > (buf_sz - from_offset)) ? (buf_sz - from_offset) : size; - from_offset = 0; - } else { - e_vaddr = (uint64_t)vaddr; - e_len = (size > buf_sz) ? buf_sz : size; - } - - to->u.s.len[i % 3] = (e_len); - to->ptr[i % 3] = (e_vaddr); - to->u.s.valid_segs = (i % 3) + 1; - - if (extra_len && (e_len >= extra_offset)) { - /* Break the data at given offset */ - uint32_t next_len = e_len - extra_offset; - uint64_t next_vaddr = e_vaddr + extra_offset; - - if (!extra_offset) { - i--; - } else { - e_len = extra_offset; - size -= e_len; - to->u.s.len[i % 3] = (e_len); - } - - extra_len = RTE_MIN(extra_len, size); - /* Insert extra data ptr */ - if (extra_len) { - i++; - to = &list[i / 3]; - to->u.s.len[i % 3] = (extra_len); - to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr); - to->u.s.valid_segs = (i % 3) + 1; - size -= extra_len; - } - - next_len = RTE_MIN(next_len, size); - /* insert the rest of the data */ - if (next_len) { - i++; - to = &list[i / 3]; - to->u.s.len[i % 3] = (next_len); - to->ptr[i % 3] = (next_vaddr); - to->u.s.valid_segs = (i % 3) + 1; - size -= next_len; - } - extra_len = 0; - - } else { - size -= e_len; - } - if (extra_offset) - extra_offset -= size; - i++; - - if (unlikely(!size)) - break; - } - - *psize = size; - return (uint32_t)i; -} - static __rte_always_inline int sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t offset_ctrl, uint8_t *iv_s, int iv_len, uint8_t pack_iv, uint8_t pdcp_alg_type, int32_t inputlen, int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag, int decrypt) { + struct roc_sglist_comp *gather_comp, *scatter_comp; void *m_vaddr = params->meta_buf.vaddr; - struct roc_se_sglist_comp *gather_comp; - struct roc_se_sglist_comp *scatter_comp; struct roc_se_buf_ptr *aad_buf = NULL; uint32_t mac_len = 0, aad_len = 0; struct roc_se_ctx *se_ctx; @@ -485,7 +228,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8); - inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE; + inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG; /* iv offset is 0 */ *offset_vaddr = offset_ctrl; @@ -503,7 +246,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t /* DPTR has SG list */ /* TODO Add error check if space will be sufficient */ - gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8); + gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8); /* * Input Gather List @@ -562,13 +305,13 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t ((uint16_t *)in_buffer)[1] = 0; ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); - g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); /* * Output Scatter List */ i = 0; - scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); + scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); if (zsk_flags == 0x1) { /* IV in SLIST only for EEA3 & UEA2 or for F8 */ @@ -626,9 +369,9 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t } } ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); - s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); - size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE; + size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; /* This is DPTR len in case of SG mode */ inst->w4.s.dlen = size; @@ -643,18 +386,17 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag, int decrypt) { + struct roc_sg2list_comp *gather_comp, *scatter_comp; void *m_vaddr = params->meta_buf.vaddr; - uint32_t i, g_size_bytes; - struct roc_se_sg2list_comp *gather_comp; - struct roc_se_sg2list_comp *scatter_comp; struct roc_se_buf_ptr *aad_buf = NULL; + uint32_t mac_len = 0, aad_len = 0; + union cpt_inst_w5 cpt_inst_w5; + union cpt_inst_w6 cpt_inst_w6; struct roc_se_ctx *se_ctx; + uint32_t i, g_size_bytes; uint64_t *offset_vaddr; - uint32_t mac_len = 0, aad_len = 0; int zsk_flags; uint32_t size; - union cpt_inst_w5 cpt_inst_w5; - union cpt_inst_w6 cpt_inst_w6; uint8_t *iv_d; se_ctx = params->ctx; @@ -672,7 +414,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8); - inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE; + inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG; /* iv offset is 0 */ *offset_vaddr = offset_ctrl; @@ -689,7 +431,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t /* DPTR has SG list */ /* TODO Add error check if space will be sufficient */ - gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr); + gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr); /* * Input Gather List @@ -746,13 +488,13 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t cpt_inst_w5.s.gather_sz = ((i + 2) / 3); - g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp); + g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp); /* * Output Scatter List */ i = 0; - scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes); + scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes); if (zsk_flags == 0x1) { /* IV in SLIST only for EEA3 & UEA2 or for F8 */ @@ -829,16 +571,15 @@ static __rte_always_inline int cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params, struct cpt_inst_s *inst) { + struct roc_sglist_comp *gather_comp, *scatter_comp; void *m_vaddr = params->meta_buf.vaddr; - uint32_t size, i; + uint32_t g_size_bytes, s_size_bytes; uint16_t data_len, mac_len, key_len; + union cpt_inst_w4 cpt_inst_w4; roc_se_auth_type hash_type; struct roc_se_ctx *ctx; - struct roc_se_sglist_comp *gather_comp; - struct roc_se_sglist_comp *scatter_comp; uint8_t *in_buffer; - uint32_t g_size_bytes, s_size_bytes; - union cpt_inst_w4 cpt_inst_w4; + uint32_t size, i; ctx = params->ctx; @@ -851,13 +592,11 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa cpt_inst_w4.s.opcode_minor = 0; cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len; if (ctx->hmac) { - cpt_inst_w4.s.opcode_major = - ROC_SE_MAJOR_OP_HMAC | ROC_SE_DMA_MODE; + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HMAC | ROC_DMA_MODE_SG; cpt_inst_w4.s.param1 = key_len; cpt_inst_w4.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); } else { - cpt_inst_w4.s.opcode_major = - ROC_SE_MAJOR_OP_HASH | ROC_SE_DMA_MODE; + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HASH | ROC_DMA_MODE_SG; cpt_inst_w4.s.param1 = 0; cpt_inst_w4.s.dlen = data_len; } @@ -878,7 +617,7 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa ((uint16_t *)in_buffer)[1] = 0; /* TODO Add error check if space will be sufficient */ - gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8); + gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8); /* * Input gather list @@ -901,15 +640,14 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa return -1; } ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); - g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); /* * Output Gather list */ i = 0; - scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + - g_size_bytes); + scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); if (flags & ROC_SE_VALID_MAC_BUF) { if (unlikely(params->mac_buf.size < mac_len)) { @@ -932,9 +670,9 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa } ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); - s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); - size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE; + size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; /* This is DPTR len in case of SG mode */ cpt_inst_w4.s.dlen = size; @@ -949,17 +687,16 @@ static __rte_always_inline int cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params, struct cpt_inst_s *inst) { + struct roc_sg2list_comp *gather_comp, *scatter_comp; void *m_vaddr = params->meta_buf.vaddr; - uint32_t size, i; uint16_t data_len, mac_len, key_len; - roc_se_auth_type hash_type; - struct roc_se_ctx *ctx; - struct roc_se_sg2list_comp *gather_comp; - struct roc_se_sg2list_comp *scatter_comp; + union cpt_inst_w4 cpt_inst_w4; union cpt_inst_w5 cpt_inst_w5; union cpt_inst_w6 cpt_inst_w6; + roc_se_auth_type hash_type; + struct roc_se_ctx *ctx; uint32_t g_size_bytes; - union cpt_inst_w4 cpt_inst_w4; + uint32_t size, i; ctx = params->ctx; @@ -993,7 +730,7 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa /* DPTR has SG list */ /* TODO Add error check if space will be sufficient */ - gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr + 0); + gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr + 0); /* * Input gather list @@ -1016,14 +753,14 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa } cpt_inst_w5.s.gather_sz = ((i + 2) / 3); - g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp); + g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp); /* * Output Gather list */ i = 0; - scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes); + scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes); if (flags & ROC_SE_VALID_MAC_BUF) { if (unlikely(params->mac_buf.size < mac_len)) { @@ -1482,8 +1219,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, pdcp_iv_copy(iv_d, auth_iv, pdcp_auth_alg, pack_iv); } else { - - struct roc_se_sglist_comp *scatter_comp, *gather_comp; + struct roc_sglist_comp *scatter_comp, *gather_comp; void *m_vaddr = params->meta_buf.vaddr; uint32_t i, g_size_bytes, s_size_bytes; uint8_t *in_buffer; @@ -1494,7 +1230,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, m_vaddr = PLT_PTR_ADD(m_vaddr, ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(hdr_len, 8)); - cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE; + cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG; /* DPTR has SG list */ in_buffer = m_vaddr; @@ -1502,8 +1238,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, ((uint16_t *)in_buffer)[0] = 0; ((uint16_t *)in_buffer)[1] = 0; - gather_comp = - (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8); + gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8); /* Input Gather List */ i = 0; @@ -1536,15 +1271,14 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, } } ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); - g_size_bytes = - ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); /* * Output Scatter List */ i = 0; - scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); + scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes); if ((hdr_len)) { i = fill_sg_comp(scatter_comp, i, @@ -1573,10 +1307,9 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, } ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); - s_size_bytes = - ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); + s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp); - size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE; + size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; /* This is DPTR len in case of SG mode */ cpt_inst_w4.s.dlen = size; @@ -1800,7 +1533,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, } } - cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE; + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG; /* Indicate ECB/CBC, direction, CTX from CPTR, IV from DPTR */ cpt_inst_w4.s.opcode_minor = @@ -1841,11 +1574,11 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens, struct roc_se_fc_params *p flags = se_ctx->zsk_flags; cpt_inst_w4.u64 = 0; - cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE; + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG; /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ - cpt_inst_w4.s.opcode_minor = ((1 << 6) | (se_ctx->k_ecb << 5) | - (dir << 4) | (0 << 3) | (flags & 0x7)); + cpt_inst_w4.s.opcode_minor = + ((1 << 6) | (se_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) | (flags & 0x7)); /* * GP op header, lengths are expected in bits. @@ -2290,28 +2023,8 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess) return 0; } -static __rte_always_inline void * -alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len, - struct rte_mempool *cpt_meta_pool, - struct cpt_inflight_req *infl_req) -{ - uint8_t *mdata; - - if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) - return NULL; - - buf->vaddr = mdata; - buf->size = len; - - infl_req->mdata = mdata; - infl_req->op_flags |= CPT_OP_FLAGS_METABUF; - - return mdata; -} - static __rte_always_inline uint32_t -prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec, - uint32_t start_offset) +prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec, uint32_t start_offset) { uint16_t index = 0; void *seg_data = NULL; diff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h new file mode 100644 index 0000000000..1dfca261cf --- /dev/null +++ b/drivers/crypto/cnxk/cnxk_sg.h @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2022 Marvell. + */ + +#ifndef _CNXK_SG_H_ +#define _CNXK_SG_H_ + +static __rte_always_inline uint32_t +fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = rte_cpu_to_be_16(size); + to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr); + return ++i; +} + +static __rte_always_inline uint32_t +fill_sg_comp_from_buf(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size); + to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr); + return ++i; +} + +static __rte_always_inline uint32_t +fill_sg_comp_from_buf_min(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from, + uint32_t *psize) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + uint32_t size = *psize; + uint32_t e_len; + + e_len = RTE_MIN(from->size, size); + to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); + to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr); + *psize -= e_len; + return ++i; +} + +/* + * This fills the MC expected SGIO list + * from IOV given by user. + */ +static __rte_always_inline uint32_t +fill_sg_comp_from_iov(struct roc_sglist_comp *list, uint32_t i, struct roc_se_iov_ptr *from, + uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf, + uint32_t extra_offset) +{ + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + int32_t j; + + for (j = 0; j < from->buf_cnt; j++) { + struct roc_sglist_comp *to = &list[i >> 2]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; + uint64_t e_vaddr; + uint32_t e_len; + + if (unlikely(from_offset)) { + if (from_offset >= buf_sz) { + from_offset -= buf_sz; + continue; + } + e_vaddr = (uint64_t)vaddr + from_offset; + e_len = RTE_MIN((buf_sz - from_offset), size); + from_offset = 0; + } else { + e_vaddr = (uint64_t)vaddr; + e_len = RTE_MIN(buf_sz, size); + } + + to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); + to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr); + + if (extra_len && (e_len >= extra_offset)) { + /* Break the data at given offset */ + uint32_t next_len = e_len - extra_offset; + uint64_t next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) { + i--; + } else { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); + } + + extra_len = RTE_MIN(extra_len, size); + /* Insert extra data ptr */ + if (extra_len) { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = rte_cpu_to_be_16(extra_len); + to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)extra_buf->vaddr); + size -= extra_len; + } + + next_len = RTE_MIN(next_len, size); + /* insert the rest of the data */ + if (next_len) { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); + to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr); + size -= next_len; + } + extra_len = 0; + + } else { + size -= e_len; + } + if (extra_offset) + extra_offset -= size; + i++; + + if (unlikely(!size)) + break; + } + + *psize = size; + return (uint32_t)i; +} + +static __rte_always_inline uint32_t +fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte_mbuf *pkt) +{ + uint32_t buf_sz; + void *vaddr; + + while (unlikely(pkt != NULL)) { + struct roc_sglist_comp *to = &list[i >> 2]; + buf_sz = pkt->data_len; + vaddr = rte_pktmbuf_mtod(pkt, void *); + + to->u.s.len[i % 4] = rte_cpu_to_be_16(buf_sz); + to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)vaddr); + + pkt = pkt->next; + i++; + } + + return i; +} + +static __rte_always_inline uint32_t +fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size) +{ + struct roc_sg2list_comp *to = &list[i / 3]; + + to->u.s.len[i % 3] = (size); + to->ptr[i % 3] = (dma_addr); + to->u.s.valid_segs = (i % 3) + 1; + return ++i; +} + +static __rte_always_inline uint32_t +fill_sg2_comp_from_buf(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from) +{ + struct roc_sg2list_comp *to = &list[i / 3]; + + to->u.s.len[i % 3] = (from->size); + to->ptr[i % 3] = ((uint64_t)from->vaddr); + to->u.s.valid_segs = (i % 3) + 1; + return ++i; +} + +static __rte_always_inline uint32_t +fill_sg2_comp_from_buf_min(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from, + uint32_t *psize) +{ + struct roc_sg2list_comp *to = &list[i / 3]; + uint32_t size = *psize; + uint32_t e_len; + + e_len = RTE_MIN(from->size, size); + to->u.s.len[i % 3] = (e_len); + to->ptr[i % 3] = ((uint64_t)from->vaddr); + to->u.s.valid_segs = (i % 3) + 1; + *psize -= e_len; + return ++i; +} + +static __rte_always_inline uint32_t +fill_sg2_comp_from_iov(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from, + uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf, + uint32_t extra_offset) +{ + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + int32_t j; + + rte_prefetch2(psize); + + for (j = 0; j < from->buf_cnt; j++) { + struct roc_sg2list_comp *to = &list[i / 3]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; + uint64_t e_vaddr; + uint32_t e_len; + + if (unlikely(from_offset)) { + if (from_offset >= buf_sz) { + from_offset -= buf_sz; + continue; + } + e_vaddr = (uint64_t)vaddr + from_offset; + e_len = RTE_MIN((buf_sz - from_offset), size); + from_offset = 0; + } else { + e_vaddr = (uint64_t)vaddr; + e_len = RTE_MIN(buf_sz, size); + } + + to->u.s.len[i % 3] = (e_len); + to->ptr[i % 3] = (e_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + + if (extra_len && (e_len >= extra_offset)) { + /* Break the data at given offset */ + uint32_t next_len = e_len - extra_offset; + uint64_t next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) { + i--; + } else { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 3] = (e_len); + } + + extra_len = RTE_MIN(extra_len, size); + /* Insert extra data ptr */ + if (extra_len) { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (extra_len); + to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= extra_len; + } + + next_len = RTE_MIN(next_len, size); + /* insert the rest of the data */ + if (next_len) { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (next_len); + to->ptr[i % 3] = (next_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= next_len; + } + extra_len = 0; + + } else { + size -= e_len; + } + if (extra_offset) + extra_offset -= size; + i++; + + if (unlikely(!size)) + break; + } + + *psize = size; + return (uint32_t)i; +} + +#endif /*_CNXK_SG_H_ */