@@ -57,9 +57,10 @@
#define PLT_ALIGN RTE_ALIGN
#define PLT_ALIGN_MUL_CEIL RTE_ALIGN_MUL_CEIL
#define PLT_MODEL_MZ_NAME "roc_model_mz"
-#define PLT_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
+#define PLT_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
#define BITMASK_ULL GENMASK_ULL
#define PLT_ALIGN_CEIL RTE_ALIGN_CEIL
+#define PLT_ALIGN_FLOOR RTE_ALIGN_FLOOR
#define PLT_INIT RTE_INIT
#define PLT_MAX_ETHPORTS RTE_MAX_ETHPORTS
#define PLT_TAILQ_FOREACH_SAFE RTE_TAILQ_FOREACH_SAFE
@@ -726,3 +726,50 @@ roc_se_ctx_swap(struct roc_se_ctx *se_ctx)
zs_ctx->zuc.otk_ctx.w0.u64 = htobe64(zs_ctx->zuc.otk_ctx.w0.u64);
}
+
+void
+roc_se_ctx_init(struct roc_se_ctx *roc_se_ctx)
+{
+ struct se_ctx_s *ctx = &roc_se_ctx->se_ctx;
+ uint64_t ctx_len, *uc_ctx;
+ uint8_t i;
+
+ switch (roc_se_ctx->fc_type) {
+ case ROC_SE_FC_GEN:
+ ctx_len = sizeof(struct roc_se_context);
+ break;
+ case ROC_SE_PDCP:
+ ctx_len = sizeof(struct roc_se_zuc_snow3g_ctx);
+ break;
+ case ROC_SE_KASUMI:
+ ctx_len = sizeof(struct roc_se_kasumi_ctx);
+ break;
+ case ROC_SE_PDCP_CHAIN:
+ ctx_len = sizeof(struct roc_se_zuc_snow3g_chain_ctx);
+ break;
+ default:
+ ctx_len = 0;
+ }
+
+ ctx_len = PLT_ALIGN_CEIL(ctx_len, 8);
+
+ /* Skip w0 for swap */
+ uc_ctx = PLT_PTR_ADD(ctx, sizeof(ctx->w0));
+ for (i = 0; i < (ctx_len / 8); i++)
+ uc_ctx[i] = plt_cpu_to_be_64(((uint64_t *)uc_ctx)[i]);
+
+ /* Include w0 */
+ ctx_len += sizeof(ctx->w0);
+ ctx_len = PLT_ALIGN_CEIL(ctx_len, 8);
+
+ ctx->w0.s.aop_valid = 1;
+ ctx->w0.s.ctx_hdr_size = 0;
+
+ ctx->w0.s.ctx_size = PLT_ALIGN_FLOOR(ctx_len, 128);
+ if (ctx->w0.s.ctx_size == 0)
+ ctx->w0.s.ctx_size = 1;
+
+ ctx->w0.s.ctx_push_size = ctx_len / 8;
+ if (ctx->w0.s.ctx_push_size > 32)
+ ctx->w0.s.ctx_push_size = 32;
+}
@@ -300,14 +300,31 @@ struct roc_se_ctx {
uint64_t rsvd : 17;
union cpt_inst_w4 template_w4;
/* Below fields are accessed by hardware */
- union {
- struct roc_se_context fctx;
- struct roc_se_zuc_snow3g_ctx zs_ctx;
- struct roc_se_zuc_snow3g_chain_ctx zs_ch_ctx;
- struct roc_se_kasumi_ctx k_ctx;
- } se_ctx;
+ struct se_ctx_s {
+ /* Word0 */
+ union {
+ struct {
+ uint64_t rsvd : 48;
+
+ uint64_t ctx_push_size : 7;
+ uint64_t rsvd1 : 1;
+
+ uint64_t ctx_hdr_size : 2;
+ uint64_t aop_valid : 1;
+ uint64_t rsvd2 : 1;
+ uint64_t ctx_size : 4;
+ } s;
+ uint64_t u64;
+ } w0;
+ union {
+ struct roc_se_context fctx;
+ struct roc_se_zuc_snow3g_ctx zs_ctx;
+ struct roc_se_zuc_snow3g_chain_ctx zs_ch_ctx;
+ struct roc_se_kasumi_ctx k_ctx;
+ };
+ } se_ctx __plt_aligned(ROC_ALIGN);
uint8_t *auth_key;
-};
+} __plt_aligned(ROC_ALIGN);
struct roc_se_fc_params {
union {
@@ -349,14 +366,13 @@ roc_se_zuc_bytes_swap(uint8_t *arr, int len)
}
}
-int __roc_api roc_se_auth_key_set(struct roc_se_ctx *se_ctx,
- roc_se_auth_type type, const uint8_t *key,
- uint16_t key_len, uint16_t mac_len);
+int __roc_api roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
+ const uint8_t *key, uint16_t key_len, uint16_t mac_len);
-int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx,
- roc_se_cipher_type type, const uint8_t *key,
- uint16_t key_len, uint8_t *salt);
+int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
+ const uint8_t *key, uint16_t key_len, uint8_t *salt);
void __roc_api roc_se_ctx_swap(struct roc_se_ctx *se_ctx);
+void __roc_api roc_se_ctx_init(struct roc_se_ctx *se_ctx);
#endif /* __ROC_SE_H__ */
@@ -99,6 +99,7 @@ INTERNAL {
roc_model;
roc_se_auth_key_set;
roc_se_ciph_key_set;
+ roc_se_ctx_init;
roc_nix_bpf_alloc;
roc_nix_bpf_config;
roc_nix_bpf_connect;
@@ -52,14 +52,13 @@ cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
if (rte_mempool_get(qp->sess_mp, (void **)&sess) < 0)
return NULL;
- ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform,
- sess);
+ ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess, true);
if (ret) {
rte_mempool_put(qp->sess_mp, (void *)sess);
goto sess_put;
}
- priv = (void *)sess->driver_priv_data;
+ priv = (void *)sess;
sym_op->session = sess;
return priv;
@@ -121,13 +120,13 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
- sec_sess = (struct cn10k_sec_session *)(sym_op->session);
+ sec_sess = (struct cn10k_sec_session *)sym_op->session;
ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0]);
if (unlikely(ret))
return 0;
w7 = sec_sess->inst.w7;
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
- sess = CRYPTODEV_GET_SYM_SESS_PRIV(sym_op->session);
+ sess = (struct cnxk_se_sess *)(sym_op->session);
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, &inst[0], is_sg_ver2);
if (unlikely(ret))
return 0;
@@ -141,7 +140,7 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, &inst[0], is_sg_ver2);
if (unlikely(ret)) {
- sym_session_clear(op->sym->session);
+ sym_session_clear(op->sym->session, true);
rte_mempool_put(qp->sess_mp, op->sym->session);
return 0;
}
@@ -316,7 +315,7 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused, vo
} else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
priv->qp = qp;
priv->cpt_inst_w2 = w2;
} else
@@ -351,7 +350,7 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op, struct cnxk_cpt_qp **qp, ui
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ priv = (struct cnxk_se_sess *)op->sym->session;
*qp = priv->qp;
*w2 = priv->cpt_inst_w2;
} else {
@@ -914,7 +913,7 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
temp_sess_free:
if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
- sym_session_clear(cop->sym->session);
+ sym_session_clear(cop->sym->session, true);
rte_mempool_put(qp->sess_mp, cop->sym->session);
cop->sym->session = NULL;
}
@@ -49,11 +49,11 @@ cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
if (rte_mempool_get(qp->sess_mp, (void **)&sess) < 0)
return NULL;
- ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess);
+ ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess, true);
if (ret)
goto sess_put;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
sym_op->session = sess;
@@ -76,7 +76,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
sym_op = op->sym;
- sess = CRYPTODEV_GET_SYM_SESS_PRIV(sym_op->session);
+ sess = (struct cnxk_se_sess *)sym_op->session;
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);
inst->w7.u64 = sess->cpt_inst_w7;
} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
@@ -90,7 +90,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);
if (unlikely(ret)) {
- sym_session_clear(op->sym->session);
+ sym_session_clear(op->sym->session, true);
rte_mempool_put(qp->sess_mp, op->sym->session);
}
inst->w7.u64 = sess->cpt_inst_w7;
@@ -326,7 +326,7 @@ cn9k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
} else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
priv->qp = qp;
priv->cpt_inst_w2 = w2;
} else
@@ -361,7 +361,7 @@ cn9k_ca_meta_info_extract(struct rte_crypto_op *op,
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ priv = (struct cnxk_se_sess *)op->sym->session;
*qp = priv->qp;
inst->w2.u64 = priv->cpt_inst_w2;
} else {
@@ -630,7 +630,7 @@ cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
temp_sess_free:
if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
- sym_session_clear(cop->sym->session);
+ sym_session_clear(cop->sym->session, true);
rte_mempool_put(qp->sess_mp, cop->sym->session);
cop->sym->session = NULL;
}
@@ -626,6 +626,11 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
+ if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+ inst_w7.s.ctx_val = 1;
+ else
+ inst_w7.s.cptr += 8;
+
/* Set the engine group */
if (sess->zsk_flag || sess->aes_ctr_eea2 || sess->is_sha3)
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
@@ -636,19 +641,22 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
}
int
-sym_session_configure(struct roc_cpt *roc_cpt,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess)
+sym_session_configure(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess, bool is_session_less)
{
enum cpt_dp_thread_type thr_type;
- struct cnxk_se_sess *sess_priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ struct cnxk_se_sess *sess_priv = (struct cnxk_se_sess *)sess;
int ret;
- memset(sess_priv, 0, sizeof(struct cnxk_se_sess));
+ if (is_session_less)
+ memset(sess_priv, 0, sizeof(struct cnxk_se_sess));
+
ret = cnxk_sess_fill(roc_cpt, xform, sess_priv);
if (ret)
goto priv_put;
+ sess_priv->lf = roc_cpt->lf[0];
+
if (sess_priv->cpt_op & ROC_SE_OP_CIPHER_MASK) {
switch (sess_priv->roc_se_ctx.fc_type) {
case ROC_SE_FC_GEN:
@@ -690,6 +698,10 @@ sym_session_configure(struct roc_cpt *roc_cpt,
}
sess_priv->cpt_inst_w7 = cnxk_cpt_inst_w7_get(sess_priv, roc_cpt);
+
+ if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+ roc_se_ctx_init(&sess_priv->roc_se_ctx);
+
return 0;
priv_put:
@@ -704,25 +716,31 @@ cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev,
struct cnxk_cpt_vf *vf = dev->data->dev_private;
struct roc_cpt *roc_cpt = &vf->cpt;
- return sym_session_configure(roc_cpt, xform, sess);
+ return sym_session_configure(roc_cpt, xform, sess, false);
}
void
-sym_session_clear(struct rte_cryptodev_sym_session *sess)
+sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less)
{
- struct cnxk_se_sess *sess_priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ struct cnxk_se_sess *sess_priv = (struct cnxk_se_sess *)sess;
+
+ /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+ roc_cpt_lf_ctx_flush(sess_priv->lf, &sess_priv->roc_se_ctx.se_ctx, true);
+
+ plt_delay_ms(1);
if (sess_priv->roc_se_ctx.auth_key != NULL)
plt_free(sess_priv->roc_se_ctx.auth_key);
- memset(sess_priv, 0, cnxk_cpt_sym_session_get_size(NULL));
+ if (is_session_less)
+ memset(sess_priv, 0, cnxk_cpt_sym_session_get_size(NULL));
}
void
cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev __rte_unused,
struct rte_cryptodev_sym_session *sess)
{
- return sym_session_clear(sess);
+ return sym_session_clear(sess, false);
}
unsigned int
@@ -107,18 +107,15 @@ int cnxk_cpt_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id);
unsigned int cnxk_cpt_sym_session_get_size(struct rte_cryptodev *dev);
-int cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev,
- struct rte_crypto_sym_xform *xform,
+int cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform,
struct rte_cryptodev_sym_session *sess);
-int sym_session_configure(struct roc_cpt *roc_cpt,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess);
+int sym_session_configure(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess, bool is_session_less);
-void cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess);
+void cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess);
-void sym_session_clear(struct rte_cryptodev_sym_session *sess);
+void sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less);
unsigned int cnxk_ae_session_size_get(struct rte_cryptodev *dev __rte_unused);
@@ -6,6 +6,8 @@
#define _CNXK_SE_H_
#include <stdbool.h>
+#include <rte_cryptodev.h>
+
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
#include "cnxk_sg.h"
@@ -25,6 +27,7 @@ enum cpt_dp_thread_type {
};
struct cnxk_se_sess {
+ struct rte_cryptodev_sym_session rte_sess;
uint16_t cpt_op : 4;
uint16_t zsk_flag : 4;
uint16_t aes_gcm : 1;
@@ -51,10 +54,11 @@ struct cnxk_se_sess {
uint64_t cpt_inst_w2;
struct cnxk_cpt_qp *qp;
struct roc_se_ctx roc_se_ctx;
-} __rte_cache_aligned;
+ struct roc_cpt_lf *lf;
+} __rte_aligned(ROC_ALIGN);
-static __rte_always_inline int
-fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess);
+static __rte_always_inline int fill_sess_gmac(struct rte_crypto_sym_xform *xform,
+ struct cnxk_se_sess *sess);
static inline void
cpt_pack_iv(uint8_t *iv_src, uint8_t *iv_dst)