@@ -56,7 +56,9 @@ Limitations
-----------
* Chained mbufs are not supported.
-* Only in-place is currently supported (destination address is the same as source address).
+* OOP (different source/destination m_bufs) is supported for Linear buffers, but
+ is limited to cipher-only in the encrypt direction, i.e. encrypt & generate is
+ not supported with OOP.
* RTE_CRYPTO_HASH_AES_GMAC is supported by library version v0.51 or later.
* RTE_CRYPTO_HASH_SHA* is supported by library version v0.52 or later.
@@ -11,6 +11,7 @@ CPU AVX = Y
CPU AVX2 = Y
CPU AVX512 = Y
CPU AESNI = Y
+OOP LB In LB Out = Y
;
; Supported crypto algorithms of the 'aesni_mb' crypto driver.
@@ -754,9 +754,10 @@ static inline int
set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
struct rte_crypto_op *op, uint8_t *digest_idx)
{
- struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct rte_mbuf *m_src = op->sym->m_src;
+ struct rte_mbuf *m_dst = op->sym->m_dst;
struct aesni_mb_session *session;
- uint16_t m_offset = 0;
+ uint16_t dst_offset = 0;
session = get_session(qp, op);
if (session == NULL) {
@@ -840,30 +841,20 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
}
/* Mutable crypto operation parameters */
- if (op->sym->m_dst) {
- m_src = m_dst = op->sym->m_dst;
-
- /* append space for output data to mbuf */
- char *odata = rte_pktmbuf_append(m_dst,
- rte_pktmbuf_data_len(op->sym->m_src));
- if (odata == NULL) {
- AESNI_MB_LOG(ERR, "failed to allocate space in destination "
- "mbuf for source data");
- op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ if (!(op->sym->m_dst))
+ /* in-place operation */
+ m_dst = m_src;
+ else
+ /* out-of-place operation */
+ /* lib doesn't support encrypt + generate for OOP yet
+ * so only allow for cipher-only in encrypt direction
+ */
+ if ((session->chain_order == CIPHER_HASH) &&
+ (session->auth.algo != NULL_HASH)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
}
- memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),
- rte_pktmbuf_data_len(op->sym->m_src));
- } else {
- m_dst = m_src;
- if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
- session->cipher.mode == GCM))
- m_offset = op->sym->aead.data.offset;
- else
- m_offset = op->sym->cipher.data.offset;
- }
-
/* Set digest output location */
if (job->hash_alg != NULL_HASH &&
session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
@@ -892,16 +883,15 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
/* Set IV parameters */
job->iv_len_in_bytes = session->iv.length;
- /* Data Parameter */
- job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
- job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
-
+ /* Data Parameters */
switch (job->hash_alg) {
case AES_CCM:
+ dst_offset = op->sym->aead.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
- job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -910,6 +900,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
case AES_GMAC:
if (session->cipher.mode == GCM) {
+ dst_offset = op->sym->aead.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->hash_start_src_offset_in_bytes =
@@ -919,6 +910,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->msg_len_to_hash_in_bytes =
op->sym->aead.data.length;
} else {
+ dst_offset = op->sym->cipher.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->auth.data.offset;
job->hash_start_src_offset_in_bytes =
@@ -932,6 +924,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
break;
default:
+ dst_offset = op->sym->cipher.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->cipher.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
@@ -942,6 +935,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
session->iv.offset);
}
+ job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+ job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, dst_offset);
/* Set user data to be crypto operation data struct */
job->user_data = op;
@@ -961,7 +956,7 @@ static inline void
generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
struct aesni_mb_session *sess)
{
- /* No extra copy neeed */
+ /* No extra copy needed */
if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
return;
@@ -1215,7 +1210,9 @@ cryptodev_aesni_mb_create(const char *name,
dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
- RTE_CRYPTODEV_FF_CPU_AESNI;
+ RTE_CRYPTODEV_FF_CPU_AESNI |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
mb_mgr = alloc_mb_mgr(0);
if (mb_mgr == NULL)
@@ -754,9 +754,10 @@ static inline int
set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
struct rte_crypto_op *op, uint8_t *digest_idx)
{
- struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct rte_mbuf *m_src = op->sym->m_src;
+ struct rte_mbuf *m_dst = op->sym->m_dst;
struct aesni_mb_session *session;
- uint16_t m_offset = 0;
+ uint16_t dst_offset = 0;
session = get_session(qp, op);
if (session == NULL) {
@@ -840,30 +841,20 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
}
/* Mutable crypto operation parameters */
- if (op->sym->m_dst) {
- m_src = m_dst = op->sym->m_dst;
-
- /* append space for output data to mbuf */
- char *odata = rte_pktmbuf_append(m_dst,
- rte_pktmbuf_data_len(op->sym->m_src));
- if (odata == NULL) {
- AESNI_MB_LOG(ERR, "failed to allocate space in destination "
- "mbuf for source data");
- op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ if (!(op->sym->m_dst))
+ /* in-place operation */
+ m_dst = m_src;
+ else
+ /* out-of-place operation */
+ /* lib doesn't support encrypt + generate for OOP yet
+ * so only allow for cipher-only in encrypt direction
+ */
+ if ((session->chain_order == CIPHER_HASH) &&
+ (session->auth.algo != NULL_HASH)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
}
- memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),
- rte_pktmbuf_data_len(op->sym->m_src));
- } else {
- m_dst = m_src;
- if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
- session->cipher.mode == GCM))
- m_offset = op->sym->aead.data.offset;
- else
- m_offset = op->sym->cipher.data.offset;
- }
-
/* Set digest output location */
if (job->hash_alg != NULL_HASH &&
session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
@@ -892,16 +883,15 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
/* Set IV parameters */
job->iv_len_in_bytes = session->iv.length;
- /* Data Parameter */
- job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
- job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
-
+ /* Data Parameters */
switch (job->hash_alg) {
case AES_CCM:
+ dst_offset = op->sym->aead.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
- job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -910,6 +900,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
case AES_GMAC:
if (session->cipher.mode == GCM) {
+ dst_offset = op->sym->aead.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->aead.data.offset;
job->hash_start_src_offset_in_bytes =
@@ -919,6 +910,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->msg_len_to_hash_in_bytes =
op->sym->aead.data.length;
} else {
+ dst_offset = op->sym->cipher.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->auth.data.offset;
job->hash_start_src_offset_in_bytes =
@@ -932,6 +924,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
break;
default:
+ dst_offset = op->sym->cipher.data.offset;
job->cipher_start_src_offset_in_bytes =
op->sym->cipher.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
@@ -942,6 +935,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
session->iv.offset);
}
+ job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+ job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, dst_offset);
/* Set user data to be crypto operation data struct */
job->user_data = op;
@@ -960,7 +955,7 @@ static inline void
generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
struct aesni_mb_session *sess)
{
- /* No extra copy neeed */
+ /* No extra copy needed */
if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
return;
@@ -1210,7 +1205,8 @@ cryptodev_aesni_mb_create(const char *name,
dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
- RTE_CRYPTODEV_FF_CPU_AESNI;
+ RTE_CRYPTODEV_FF_CPU_AESNI |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
switch (vector_mode) {
case RTE_AESNI_MB_SSE:
@@ -1485,7 +1485,8 @@ static const struct blockcipher_test_case aes_chain_test_cases[] = {
BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR |
- BLOCKCIPHER_TEST_TARGET_PMD_OCTEONTX
+ BLOCKCIPHER_TEST_TARGET_PMD_OCTEONTX |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-128-CBC HMAC-SHA224 Encryption Digest",
@@ -1698,7 +1699,8 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR |
BLOCKCIPHER_TEST_TARGET_PMD_CCP |
- BLOCKCIPHER_TEST_TARGET_PMD_VIRTIO
+ BLOCKCIPHER_TEST_TARGET_PMD_VIRTIO |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-256-CBC OOP Decryption",
@@ -1711,7 +1713,8 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR |
BLOCKCIPHER_TEST_TARGET_PMD_CCP |
- BLOCKCIPHER_TEST_TARGET_PMD_VIRTIO
+ BLOCKCIPHER_TEST_TARGET_PMD_VIRTIO |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-128-CTR Encryption",
@@ -1873,42 +1876,48 @@ static const struct blockcipher_test_case aes_docsis_test_cases[] = {
.test_data = &aes_test_data_docsis_1,
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-DOCSIS-BPI OOP Runt Block Encryption",
.test_data = &aes_test_data_docsis_2,
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-DOCSIS-BPI OOP Uneven Block Encryption",
.test_data = &aes_test_data_docsis_3,
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-DOCSIS-BPI OOP Full Block Decryption",
.test_data = &aes_test_data_docsis_1,
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-DOCSIS-BPI OOP Runt Block Decryption",
.test_data = &aes_test_data_docsis_2,
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "AES-DOCSIS-BPI OOP Uneven Block Decryption",
.test_data = &aes_test_data_docsis_3,
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
- .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
}
};
#endif /* TEST_CRYPTODEV_AES_TEST_VECTORS_H_ */
@@ -1016,7 +1016,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "DES-DOCSIS-BPI OOP Runt Block Encryption",
@@ -1024,7 +1025,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "DES-DOCSIS-BPI OOP Uneven Encryption",
@@ -1032,7 +1034,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "DES-DOCSIS-BPI OOP Full Block Decryption",
@@ -1040,7 +1043,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "DES-DOCSIS-BPI OOP Runt Block Decryption",
@@ -1048,7 +1052,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "DES-DOCSIS-BPI OOP Uneven Decryption",
@@ -1056,7 +1061,8 @@ static const struct blockcipher_test_case des_docsis_test_cases[] = {
.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP,
.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
- BLOCKCIPHER_TEST_TARGET_PMD_QAT
+ BLOCKCIPHER_TEST_TARGET_PMD_QAT |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
}
};
@@ -1200,7 +1206,8 @@ static const struct blockcipher_test_case triple_des_chain_test_cases[] = {
BLOCKCIPHER_TEST_TARGET_PMD_QAT |
BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
- BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR
+ BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "3DES-128-CBC HMAC-SHA1 Decryption Digest"
@@ -1212,7 +1219,8 @@ static const struct blockcipher_test_case triple_des_chain_test_cases[] = {
BLOCKCIPHER_TEST_TARGET_PMD_QAT |
BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
- BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR
+ BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR |
+ BLOCKCIPHER_TEST_TARGET_PMD_MB
},
{
.test_descr = "3DES-128-CBC HMAC-SHA1 Encryption Digest"