From patchwork Wed Nov 30 15:52:14 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Arkadiusz Kusztal X-Patchwork-Id: 17343 X-Patchwork-Delegate: pablo.de.lara.guarch@intel.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id A2076559A; Wed, 30 Nov 2016 16:52:50 +0100 (CET) Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by dpdk.org (Postfix) with ESMTP id C1787558B for ; Wed, 30 Nov 2016 16:52:27 +0100 (CET) Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga103.jf.intel.com with ESMTP; 30 Nov 2016 07:52:27 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.31,574,1473145200"; d="scan'208";a="792640430" Received: from sivswdev03.ir.intel.com (HELO localhost.localdomain) ([10.237.217.157]) by FMSMGA003.fm.intel.com with ESMTP; 30 Nov 2016 07:52:25 -0800 From: Arek Kusztal To: dev@dpdk.org Cc: fiona.trahe@intel.com, pablo.de.lara.guarch@intel.com, john.griffin@intel.com, deepak.k.jain@intel.com, Arek Kusztal Date: Wed, 30 Nov 2016 15:52:14 +0000 Message-Id: <1480521135-14869-3-git-send-email-arkadiuszx.kusztal@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1480521135-14869-1-git-send-email-arkadiuszx.kusztal@intel.com> References: <1480521135-14869-1-git-send-email-arkadiuszx.kusztal@intel.com> Subject: [dpdk-dev] [PATCH 2/3] crypto/qat: add SGL capability to Intel QuickAssist driver X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit adds scatter-gather list capability to Intel QuickAssist Technology driver. Signed-off-by: Arek Kusztal --- doc/guides/rel_notes/release_17_02.rst | 5 ++ drivers/crypto/qat/qat_crypto.c | 145 ++++++++++++++++++++++++++++++--- drivers/crypto/qat/qat_crypto.h | 6 ++ drivers/crypto/qat/qat_qp.c | 28 +++++++ 4 files changed, 173 insertions(+), 11 deletions(-) diff --git a/doc/guides/rel_notes/release_17_02.rst b/doc/guides/rel_notes/release_17_02.rst index 3b65038..873333b 100644 --- a/doc/guides/rel_notes/release_17_02.rst +++ b/doc/guides/rel_notes/release_17_02.rst @@ -38,6 +38,11 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Updated the QAT PMD.** + + The QAT PMD was updated with additional support for: + + * Scatter-gather list (SGL) support. Resolved Issues --------------- diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index afce4ac..fa3c277 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -67,6 +67,13 @@ #define BYTE_LENGTH 8 +#define SGL_2ND_COOKIE_OFF (QAT_SGL_MAX_NUMBER \ + * sizeof(struct qat_alg_buf) \ + + sizeof(struct qat_alg_buf_list)) + +#define SGL_SECOND_COOKIE_ADDR(arg, cast) ((cast)(arg) \ + + SGL_2ND_COOKIE_OFF) + static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = { { /* SHA1 HMAC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, @@ -503,7 +510,8 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift); static inline int -qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg); +qat_write_hw_desc_entry(struct rte_crypto_op *op, + uint8_t *out_msg, struct qat_qp *qp); void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session) @@ -839,7 +847,6 @@ unsigned qat_crypto_sym_get_session_private_size( return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8); } - uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -873,9 +880,16 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, } while (nb_ops_sent != nb_ops_possible) { - ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail); + ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, + tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; + /* + * This message cannot be enqueued, + * decrease number of ops that wasnt sent + */ + rte_atomic16_sub(&tmp_qp->inflights16, + nb_ops_possible - nb_ops_sent); if (nb_ops_sent == 0) return 0; goto kick_tail; @@ -884,6 +898,7 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, tail = adf_modulo(tail + queue->msg_size, queue->modulo); nb_ops_sent++; cur_op++; + } kick_tail: WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, @@ -914,7 +929,7 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, - sizeof(struct icp_qat_fw_comn_resp)); + sizeof(struct icp_qat_fw_comn_resp)); #endif if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( @@ -923,6 +938,15 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, } else { rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } + /* + * _priv set for symmetric means it is SGL + * free this address from qp mempool of SGL's + */ + if (rx_op->_priv) { + rte_mempool_put(tmp_qp->sgl_pool, rx_op->_priv); + rx_op->_priv = NULL; + } + *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG; queue->head = adf_modulo(queue->head + queue->msg_size, @@ -945,8 +969,51 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, } static inline int -qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) +qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, + void *sgl_cookie, int32_t total_pck_len) { + int nr = 0; + struct qat_alg_buf_list *list = sgl_cookie; + + int32_t total_len = rte_pktmbuf_mtophys(buf) + - buff_start; + + uint32_t first_buf_len = rte_pktmbuf_mtophys(buf) - + buff_start + rte_pktmbuf_data_len(buf); + + while (buf) { + if (unlikely(nr == QAT_SGL_MAX_NUMBER)) { + PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL" + " entry(%u)", + QAT_SGL_MAX_NUMBER); + return -EINVAL; + } + + list->bufers[nr].len = rte_pktmbuf_data_len(buf); + list->bufers[nr].resrvd = 0; + list->bufers[nr].addr = rte_pktmbuf_mtophys(buf); + + ++nr; + buf = buf->next; + + total_len += list->bufers[nr].len; + if (total_len >= total_pck_len) + buf = NULL; + } + + list->bufers[0].addr = buff_start; + list->bufers[0].len = first_buf_len; + + list->num_bufs = nr; + + return 0; +} + +static inline int +qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, + struct qat_qp *qp) +{ + int ret = 0; struct qat_session *ctx; struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; @@ -956,6 +1023,9 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) uint32_t auth_len = 0, auth_ofs = 0; uint32_t min_ofs = 0; uint64_t src_buf_start = 0, dst_buf_start = 0; + void *cookie_virt_addr = NULL; + phys_addr_t cookie_phys_addr = 0; + uint8_t do_sgl = 0; #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX @@ -1073,10 +1143,25 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) } + if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next)) + do_sgl = 1; + + if (do_sgl) { + if (rte_mempool_get(qp->sgl_pool, &cookie_virt_addr)) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot get sgl_cookie"); + return -EFAULT; + } + cookie_phys_addr = rte_mempool_virt2phy(qp->sgl_pool, + cookie_virt_addr); + } + /* adjust for chain case */ if (do_cipher && do_auth) min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs; + if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl)) + min_ofs = 0; + if (unlikely(op->sym->m_dst != NULL)) { /* Out-of-place operation (OOP) * Don't align DMA start. DMA the minimum data-set @@ -1086,6 +1171,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs); dst_buf_start = rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs); + } else { /* In-place operation * Start DMA at nearest aligned address below min_ofs @@ -1131,8 +1217,49 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) (cipher_param->cipher_offset + cipher_param->cipher_length) : (auth_param->auth_off + auth_param->auth_len); - qat_req->comn_mid.src_data_addr = src_buf_start; - qat_req->comn_mid.dest_data_addr = dst_buf_start; + if (do_sgl) { + + op->_priv = cookie_virt_addr; + + ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags, + QAT_COMN_PTR_TYPE_SGL); + ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start, + cookie_virt_addr, qat_req->comn_mid.src_length); + if (ret) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array"); + rte_mempool_put(qp->sgl_pool, cookie_virt_addr); + return ret; + } + + if (likely(op->sym->m_dst == NULL)) + qat_req->comn_mid.dest_data_addr = + qat_req->comn_mid.src_data_addr = + cookie_phys_addr; + else { + ret = qat_sgl_fill_array(op->sym->m_dst, + dst_buf_start, + SGL_SECOND_COOKIE_ADDR(cookie_virt_addr, + uint8_t *), + qat_req->comn_mid.dst_length); + + if (ret) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array"); + rte_mempool_put(qp->sgl_pool, cookie_virt_addr); + return ret; + } + + qat_req->comn_mid.src_data_addr = + cookie_phys_addr; + + qat_req->comn_mid.dest_data_addr = + SGL_SECOND_COOKIE_ADDR(cookie_phys_addr, + phys_addr_t); + } + + } else { + qat_req->comn_mid.src_data_addr = src_buf_start; + qat_req->comn_mid.dest_data_addr = dst_buf_start; + } if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { @@ -1164,13 +1291,9 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) } } - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "qat_req:", qat_req, sizeof(struct icp_qat_fw_la_bulk_req)); - rte_hexdump(stdout, "src_data:", - rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), - rte_pktmbuf_data_len(op->sym->m_src)); rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data, op->sym->cipher.iv.length); rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index 6b84488..8612706 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -45,6 +45,11 @@ (((num) + (align) - 1) & ~((align) - 1)) #define QAT_64_BTYE_ALIGN_MASK (~0x3f) +/* + * Maximum number of SGL entries + */ +#define QAT_SGL_MAX_NUMBER 16 + /** * Structure associated with each queue. */ @@ -69,6 +74,7 @@ struct qat_qp { struct qat_queue tx_q; struct qat_queue rx_q; struct rte_cryptodev_stats stats; + struct rte_mempool *sgl_pool; } __rte_cache_aligned; /** private data structure for each QAT device */ diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index 2e7188b..27c5f9a 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -41,6 +41,7 @@ #include "qat_logs.h" #include "qat_crypto.h" +#include "qat_algs.h" #include "adf_transport_access_macros.h" #define ADF_MAX_SYM_DESC 4096 @@ -136,6 +137,7 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, { struct qat_qp *qp; int ret; + char sgl_pool_name[RTE_RING_NAMESIZE]; PMD_INIT_FUNC_TRACE(); @@ -191,9 +193,31 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, qat_queue_delete(&(qp->tx_q)); goto create_err; } + adf_configure_queues(qp); adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr); + + snprintf(sgl_pool_name, RTE_RING_NAMESIZE, "%s_qp_sgl_%d_%hu", + dev->driver->pci_drv.driver.name, dev->data->dev_id, + queue_pair_id); + + qp->sgl_pool = rte_mempool_lookup(sgl_pool_name); + + if (qp->sgl_pool == NULL) + qp->sgl_pool = rte_mempool_create(sgl_pool_name, + qp->tx_q.max_inflights, + (sizeof(struct qat_alg_buf_list) + + sizeof(struct qat_alg_buf) * + QAT_SGL_MAX_NUMBER) * 2, 0, 0, + NULL, NULL, NULL, NULL, socket_id, 0); + if (qp->sgl_pool == NULL) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot create" + " sgl mempool"); + goto create_err; + } + dev->data->queue_pairs[queue_pair_id] = qp; + return 0; create_err: @@ -221,6 +245,10 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) } adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr); + + if (qp->sgl_pool) + rte_mempool_free(qp->sgl_pool); + rte_free(qp); dev->data->queue_pairs[queue_pair_id] = NULL; return 0;