From patchwork Tue Sep 12 09:31:18 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Burakov, Anatoly" X-Patchwork-Id: 28612 X-Patchwork-Delegate: pablo.de.lara.guarch@intel.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id E15571AF03; Tue, 12 Sep 2017 11:31:28 +0200 (CEST) Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by dpdk.org (Postfix) with ESMTP id 76B6F56A1 for ; Tue, 12 Sep 2017 11:31:23 +0200 (CEST) Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 12 Sep 2017 02:31:21 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.42,382,1500966000"; d="scan'208";a="150841056" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by fmsmga006.fm.intel.com with ESMTP; 12 Sep 2017 02:31:19 -0700 Received: from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com [10.237.217.45]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id v8C9VJpU004579; Tue, 12 Sep 2017 10:31:19 +0100 Received: from sivswdev01.ir.intel.com (localhost [127.0.0.1]) by sivswdev01.ir.intel.com with ESMTP id v8C9VJQc030070; Tue, 12 Sep 2017 10:31:19 +0100 Received: (from aburakov@localhost) by sivswdev01.ir.intel.com with LOCAL id v8C9VIJT030066; Tue, 12 Sep 2017 10:31:18 +0100 From: Anatoly Burakov To: dev@dpdk.org Cc: fiona.trahe@intel.com, john.griffin@intel.com, deepak.k.jain@intel.com, pablo.de.lara.guarch@intel.com Date: Tue, 12 Sep 2017 10:31:18 +0100 Message-Id: <8aada4555a65d6ceb8b581f7909020a487ee9f3b.1505142402.git.anatoly.burakov@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <7ebf9384e20ea6fde085044c505e19719d041d25.1503651900.git.anatoly.burakov@intel.com> References: <7ebf9384e20ea6fde085044c505e19719d041d25.1503651900.git.anatoly.burakov@intel.com> In-Reply-To: References: Subject: [dpdk-dev] [PATCH v2 3/3] crypto/qat: enable TX tail writes coalescing X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Don't write CSR tail until we processed enough TX descriptors. To avoid crypto operations sitting in the TX ring indefinitely, the "force write" threshold is used: - on TX, no tail write coalescing will occur if number of inflights is below force write threshold - on RX, check if we have a number of crypto ops enqueued that is below force write threshold that are not yet submitted to processing. Signed-off-by: Anatoly Burakov Acked-by: Fiona Trahe --- v2: fixed commit message doc/guides/rel_notes/release_17_11.rst | 1 + drivers/crypto/qat/qat_crypto.c | 41 ++++++++++++++++++++++++---------- drivers/crypto/qat/qat_crypto.h | 7 ++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/doc/guides/rel_notes/release_17_11.rst b/doc/guides/rel_notes/release_17_11.rst index 0b77095..f0d3960 100644 --- a/doc/guides/rel_notes/release_17_11.rst +++ b/doc/guides/rel_notes/release_17_11.rst @@ -47,6 +47,7 @@ New Features * Removed atomics from the internal queue pair structure. * Coalesce writes to HEAD CSR on response processing. + * Coalesce writes to TAIL CSR on request processing. Resolved Issues diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 1656e0f..a2b202f 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -921,6 +921,14 @@ qat_bpicipher_postprocess(struct qat_session *ctx, return sym_op->cipher.data.length - last_block_len; } +static inline void +txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); + q->nb_pending_requests = 0; + q->csr_tail = q->tail; +} + uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -973,10 +981,13 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, cur_op++; } kick_tail: - WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, tail); queue->tail = tail; tmp_qp->stats.enqueued_count += nb_ops_sent; + queue->nb_pending_requests += nb_ops_sent; + if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH || + queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) { + txq_write_tail(tmp_qp, queue); + } return nb_ops_sent; } @@ -1011,17 +1022,18 @@ uint16_t qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct qat_queue *queue; + struct qat_queue *rx_queue, *tx_queue; struct qat_qp *tmp_qp = (struct qat_qp *)qp; uint32_t msg_counter = 0; struct rte_crypto_op *rx_op; struct icp_qat_fw_comn_resp *resp_msg; uint32_t head; - queue = &(tmp_qp->rx_q); - head = queue->head; + rx_queue = &(tmp_qp->rx_q); + tx_queue = &(tmp_qp->tx_q); + head = rx_queue->head; resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + head); + ((uint8_t *)rx_queue->base_addr + head); while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && msg_counter != nb_ops) { @@ -1048,21 +1060,26 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } - head = adf_modulo(head + queue->msg_size, queue->modulo); + head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + head); + ((uint8_t *)rx_queue->base_addr + head); *ops = rx_op; ops++; msg_counter++; } if (msg_counter > 0) { - queue->head = head; + rx_queue->head = head; tmp_qp->stats.dequeued_count += msg_counter; - queue->nb_processed_responses += msg_counter; + rx_queue->nb_processed_responses += msg_counter; tmp_qp->inflights16 -= msg_counter; - if (queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) - rxq_free_desc(tmp_qp, queue); + if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) + rxq_free_desc(tmp_qp, rx_queue); + } + /* also check if tail needs to be advanced */ + if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH && + tx_queue->tail != tx_queue->csr_tail) { + txq_write_tail(tmp_qp, tx_queue); } return msg_counter; } diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index d78957c..0ebb083 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -52,6 +52,10 @@ #define QAT_CSR_HEAD_WRITE_THRESH 32U /* number of requests to accumulate before writing head CSR */ +#define QAT_CSR_TAIL_WRITE_THRESH 32U +/* number of requests to accumulate before writing tail CSR */ +#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U +/* number of inflights below which no tail write coalescing should occur */ struct qat_session; @@ -77,8 +81,11 @@ struct qat_queue { uint8_t hw_queue_number; /* HW queue aka ring offset on bundle */ uint32_t csr_head; /* last written head value */ + uint32_t csr_tail; /* last written tail value */ uint16_t nb_processed_responses; /* number of responses processed since last CSR head write */ + uint16_t nb_pending_requests; + /* number of requests pending since last CSR tail write */ }; struct qat_qp {