From patchwork Mon Aug 26 07:44:59 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Dybkowski, AdamX" X-Patchwork-Id: 58410 X-Patchwork-Delegate: gakhil@marvell.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id EBF831E9C2; Mon, 2 Sep 2019 16:30:27 +0200 (CEST) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by dpdk.org (Postfix) with ESMTP id 703351E990 for ; Mon, 2 Sep 2019 16:30:25 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 02 Sep 2019 07:30:25 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,459,1559545200"; d="scan'208";a="211705518" Received: from adamdybx-mobl.ger.corp.intel.com (HELO localhost.localdomain) ([10.104.14.190]) by fmsmga002.fm.intel.com with ESMTP; 02 Sep 2019 07:30:23 -0700 From: Adam Dybkowski To: dev@dpdk.org, fiona.trahe@intel.com, akhil.goyal@nxp.com, arturx.trybula@intel.com Cc: Adam Dybkowski Date: Mon, 26 Aug 2019 09:44:59 +0200 Message-Id: <20190826074501.10653-2-adamx.dybkowski@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190826074501.10653-1-adamx.dybkowski@intel.com> References: <20190826074501.10653-1-adamx.dybkowski@intel.com> Subject: [dpdk-dev] [PATCH 2/4] compress/qat: add stateful decompression X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds the stateful decompression feature to the DPDK QAT PMD. Signed-off-by: Adam Dybkowski --- drivers/compress/qat/qat_comp.c | 256 +++++++++++++++++++++++++--- drivers/compress/qat/qat_comp.h | 32 ++++ drivers/compress/qat/qat_comp_pmd.c | 166 ++++++++++++++++-- drivers/compress/qat/qat_comp_pmd.h | 2 + 4 files changed, 423 insertions(+), 33 deletions(-) diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c index 835aaa838..a80cd6864 100644 --- a/drivers/compress/qat/qat_comp.c +++ b/drivers/compress/qat/qat_comp.c @@ -27,22 +27,51 @@ qat_comp_build_request(void *in_op, uint8_t *out_msg, struct rte_comp_op *op = in_op; struct qat_comp_op_cookie *cookie = (struct qat_comp_op_cookie *)op_cookie; - struct qat_comp_xform *qat_xform = op->private_xform; - const uint8_t *tmpl = (uint8_t *)&qat_xform->qat_comp_req_tmpl; + struct qat_comp_stream *stream; + struct qat_comp_xform *qat_xform; + const uint8_t *tmpl; struct icp_qat_fw_comp_req *comp_req = (struct icp_qat_fw_comp_req *)out_msg; - if (unlikely(op->op_type != RTE_COMP_OP_STATELESS)) { - QAT_DP_LOG(ERR, "QAT PMD only supports stateless compression " - "operation requests, op (%p) is not a " - "stateless operation.", op); - op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; - return -EINVAL; + if (op->op_type == RTE_COMP_OP_STATEFUL) { + stream = op->stream; + qat_xform = &stream->qat_xform; + if (unlikely(qat_xform->qat_comp_request_type != + QAT_COMP_REQUEST_DECOMPRESS)) { + QAT_DP_LOG(ERR, "QAT PMD does not support stateful compression"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -EINVAL; + } + if (unlikely(stream->op_in_progress)) { + QAT_DP_LOG(ERR, "QAT PMD does not support running multiple stateless operations on the same stream at once"); + op->status = RTE_COMP_OP_STATUS_INVALID_STATE; + return -EINVAL; + } + stream->op_in_progress = 1; + } else { + stream = NULL; + qat_xform = op->private_xform; } + tmpl = (uint8_t *)&qat_xform->qat_comp_req_tmpl; rte_mov128(out_msg, tmpl); comp_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; + if (op->op_type == RTE_COMP_OP_STATEFUL) { + comp_req->comp_pars.req_par_flags = + ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( + (stream->start_of_packet) ? + ICP_QAT_FW_COMP_SOP + : ICP_QAT_FW_COMP_NOT_SOP, + (op->flush_flag == RTE_COMP_FLUSH_FULL || + op->flush_flag == RTE_COMP_FLUSH_FINAL) ? + ICP_QAT_FW_COMP_EOP + : ICP_QAT_FW_COMP_NOT_EOP, + ICP_QAT_FW_COMP_NOT_BFINAL, + ICP_QAT_FW_COMP_NO_CNV, + ICP_QAT_FW_COMP_NO_CNV_RECOVERY); + } + if (likely(qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) { if (unlikely(op->src.length > QAT_FALLBACK_THLD)) { @@ -94,6 +123,9 @@ qat_comp_build_request(void *in_op, uint8_t *out_msg, " for %d elements of SGL", op->m_src->nb_segs); op->status = RTE_COMP_OP_STATUS_ERROR; + /* clear op-in-progress flag */ + if (stream) + stream->op_in_progress = 0; return -ENOMEM; } /* new SGL is valid now */ @@ -111,6 +143,9 @@ qat_comp_build_request(void *in_op, uint8_t *out_msg, if (ret) { QAT_DP_LOG(ERR, "QAT PMD Cannot fill source sgl array"); op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + /* clear op-in-progress flag */ + if (stream) + stream->op_in_progress = 0; return ret; } @@ -129,6 +164,9 @@ qat_comp_build_request(void *in_op, uint8_t *out_msg, " for %d elements of SGL", op->m_dst->nb_segs); op->status = RTE_COMP_OP_STATUS_ERROR; + /* clear op-in-progress flag */ + if (stream) + stream->op_in_progress = 0; return -ENOMEM; } /* new SGL is valid now */ @@ -146,6 +184,9 @@ qat_comp_build_request(void *in_op, uint8_t *out_msg, if (ret) { QAT_DP_LOG(ERR, "QAT PMD Cannot fill dest. sgl array"); op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + /* clear op-in-progress flag */ + if (stream) + stream->op_in_progress = 0; return ret; } @@ -202,12 +243,22 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie, (struct qat_comp_op_cookie *)op_cookie; struct rte_comp_op *rx_op = (struct rte_comp_op *)(uintptr_t) (resp_msg->opaque_data); - struct qat_comp_xform *qat_xform = (struct qat_comp_xform *) - (rx_op->private_xform); + struct qat_comp_stream *stream; + struct qat_comp_xform *qat_xform; int err = resp_msg->comn_resp.comn_status & ((1 << QAT_COMN_RESP_CMP_STATUS_BITPOS) | (1 << QAT_COMN_RESP_XLAT_STATUS_BITPOS)); + if (rx_op->op_type == RTE_COMP_OP_STATEFUL) { + stream = rx_op->stream; + qat_xform = &stream->qat_xform; + /* clear op-in-progress flag */ + stream->op_in_progress = 0; + } else { + stream = NULL; + qat_xform = rx_op->private_xform; + } + #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG QAT_DP_LOG(DEBUG, "Direction: %s", qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS ? @@ -254,7 +305,21 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie, int8_t xlat_err_code = (int8_t)resp_msg->comn_resp.comn_error.xlat_err_code; - if ((cmp_err_code == ERR_CODE_OVERFLOW_ERROR && !xlat_err_code) + /* handle recoverable out-of-buffer condition in stateful */ + /* decompression scenario */ + if (cmp_err_code == ERR_CODE_OVERFLOW_ERROR && !xlat_err_code + && qat_xform->qat_comp_request_type + == QAT_COMP_REQUEST_DECOMPRESS + && rx_op->op_type == RTE_COMP_OP_STATEFUL) { + struct icp_qat_fw_resp_comp_pars *comp_resp = + &resp_msg->comp_resp_pars; + rx_op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; + rx_op->consumed = comp_resp->input_byte_counter; + rx_op->produced = comp_resp->output_byte_counter; + stream->start_of_packet = 0; + } else if ((cmp_err_code == ERR_CODE_OVERFLOW_ERROR + && !xlat_err_code) || (!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR) || @@ -275,6 +340,8 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie, rx_op->status = RTE_COMP_OP_STATUS_SUCCESS; rx_op->consumed = comp_resp->input_byte_counter; rx_op->produced = comp_resp->output_byte_counter; + if (stream) + stream->start_of_packet = 0; if (qat_xform->checksum_type != RTE_COMP_CHECKSUM_NONE) { if (qat_xform->checksum_type == RTE_COMP_CHECKSUM_CRC32) @@ -297,6 +364,12 @@ qat_comp_xform_size(void) return RTE_ALIGN_CEIL(sizeof(struct qat_comp_xform), 8); } +unsigned int +qat_comp_stream_size(void) +{ + return RTE_ALIGN_CEIL(sizeof(struct qat_comp_stream), 8); +} + static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header, enum qat_comp_request_type request) { @@ -317,7 +390,9 @@ static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header, static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, const struct rte_memzone *interm_buff_mz, - const struct rte_comp_xform *xform) + const struct rte_comp_xform *xform, + const struct qat_comp_stream *stream, + enum rte_comp_op_type op_type) { struct icp_qat_fw_comp_req *comp_req; int comp_level, algo; @@ -329,6 +404,18 @@ static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, return -EINVAL; } + if (op_type == RTE_COMP_OP_STATEFUL) { + if (unlikely(stream == NULL)) { + QAT_LOG(ERR, "Stream must be non null for stateful op"); + return -EINVAL; + } + if (unlikely(qat_xform->qat_comp_request_type != + QAT_COMP_REQUEST_DECOMPRESS)) { + QAT_LOG(ERR, "QAT PMD does not support stateful compression"); + return -ENOTSUP; + } + } + if (qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS) { direction = ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS; comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_1; @@ -376,12 +463,43 @@ static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, qat_comp_create_req_hdr(&comp_req->comn_hdr, qat_xform->qat_comp_request_type); - comp_req->comn_hdr.serv_specif_flags = ICP_QAT_FW_COMP_FLAGS_BUILD( - ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); + if (op_type == RTE_COMP_OP_STATEFUL) { + comp_req->comn_hdr.serv_specif_flags = + ICP_QAT_FW_COMP_FLAGS_BUILD( + ICP_QAT_FW_COMP_STATEFUL_SESSION, + ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); + + /* Decompression state registers */ + comp_req->comp_cd_ctrl.comp_state_addr = + stream->state_registers_decomp_phys; + + /* Enable A, B, C, D, and E (CAMs). */ + comp_req->comp_cd_ctrl.ram_bank_flags = + ICP_QAT_FW_COMP_RAM_FLAGS_BUILD( + ICP_QAT_FW_COMP_BANK_DISABLED, /* Bank I */ + ICP_QAT_FW_COMP_BANK_DISABLED, /* Bank H */ + ICP_QAT_FW_COMP_BANK_DISABLED, /* Bank G */ + ICP_QAT_FW_COMP_BANK_DISABLED, /* Bank F */ + ICP_QAT_FW_COMP_BANK_ENABLED, /* Bank E */ + ICP_QAT_FW_COMP_BANK_ENABLED, /* Bank D */ + ICP_QAT_FW_COMP_BANK_ENABLED, /* Bank C */ + ICP_QAT_FW_COMP_BANK_ENABLED, /* Bank B */ + ICP_QAT_FW_COMP_BANK_ENABLED); /* Bank A */ + + comp_req->comp_cd_ctrl.ram_banks_addr = + stream->inflate_context_phys; + } else { + comp_req->comn_hdr.serv_specif_flags = + ICP_QAT_FW_COMP_FLAGS_BUILD( + ICP_QAT_FW_COMP_STATELESS_SESSION, + ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); + } comp_req->cd_pars.sl.comp_slice_cfg_word[0] = ICP_QAT_HW_COMPRESSION_CONFIG_BUILD( @@ -497,7 +615,8 @@ qat_comp_private_xform_create(struct rte_compressdev *dev, qat_xform->checksum_type = xform->decompress.chksum; } - if (qat_comp_create_templates(qat_xform, qat->interm_buff_mz, xform)) { + if (qat_comp_create_templates(qat_xform, qat->interm_buff_mz, xform, + NULL, RTE_COMP_OP_STATELESS)) { QAT_LOG(ERR, "QAT: Problem with setting compression"); return -EINVAL; } @@ -532,3 +651,102 @@ qat_comp_private_xform_free(struct rte_compressdev *dev __rte_unused, } return -EINVAL; } + +/** + * Reset stream state for the next use. + * + * @param stream + * handle of pmd's private stream data + */ +static void +qat_comp_stream_reset(struct qat_comp_stream *stream) +{ + if (stream) { + memset(&stream->qat_xform, 0, sizeof(struct qat_comp_xform)); + stream->start_of_packet = 1; + stream->op_in_progress = 0; + } +} + +/** + * Create driver private stream data. + * + * @param dev + * Compressdev device + * @param xform + * xform data + * @param stream + * ptr where handle of pmd's private stream data should be stored + * @return + * - Returns 0 if private stream structure has been created successfully. + * - Returns -EINVAL if input parameters are invalid. + * - Returns -ENOTSUP if comp device does not support STATEFUL operations. + * - Returns -ENOTSUP if comp device does not support the comp transform. + * - Returns -ENOMEM if the private stream could not be allocated. + */ +int +qat_comp_stream_create(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, + void **stream) +{ + struct qat_comp_dev_private *qat = dev->data->dev_private; + struct qat_comp_stream *ptr; + + if (unlikely(stream == NULL)) { + QAT_LOG(ERR, "QAT: stream parameter is NULL"); + return -EINVAL; + } + if (unlikely(xform->type == RTE_COMP_COMPRESS)) { + QAT_LOG(ERR, "QAT: stateful compression not supported"); + return -ENOTSUP; + } + if (unlikely(qat->streampool == NULL)) { + QAT_LOG(ERR, "QAT device has no stream mempool"); + return -ENOMEM; + } + if (rte_mempool_get(qat->streampool, stream)) { + QAT_LOG(ERR, "Couldn't get object from qat stream mempool"); + return -ENOMEM; + } + + ptr = (struct qat_comp_stream *) *stream; + qat_comp_stream_reset(ptr); + ptr->qat_xform.qat_comp_request_type = QAT_COMP_REQUEST_DECOMPRESS; + ptr->qat_xform.checksum_type = xform->decompress.chksum; + + if (qat_comp_create_templates(&ptr->qat_xform, qat->interm_buff_mz, + xform, ptr, RTE_COMP_OP_STATEFUL)) { + QAT_LOG(ERR, "QAT: problem with creating descriptor template for stream"); + rte_mempool_put(qat->streampool, *stream); + *stream = NULL; + return -EINVAL; + } + + return 0; +} + +/** + * Free driver private stream data. + * + * @param dev + * Compressdev device + * @param stream + * handle of pmd's private stream data + * @return + * - 0 if successful + * - <0 in error cases + * - Returns -EINVAL if input parameters are invalid. + * - Returns -ENOTSUP if comp device does not support STATEFUL operations. + * - Returns -EBUSY if can't free stream as there are inflight operations + */ +int +qat_comp_stream_free(struct rte_compressdev *dev, void *stream) +{ + if (stream) { + struct qat_comp_dev_private *qat = dev->data->dev_private; + qat_comp_stream_reset((struct qat_comp_stream *) stream); + rte_mempool_put(qat->streampool, stream); + return 0; + } + return -EINVAL; +} diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h index 61d12ecf4..2231451a1 100644 --- a/drivers/compress/qat/qat_comp.h +++ b/drivers/compress/qat/qat_comp.h @@ -26,6 +26,16 @@ #define QAT_MIN_OUT_BUF_SIZE 46 +/* maximum size of the state registers */ +#define QAT_STATE_REGISTERS_MAX_SIZE 64 + +/* decompressor context size */ +#define QAT_INFLATE_CONTEXT_SIZE_GEN1 36864 +#define QAT_INFLATE_CONTEXT_SIZE_GEN2 34032 +#define QAT_INFLATE_CONTEXT_SIZE_GEN3 34032 +#define QAT_INFLATE_CONTEXT_SIZE RTE_MAX(RTE_MAX(QAT_INFLATE_CONTEXT_SIZE_GEN1,\ + QAT_INFLATE_CONTEXT_SIZE_GEN2), QAT_INFLATE_CONTEXT_SIZE_GEN3) + enum qat_comp_request_type { QAT_COMP_REQUEST_FIXED_COMP_STATELESS, QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS, @@ -61,6 +71,17 @@ struct qat_comp_xform { enum rte_comp_checksum_type checksum_type; }; +struct qat_comp_stream { + struct qat_comp_xform qat_xform; + void *state_registers_decomp; + phys_addr_t state_registers_decomp_phys; + void *inflate_context; + phys_addr_t inflate_context_phys; + const struct rte_memzone *memzone; + uint8_t start_of_packet; + volatile uint8_t op_in_progress; +}; + int qat_comp_build_request(void *in_op, uint8_t *out_msg, void *op_cookie, enum qat_device_gen qat_dev_gen __rte_unused); @@ -80,5 +101,16 @@ qat_comp_private_xform_free(struct rte_compressdev *dev, void *private_xform); unsigned int qat_comp_xform_size(void); +unsigned int +qat_comp_stream_size(void); + +int +qat_comp_stream_create(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, + void **stream); + +int +qat_comp_stream_free(struct rte_compressdev *dev, void *stream); + #endif #endif diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c index 072647217..05b7dfe77 100644 --- a/drivers/compress/qat/qat_comp_pmd.c +++ b/drivers/compress/qat/qat_comp_pmd.c @@ -9,6 +9,12 @@ #define QAT_PMD_COMP_SGL_DEF_SEGMENTS 16 +struct stream_create_info { + struct qat_comp_dev_private *comp_dev; + int socket_id; + int error; +}; + static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = { {/* COMPRESSION - deflate */ .algo = RTE_COMP_ALGO_DEFLATE, @@ -21,7 +27,8 @@ static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = { RTE_COMP_FF_HUFFMAN_DYNAMIC | RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT | - RTE_COMP_FF_OOP_LB_IN_SGL_OUT, + RTE_COMP_FF_OOP_LB_IN_SGL_OUT | + RTE_COMP_FF_STATEFUL_DECOMPRESSION, .window_size = {.min = 15, .max = 15, .increment = 0} }, {RTE_COMP_ALGO_LIST_END, 0, {0, 0, 0} } }; @@ -315,6 +322,120 @@ qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, return mp; } +static void +qat_comp_stream_init(struct rte_mempool *mp __rte_unused, void *opaque, + void *obj, unsigned int obj_idx) +{ + struct stream_create_info *info = opaque; + struct qat_comp_stream *stream = obj; + char mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *memzone; + struct qat_inter_sgl *ram_banks_desc; + + /* find a memzone for RAM banks */ + snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "%s_%u_rambanks", + info->comp_dev->qat_dev->name, obj_idx); + memzone = rte_memzone_lookup(mz_name); + if (memzone == NULL) { + /* allocate a memzone for compression state and RAM banks */ + memzone = rte_memzone_reserve_aligned(mz_name, + QAT_STATE_REGISTERS_MAX_SIZE + + sizeof(struct qat_inter_sgl) + + QAT_INFLATE_CONTEXT_SIZE, + info->socket_id, + RTE_MEMZONE_IOVA_CONTIG, QAT_64_BYTE_ALIGN); + if (memzone == NULL) { + QAT_LOG(ERR, + "Can't allocate RAM banks for device %s, object %u", + info->comp_dev->qat_dev->name, obj_idx); + info->error = -ENOMEM; + return; + } + } + + /* prepare the buffer list descriptor for RAM banks */ + ram_banks_desc = (struct qat_inter_sgl *) + (((uint8_t *) memzone->addr) + QAT_STATE_REGISTERS_MAX_SIZE); + ram_banks_desc->num_bufs = 1; + ram_banks_desc->buffers[0].len = QAT_INFLATE_CONTEXT_SIZE; + ram_banks_desc->buffers[0].addr = memzone->iova + + QAT_STATE_REGISTERS_MAX_SIZE + + sizeof(struct qat_inter_sgl); + + memset(stream, 0, qat_comp_stream_size()); + stream->memzone = memzone; + stream->state_registers_decomp = memzone->addr; + stream->state_registers_decomp_phys = memzone->iova; + stream->inflate_context = ((uint8_t *) memzone->addr) + + QAT_STATE_REGISTERS_MAX_SIZE; + stream->inflate_context_phys = memzone->iova + + QAT_STATE_REGISTERS_MAX_SIZE; +} + +static void +qat_comp_stream_destroy(struct rte_mempool *mp __rte_unused, + void *opaque __rte_unused, void *obj, + unsigned obj_idx __rte_unused) +{ + struct qat_comp_stream *stream = obj; + + rte_memzone_free(stream->memzone); +} + +static struct rte_mempool * +qat_comp_create_stream_pool(struct qat_comp_dev_private *comp_dev, + int socket_id, + uint32_t num_elements) +{ + char stream_pool_name[RTE_MEMPOOL_NAMESIZE]; + struct rte_mempool *mp; + + snprintf(stream_pool_name, RTE_MEMPOOL_NAMESIZE, + "%s_streams", comp_dev->qat_dev->name); + + QAT_LOG(DEBUG, "streampool: %s", stream_pool_name); + mp = rte_mempool_lookup(stream_pool_name); + + if (mp != NULL) { + QAT_LOG(DEBUG, "streampool already created"); + if (mp->size != num_elements) { + QAT_LOG(DEBUG, "streampool wrong size - delete it"); + rte_mempool_obj_iter(mp, qat_comp_stream_destroy, NULL); + rte_mempool_free(mp); + mp = NULL; + comp_dev->streampool = NULL; + } + } + + if (mp == NULL) { + struct stream_create_info info = { + .comp_dev = comp_dev, + .socket_id = socket_id, + .error = 0 + }; + mp = rte_mempool_create(stream_pool_name, + num_elements, + qat_comp_stream_size(), 0, 0, + NULL, NULL, qat_comp_stream_init, &info, + socket_id, 0); + if (mp == NULL) { + QAT_LOG(ERR, + "Err creating mempool %s w %d elements of size %d", + stream_pool_name, num_elements, + qat_comp_stream_size()); + } else if (info.error) { + rte_mempool_obj_iter(mp, qat_comp_stream_destroy, NULL); + QAT_LOG(ERR, + "Destoying mempool %s as at least one element failed initialisation", + stream_pool_name); + rte_mempool_free(mp); + mp = NULL; + } + } + + return mp; +} + static void _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev) { @@ -330,6 +451,14 @@ _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev) rte_mempool_free(comp_dev->xformpool); comp_dev->xformpool = NULL; } + + /* Free stream pool */ + if (comp_dev->streampool) { + rte_mempool_obj_iter(comp_dev->streampool, + qat_comp_stream_destroy, NULL); + rte_mempool_free(comp_dev->streampool); + comp_dev->streampool = NULL; + } } static int @@ -339,12 +468,6 @@ qat_comp_dev_config(struct rte_compressdev *dev, struct qat_comp_dev_private *comp_dev = dev->data->dev_private; int ret = 0; - if (config->max_nb_streams != 0) { - QAT_LOG(ERR, - "QAT device does not support STATEFUL so max_nb_streams must be 0"); - return -EINVAL; - } - if (RTE_PMD_QAT_COMP_IM_BUFFER_SIZE == 0) { QAT_LOG(WARNING, "RTE_PMD_QAT_COMP_IM_BUFFER_SIZE = 0 in config file, so" @@ -360,13 +483,26 @@ qat_comp_dev_config(struct rte_compressdev *dev, } } - comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev, config, - config->max_nb_priv_xforms); - if (comp_dev->xformpool == NULL) { + if (config->max_nb_priv_xforms) { + comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev, + config, config->max_nb_priv_xforms); + if (comp_dev->xformpool == NULL) { + ret = -ENOMEM; + goto error_out; + } + } else + comp_dev->xformpool = NULL; + + if (config->max_nb_streams) { + comp_dev->streampool = qat_comp_create_stream_pool(comp_dev, + config->socket_id, config->max_nb_streams); + if (comp_dev->streampool == NULL) { + ret = -ENOMEM; + goto error_out; + } + } else + comp_dev->streampool = NULL; - ret = -ENOMEM; - goto error_out; - } return 0; error_out: @@ -508,7 +644,9 @@ static struct rte_compressdev_ops compress_qat_ops = { /* Compression related operations */ .private_xform_create = qat_comp_private_xform_create, - .private_xform_free = qat_comp_private_xform_free + .private_xform_free = qat_comp_private_xform_free, + .stream_create = qat_comp_stream_create, + .stream_free = qat_comp_stream_free }; /* An rte_driver is needed in the registration of the device with compressdev. diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h index b8299d43a..6979de14d 100644 --- a/drivers/compress/qat/qat_comp_pmd.h +++ b/drivers/compress/qat/qat_comp_pmd.h @@ -30,6 +30,8 @@ struct qat_comp_dev_private { /**< The device's memory for intermediate buffers */ struct rte_mempool *xformpool; /**< The device's pool for qat_comp_xforms */ + struct rte_mempool *streampool; + /**< The device's pool for qat_comp_streams */ }; int