@@ -14,6 +14,8 @@
#include "nitrox_qp.h"
#define COMPRESSDEV_NAME_NITROX_PMD compress_nitrox
+#define NITROX_COMP_WINDOW_SIZE_MIN 1
+#define NITROX_COMP_WINDOW_SIZE_MAX 15
#define NITROX_COMP_LEVEL_LOWEST_START 1
#define NITROX_COMP_LEVEL_LOWEST_END 2
#define NITROX_COMP_LEVEL_LOWER_START 3
@@ -49,10 +51,12 @@ static const struct rte_compressdev_capabilities
RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
- RTE_COMP_FF_OOP_LB_IN_SGL_OUT,
+ RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
+ RTE_COMP_FF_STATEFUL_COMPRESSION |
+ RTE_COMP_FF_STATEFUL_DECOMPRESSION,
.window_size = {
- .min = 1,
- .max = 15,
+ .min = NITROX_COMP_WINDOW_SIZE_MIN,
+ .max = NITROX_COMP_WINDOW_SIZE_MAX,
.increment = 1
},
},
@@ -64,6 +68,8 @@ static int nitrox_comp_dev_configure(struct rte_compressdev *dev,
{
struct nitrox_comp_device *comp_dev = dev->data->dev_private;
struct nitrox_device *ndev = comp_dev->ndev;
+ uint32_t xform_cnt;
+ char name[RTE_MEMPOOL_NAMESIZE];
if (config->nb_queue_pairs > ndev->nr_queues) {
NITROX_LOG(ERR, "Invalid queue pairs, max supported %d\n",
@@ -71,21 +77,21 @@ static int nitrox_comp_dev_configure(struct rte_compressdev *dev,
return -EINVAL;
}
- if (config->max_nb_priv_xforms) {
- char xform_name[RTE_MEMPOOL_NAMESIZE];
-
- snprintf(xform_name, sizeof(xform_name), "%s_xform",
- dev->data->name);
- comp_dev->xform_pool = rte_mempool_create(xform_name,
- config->max_nb_priv_xforms,
- sizeof(struct nitrox_comp_xform),
- 0, 0, NULL, NULL, NULL, NULL,
- config->socket_id, 0);
- if (comp_dev->xform_pool == NULL) {
- NITROX_LOG(ERR, "Failed to create xform pool, err %d\n",
- rte_errno);
- return -rte_errno;
- }
+ xform_cnt = config->max_nb_priv_xforms + config->max_nb_streams;
+ if (unlikely(xform_cnt == 0)) {
+ NITROX_LOG(ERR, "Invalid configuration with 0 xforms\n");
+ return -EINVAL;
+ }
+
+ snprintf(name, sizeof(name), "%s_xform", dev->data->name);
+ comp_dev->xform_pool = rte_mempool_create(name,
+ xform_cnt, sizeof(struct nitrox_comp_xform),
+ 0, 0, NULL, NULL, NULL, NULL,
+ config->socket_id, 0);
+ if (comp_dev->xform_pool == NULL) {
+ NITROX_LOG(ERR, "Failed to create xform pool, err %d\n",
+ rte_errno);
+ return -rte_errno;
}
return 0;
@@ -257,7 +263,7 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
void **private_xform)
{
struct nitrox_comp_device *comp_dev = dev->data->dev_private;
- struct nitrox_comp_xform *nitrox_xform;
+ struct nitrox_comp_xform *nxform;
enum rte_comp_checksum_type chksum_type;
int ret;
@@ -271,12 +277,13 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
return -ENOMEM;
}
- nitrox_xform = (struct nitrox_comp_xform *)*private_xform;
+ nxform = (struct nitrox_comp_xform *)*private_xform;
+ memset(nxform, 0, sizeof(*nxform));
if (xform->type == RTE_COMP_COMPRESS) {
enum rte_comp_huffman algo;
int level;
- nitrox_xform->op = NITROX_COMP_OP_COMPRESS;
+ nxform->op = NITROX_COMP_OP_COMPRESS;
if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
NITROX_LOG(ERR, "Only deflate is supported\n");
ret = -ENOTSUP;
@@ -285,11 +292,11 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
algo = xform->compress.deflate.huffman;
if (algo == RTE_COMP_HUFFMAN_DEFAULT)
- nitrox_xform->algo = NITROX_COMP_ALGO_DEFLATE_DEFAULT;
+ nxform->algo = NITROX_COMP_ALGO_DEFLATE_DEFAULT;
else if (algo == RTE_COMP_HUFFMAN_FIXED)
- nitrox_xform->algo = NITROX_COMP_ALGO_DEFLATE_FIXEDHUFF;
+ nxform->algo = NITROX_COMP_ALGO_DEFLATE_FIXEDHUFF;
else if (algo == RTE_COMP_HUFFMAN_DYNAMIC)
- nitrox_xform->algo = NITROX_COMP_ALGO_DEFLATE_DYNHUFF;
+ nxform->algo = NITROX_COMP_ALGO_DEFLATE_DYNHUFF;
else {
NITROX_LOG(ERR, "Invalid deflate algorithm %d\n", algo);
ret = -EINVAL;
@@ -298,19 +305,19 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
level = xform->compress.level;
if (level == RTE_COMP_LEVEL_PMD_DEFAULT) {
- nitrox_xform->level = NITROX_COMP_LEVEL_MEDIUM;
+ nxform->level = NITROX_COMP_LEVEL_MEDIUM;
} else if (level >= NITROX_COMP_LEVEL_LOWEST_START &&
level <= NITROX_COMP_LEVEL_LOWEST_END) {
- nitrox_xform->level = NITROX_COMP_LEVEL_LOWEST;
+ nxform->level = NITROX_COMP_LEVEL_LOWEST;
} else if (level >= NITROX_COMP_LEVEL_LOWER_START &&
level <= NITROX_COMP_LEVEL_LOWER_END) {
- nitrox_xform->level = NITROX_COMP_LEVEL_LOWER;
+ nxform->level = NITROX_COMP_LEVEL_LOWER;
} else if (level >= NITROX_COMP_LEVEL_MEDIUM_START &&
level <= NITROX_COMP_LEVEL_MEDIUM_END) {
- nitrox_xform->level = NITROX_COMP_LEVEL_MEDIUM;
+ nxform->level = NITROX_COMP_LEVEL_MEDIUM;
} else if (level >= NITROX_COMP_LEVEL_BEST_START &&
level <= NITROX_COMP_LEVEL_BEST_END) {
- nitrox_xform->level = NITROX_COMP_LEVEL_BEST;
+ nxform->level = NITROX_COMP_LEVEL_BEST;
} else {
NITROX_LOG(ERR, "Unsupported compression level %d\n",
xform->compress.level);
@@ -320,15 +327,15 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
chksum_type = xform->compress.chksum;
} else if (xform->type == RTE_COMP_DECOMPRESS) {
- nitrox_xform->op = NITROX_COMP_OP_DECOMPRESS;
+ nxform->op = NITROX_COMP_OP_DECOMPRESS;
if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
NITROX_LOG(ERR, "Only deflate is supported\n");
ret = -ENOTSUP;
goto err_exit;
}
- nitrox_xform->algo = NITROX_COMP_ALGO_DEFLATE_DEFAULT;
- nitrox_xform->level = NITROX_COMP_LEVEL_BEST;
+ nxform->algo = NITROX_COMP_ALGO_DEFLATE_DEFAULT;
+ nxform->level = NITROX_COMP_LEVEL_BEST;
chksum_type = xform->decompress.chksum;
} else {
ret = -EINVAL;
@@ -336,11 +343,11 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
}
if (chksum_type == RTE_COMP_CHECKSUM_NONE)
- nitrox_xform->chksum_type = NITROX_CHKSUM_TYPE_NONE;
+ nxform->chksum_type = NITROX_CHKSUM_TYPE_NONE;
else if (chksum_type == RTE_COMP_CHECKSUM_CRC32)
- nitrox_xform->chksum_type = NITROX_CHKSUM_TYPE_CRC32;
+ nxform->chksum_type = NITROX_CHKSUM_TYPE_CRC32;
else if (chksum_type == RTE_COMP_CHECKSUM_ADLER32)
- nitrox_xform->chksum_type = NITROX_CHKSUM_TYPE_ADLER32;
+ nxform->chksum_type = NITROX_CHKSUM_TYPE_ADLER32;
else {
NITROX_LOG(ERR, "Unsupported checksum type %d\n",
chksum_type);
@@ -348,29 +355,104 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev,
goto err_exit;
}
+ nxform->context = NULL;
+ nxform->history_window = NULL;
+ nxform->window_size = 0;
+ nxform->hlen = 0;
+ nxform->exn = 0;
+ nxform->exbits = 0;
+ nxform->bf = true;
return 0;
err_exit:
- memset(nitrox_xform, 0, sizeof(*nitrox_xform));
- rte_mempool_put(comp_dev->xform_pool, nitrox_xform);
+ memset(nxform, 0, sizeof(*nxform));
+ rte_mempool_put(comp_dev->xform_pool, nxform);
return ret;
}
static int nitrox_comp_private_xform_free(struct rte_compressdev *dev,
void *private_xform)
{
- struct nitrox_comp_xform *nitrox_xform = private_xform;
- struct rte_mempool *mp = rte_mempool_from_obj(nitrox_xform);
+ struct nitrox_comp_xform *nxform = private_xform;
+ struct rte_mempool *mp = rte_mempool_from_obj(nxform);
RTE_SET_USED(dev);
- if (nitrox_xform == NULL)
+ if (unlikely(nxform == NULL))
return -EINVAL;
- memset(nitrox_xform, 0, sizeof(*nitrox_xform));
- mp = rte_mempool_from_obj(nitrox_xform);
- rte_mempool_put(mp, nitrox_xform);
+ memset(nxform, 0, sizeof(*nxform));
+ mp = rte_mempool_from_obj(nxform);
+ rte_mempool_put(mp, nxform);
return 0;
}
+static int nitrox_comp_stream_free(struct rte_compressdev *dev, void *stream)
+{
+ struct nitrox_comp_xform *nxform = stream;
+
+ if (unlikely(nxform == NULL))
+ return -EINVAL;
+
+ rte_free(nxform->history_window);
+ nxform->history_window = NULL;
+ rte_free(nxform->context);
+ nxform->context = NULL;
+ return nitrox_comp_private_xform_free(dev, stream);
+}
+
+static int nitrox_comp_stream_create(struct rte_compressdev *dev,
+ const struct rte_comp_xform *xform, void **stream)
+{
+ int err;
+ struct nitrox_comp_xform *nxform;
+ struct nitrox_comp_device *comp_dev = dev->data->dev_private;
+
+ err = nitrox_comp_private_xform_create(dev, xform, stream);
+ if (unlikely(err))
+ return err;
+
+ nxform = *stream;
+ if (xform->type == RTE_COMP_COMPRESS) {
+ uint8_t window_size = xform->compress.window_size;
+
+ if (unlikely(window_size < NITROX_COMP_WINDOW_SIZE_MIN ||
+ window_size > NITROX_COMP_WINDOW_SIZE_MAX)) {
+ NITROX_LOG(ERR, "Invalid window size %d\n",
+ window_size);
+ return -EINVAL;
+ }
+
+ if (window_size == NITROX_COMP_WINDOW_SIZE_MAX)
+ nxform->window_size = NITROX_CONSTANTS_MAX_SEARCH_DEPTH;
+ else
+ nxform->window_size = RTE_BIT32(window_size);
+ } else {
+ nxform->window_size = NITROX_DEFAULT_DEFLATE_SEARCH_DEPTH;
+ }
+
+ nxform->history_window = rte_zmalloc_socket(NULL, nxform->window_size,
+ 8, comp_dev->xform_pool->socket_id);
+ if (unlikely(nxform->history_window == NULL)) {
+ err = -ENOMEM;
+ goto err_exit;
+ }
+
+ if (xform->type == RTE_COMP_COMPRESS)
+ return 0;
+
+ nxform->context = rte_zmalloc_socket(NULL,
+ NITROX_DECOMP_CTX_SIZE, 8,
+ comp_dev->xform_pool->socket_id);
+ if (unlikely(nxform->context == NULL)) {
+ err = -ENOMEM;
+ goto err_exit;
+ }
+
+ return 0;
+err_exit:
+ nitrox_comp_stream_free(dev, *stream);
+ return err;
+}
+
static int nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_comp_op *op)
{
struct nitrox_softreq *sr;
@@ -385,8 +467,12 @@ static int nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_comp_op *op)
return err;
}
- nitrox_qp_enqueue(qp, nitrox_comp_instr_addr(sr), sr);
- return 0;
+ if (op->status == RTE_COMP_OP_STATUS_SUCCESS)
+ err = nitrox_qp_enqueue_sr(qp, sr);
+ else
+ nitrox_qp_enqueue(qp, nitrox_comp_instr_addr(sr), sr);
+
+ return err;
}
static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair,
@@ -396,6 +482,7 @@ static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair,
struct nitrox_qp *qp = queue_pair;
uint16_t free_slots = 0;
uint16_t cnt = 0;
+ uint16_t dbcnt = 0;
bool err = false;
free_slots = nitrox_qp_free_count(qp);
@@ -407,9 +494,12 @@ static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair,
err = true;
break;
}
+
+ if (ops[cnt]->status != RTE_COMP_OP_STATUS_SUCCESS)
+ dbcnt++;
}
- nitrox_ring_dbell(qp, cnt);
+ nitrox_ring_dbell(qp, dbcnt);
qp->stats.enqueued_count += cnt;
if (unlikely(err))
qp->stats.enqueue_err_count++;
@@ -472,8 +562,8 @@ static struct rte_compressdev_ops nitrox_compressdev_ops = {
.private_xform_create = nitrox_comp_private_xform_create,
.private_xform_free = nitrox_comp_private_xform_free,
- .stream_create = NULL,
- .stream_free = NULL
+ .stream_create = nitrox_comp_stream_create,
+ .stream_free = nitrox_comp_stream_free,
};
int
@@ -531,4 +621,3 @@ nitrox_comp_pmd_destroy(struct nitrox_device *ndev)
ndev->comp_dev = NULL;
return 0;
}
-
@@ -5,11 +5,13 @@
#include <rte_compressdev_pmd.h>
#include <rte_errno.h>
#include <rte_malloc.h>
+#include <rte_hexdump.h>
#include "nitrox_comp_reqmgr.h"
#include "nitrox_logs.h"
#include "rte_comp.h"
+#define NITROX_INSTR_BUFFER_DEBUG 0
#define NITROX_ZIP_SGL_COUNT 16
#define NITROX_ZIP_MAX_ZPTRS 2048
#define NITROX_ZIP_MAX_DATASIZE ((1 << 24) - 1)
@@ -307,10 +309,222 @@ struct nitrox_softreq {
struct rte_comp_op *op;
struct nitrox_sgtable src;
struct nitrox_sgtable dst;
- struct nitrox_comp_xform xform;
uint64_t timeout;
};
+#if NITROX_INSTR_BUFFER_DEBUG
+static void nitrox_dump_databuf(const char *name, struct rte_mbuf *m,
+ uint32_t off, uint32_t datalen)
+{
+ uint32_t mlen;
+
+ if (!rte_log_can_log(nitrox_logtype, RTE_LOG_DEBUG))
+ return;
+
+ for (; m && off > rte_pktmbuf_data_len(m); m = m->next)
+ off -= rte_pktmbuf_data_len(m);
+
+ mlen = rte_pktmbuf_data_len(m) - off;
+ if (datalen <= mlen)
+ mlen = datalen;
+
+ rte_hexdump(stderr, name, rte_pktmbuf_mtod_offset(m, char *, off),
+ mlen);
+ for (m = m->next; m && datalen; m = m->next) {
+ mlen = rte_pktmbuf_data_len(m) < datalen ?
+ rte_pktmbuf_data_len(m) : datalen;
+ rte_hexdump(stderr, name, rte_pktmbuf_mtod(m, char *), mlen);
+ }
+
+ fprintf(stderr, "\n");
+}
+
+static void nitrox_dump_zip_instr(struct nitrox_zip_instr *instr,
+ union nitrox_zip_zptr *hptr_arr,
+ union nitrox_zip_zptr *iptr_arr,
+ union nitrox_zip_zptr *optr_arr)
+{
+ uint64_t value;
+ int i = 0;
+
+ if (!rte_log_can_log(nitrox_logtype, RTE_LOG_DEBUG))
+ return;
+
+ fprintf(stderr, "\nZIP instruction..(%p)\n", instr);
+ fprintf(stderr, "\tWORD0 = 0x%016"PRIx64"\n", instr->w0.u64);
+ fprintf(stderr, "\t\tTOL = %d\n", instr->w0.tol);
+ fprintf(stderr, "\t\tEXNUM = %d\n", instr->w0.exn);
+ fprintf(stderr, "\t\tEXBITS = %x\n", instr->w0.exbits);
+ fprintf(stderr, "\t\tCA = %d\n", instr->w0.ca);
+ fprintf(stderr, "\t\tSF = %d\n", instr->w0.sf);
+ fprintf(stderr, "\t\tSS = %d\n", instr->w0.ss);
+ fprintf(stderr, "\t\tCC = %d\n", instr->w0.cc);
+ fprintf(stderr, "\t\tEF = %d\n", instr->w0.ef);
+ fprintf(stderr, "\t\tBF = %d\n", instr->w0.bf);
+ fprintf(stderr, "\t\tCO = %d\n", instr->w0.co);
+ fprintf(stderr, "\t\tDS = %d\n", instr->w0.ds);
+ fprintf(stderr, "\t\tDG = %d\n", instr->w0.dg);
+ fprintf(stderr, "\t\tHG = %d\n", instr->w0.hg);
+ fprintf(stderr, "\n");
+
+ fprintf(stderr, "\tWORD1 = 0x%016"PRIx64"\n", instr->w1.u64);
+ fprintf(stderr, "\t\tHL = %d\n", instr->w1.hl);
+ fprintf(stderr, "\t\tADLERCRC32 = 0x%08x\n", instr->w1.adlercrc32);
+ fprintf(stderr, "\n");
+
+ value = instr->w2.cptr;
+ fprintf(stderr, "\tWORD2 = 0x%016"PRIx64"\n", instr->w2.u64);
+ fprintf(stderr, "\t\tCPTR = 0x%11"PRIx64"\n", value);
+ fprintf(stderr, "\n");
+
+ value = instr->w3.hptr;
+ fprintf(stderr, "\tWORD3 = 0x%016"PRIx64"\n", instr->w3.u64);
+ fprintf(stderr, "\t\tHLEN = %d\n", instr->w3.hlen);
+ fprintf(stderr, "\t\tHPTR = 0x%11"PRIx64"\n", value);
+
+ if (instr->w0.hg && hptr_arr) {
+ for (i = 0; i < instr->w3.hlen; i++) {
+ value = hptr_arr[i].s.addr;
+ fprintf(stderr, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n",
+ i, hptr_arr[i].s.length, value);
+ }
+ }
+
+ fprintf(stderr, "\n");
+
+ value = instr->w4.iptr;
+ fprintf(stderr, "\tWORD4 = 0x%016"PRIx64"\n", instr->w4.u64);
+ fprintf(stderr, "\t\tILEN = %d\n", instr->w4.ilen);
+ fprintf(stderr, "\t\tIPTR = 0x%11"PRIx64"\n", value);
+ if (instr->w0.dg && iptr_arr) {
+ for (i = 0; i < instr->w4.ilen; i++) {
+ value = iptr_arr[i].s.addr;
+ fprintf(stderr, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n",
+ i, iptr_arr[i].s.length, value);
+ }
+ }
+
+ fprintf(stderr, "\n");
+
+ value = instr->w5.optr;
+ fprintf(stderr, "\tWORD5 = 0x%016"PRIx64"\n", instr->w5.u64);
+ fprintf(stderr, "\t\t OLEN = %d\n", instr->w5.olen);
+ fprintf(stderr, "\t\t OPTR = 0x%11"PRIx64"\n", value);
+ if (instr->w0.ds && optr_arr) {
+ for (i = 0; i < instr->w5.olen; i++) {
+ value = optr_arr[i].s.addr;
+ fprintf(stderr, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n",
+ i, optr_arr[i].s.length, value);
+ }
+ }
+
+ fprintf(stderr, "\n");
+
+ value = instr->w6.rptr;
+ fprintf(stderr, "\tWORD6 = 0x%016"PRIx64"\n", instr->w6.u64);
+ fprintf(stderr, "\t\tRPTR = 0x%11"PRIx64"\n", value);
+ fprintf(stderr, "\n");
+
+ fprintf(stderr, "\tWORD7 = 0x%016"PRIx64"\n", instr->w7.u64);
+ fprintf(stderr, "\t\tGRP = %x\n", instr->w7.grp);
+ fprintf(stderr, "\t\tADDR_MSB = 0x%5x\n", instr->w7.addr_msb);
+ fprintf(stderr, "\n");
+}
+
+static void nitrox_dump_zip_result(struct nitrox_zip_instr *instr,
+ struct nitrox_zip_result *result)
+{
+ if (!rte_log_can_log(nitrox_logtype, RTE_LOG_DEBUG))
+ return;
+
+ fprintf(stderr, "ZIP result..(instr %p)\n", instr);
+ fprintf(stderr, "\tWORD0 = 0x%016"PRIx64"\n", result->w0.u64);
+ fprintf(stderr, "\t\tCRC32 = 0x%8x\n", result->w0.crc32);
+ fprintf(stderr, "\t\tADLER32 = 0x%8x\n", result->w0.adler32);
+ fprintf(stderr, "\n");
+
+ fprintf(stderr, "\tWORD1 = 0x%016"PRIx64"\n", result->w1.u64);
+ fprintf(stderr, "\t\tTBYTESWRITTEN = %u\n", result->w1.tbyteswritten);
+ fprintf(stderr, "\t\tTBYTESREAD = %u\n", result->w1.tbytesread);
+ fprintf(stderr, "\n");
+
+ fprintf(stderr, "\tWORD2 = 0x%016"PRIx64"\n", result->w2.u64);
+ fprintf(stderr, "\t\tTBITS = %u\n", result->w2.tbits);
+ fprintf(stderr, "\t\tEXN = %d\n", result->w2.exn);
+ fprintf(stderr, "\t\tEBITS = %x\n", result->w2.exbits);
+ fprintf(stderr, "\t\tEF = %d\n", result->w2.ef);
+ fprintf(stderr, "\t\tCOMPCODE = 0x%2x\n", result->w2.compcode);
+ fprintf(stderr, "\n");
+}
+#else
+#define nitrox_dump_databuf(name, m, off, datalen)
+#define nitrox_dump_zip_instr(instr, hptr_arr, iptr_arr, optr_arr)
+#define nitrox_dump_zip_result(instr, result)
+#endif
+
+static int handle_zero_length_compression(struct nitrox_softreq *sr,
+ struct nitrox_comp_xform *xform)
+{
+ union {
+ uint32_t num;
+ uint8_t bytes[4];
+ } fblk;
+ uint32_t dstlen, rlen;
+ struct rte_mbuf *m;
+ uint32_t off;
+ uint32_t mlen;
+ uint32_t i = 0;
+ uint8_t *ptr;
+
+ fblk.num = xform->exn ? (xform->exbits & 0x7F) : 0;
+ fblk.num |= (0x3 << xform->exn);
+ memset(&sr->zip_res, 0, sizeof(sr->zip_res));
+ sr->zip_res.w1.tbytesread = xform->hlen;
+ sr->zip_res.w1.tbyteswritten = 2;
+ sr->zip_res.w2.ef = 1;
+ if (xform->exn == 7)
+ sr->zip_res.w1.tbyteswritten++;
+
+ rlen = sr->zip_res.w1.tbyteswritten;
+ dstlen = rte_pktmbuf_pkt_len(sr->op->m_dst) - sr->op->dst.offset;
+ if (unlikely(dstlen < rlen))
+ return -EIO;
+
+ off = sr->op->dst.offset;
+ for (m = sr->op->m_dst; m && off > rte_pktmbuf_data_len(m); m = m->next)
+ off -= rte_pktmbuf_data_len(m);
+
+ if (unlikely(!m))
+ return -EIO;
+
+ mlen = rte_pktmbuf_data_len(m) - off;
+ if (rlen <= mlen)
+ mlen = rlen;
+
+ ptr = rte_pktmbuf_mtod_offset(m, uint8_t *, off);
+ memcpy(ptr, fblk.bytes, mlen);
+ i += mlen;
+ rlen -= mlen;
+ for (m = m->next; m && rlen; m = m->next) {
+ mlen = rte_pktmbuf_data_len(m) < rlen ?
+ rte_pktmbuf_data_len(m) : rlen;
+ ptr = rte_pktmbuf_mtod(m, uint8_t *);
+ memcpy(ptr, &fblk.bytes[i], mlen);
+ i += mlen;
+ rlen -= mlen;
+ }
+
+ if (unlikely(rlen != 0))
+ return -EIO;
+
+ sr->zip_res.w2.compcode = NITROX_CC_SUCCESS;
+ sr->op->status = RTE_COMP_OP_STATUS_SUCCESS;
+ sr->zip_res.w0.u64 = rte_cpu_to_be_64(sr->zip_res.w0.u64);
+ sr->zip_res.w1.u64 = rte_cpu_to_be_64(sr->zip_res.w1.u64);
+ sr->zip_res.w2.u64 = rte_cpu_to_be_64(sr->zip_res.w2.u64);
+ return 0;
+}
+
static int create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl,
struct rte_mbuf *mbuf, uint32_t off,
uint32_t datalen, uint8_t extra_segs,
@@ -398,10 +612,12 @@ static int create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl,
return 0;
}
-static int softreq_init(struct nitrox_softreq *sr)
+static int softreq_init(struct nitrox_softreq *sr,
+ struct nitrox_comp_xform *xform)
{
struct rte_mempool *mp;
int err;
+ bool need_decomp_threshold;
mp = rte_mempool_from_obj(sr);
if (unlikely(mp == NULL))
@@ -413,15 +629,17 @@ static int softreq_init(struct nitrox_softreq *sr)
if (unlikely(err))
return err;
+ need_decomp_threshold = (sr->op->op_type == RTE_COMP_OP_STATELESS &&
+ xform->op == NITROX_COMP_OP_DECOMPRESS);
err = create_sglist_from_mbuf(&sr->dst, sr->op->m_dst,
sr->op->dst.offset,
rte_pktmbuf_pkt_len(sr->op->m_dst) - sr->op->dst.offset,
- (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) ? 1 : 0,
+ need_decomp_threshold ? 1 : 0,
mp->socket_id);
if (unlikely(err))
return err;
- if (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) {
+ if (need_decomp_threshold) {
struct nitrox_zip_iova_addr zip_addr;
int i;
@@ -459,12 +677,12 @@ static void nitrox_zip_instr_to_b64(struct nitrox_softreq *sr)
instr->w7.u64 = rte_cpu_to_be_64(instr->w7.u64);
}
-static int process_zip_stateless(struct nitrox_softreq *sr)
+static int process_zip_request(struct nitrox_softreq *sr)
{
struct nitrox_zip_instr *instr;
struct nitrox_comp_xform *xform;
struct nitrox_zip_iova_addr zip_addr;
- uint64_t iptr_msb, optr_msb, rptr_msb;
+ uint64_t iptr_msb, optr_msb, rptr_msb, cptr_msb, hptr_msb;
int err;
xform = sr->op->private_xform;
@@ -473,7 +691,14 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
return -EINVAL;
}
- if (unlikely(xform->op == NITROX_COMP_OP_COMPRESS &&
+ if (unlikely(sr->op->op_type == RTE_COMP_OP_STATEFUL &&
+ xform->op == NITROX_COMP_OP_COMPRESS &&
+ sr->op->flush_flag == RTE_COMP_FLUSH_FINAL &&
+ sr->op->src.length == 0))
+ return handle_zero_length_compression(sr, xform);
+
+ if (unlikely(sr->op->op_type == RTE_COMP_OP_STATELESS &&
+ xform->op == NITROX_COMP_OP_COMPRESS &&
sr->op->flush_flag != RTE_COMP_FLUSH_FULL &&
sr->op->flush_flag != RTE_COMP_FLUSH_FINAL)) {
NITROX_LOG(ERR, "Invalid flush flag %d in stateless op\n",
@@ -481,8 +706,7 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
return -EINVAL;
}
- sr->xform = *xform;
- err = softreq_init(sr);
+ err = softreq_init(sr, xform);
if (unlikely(err))
return err;
@@ -490,10 +714,11 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
memset(instr, 0, sizeof(*instr));
/* word 0 */
instr->w0.tol = sr->dst.total_bytes;
- instr->w0.exn = 0;
- instr->w0.exbits = 0;
+ instr->w0.exn = xform->exn;
+ instr->w0.exbits = xform->exbits;
instr->w0.ca = 0;
if (xform->op == NITROX_COMP_OP_DECOMPRESS ||
+ sr->op->flush_flag == RTE_COMP_FLUSH_SYNC ||
sr->op->flush_flag == RTE_COMP_FLUSH_FULL)
instr->w0.sf = 1;
else
@@ -501,13 +726,12 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
instr->w0.ss = xform->level;
instr->w0.cc = xform->algo;
- if (xform->op == NITROX_COMP_OP_COMPRESS &&
- sr->op->flush_flag == RTE_COMP_FLUSH_FINAL)
+ if (sr->op->flush_flag == RTE_COMP_FLUSH_FINAL)
instr->w0.ef = 1;
else
instr->w0.ef = 0;
- instr->w0.bf = 1;
+ instr->w0.bf = xform->bf;
instr->w0.co = xform->op;
if (sr->dst.filled_sgls > 1)
instr->w0.ds = 1;
@@ -522,8 +746,11 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
instr->w0.hg = 0;
/* word 1 */
- instr->w1.hl = 0;
- if (sr->op->input_chksum != 0)
+ instr->w1.hl = xform->hlen;
+ if (sr->op->op_type == RTE_COMP_OP_STATEFUL && !xform->bf)
+ instr->w1.adlercrc32 = xform->chksum;
+ else if (sr->op->op_type == RTE_COMP_OP_STATELESS &&
+ sr->op->input_chksum != 0)
instr->w1.adlercrc32 = sr->op->input_chksum;
else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32)
instr->w1.adlercrc32 = 1;
@@ -531,11 +758,23 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
instr->w1.adlercrc32 = 0;
/* word 2 */
- instr->w2.cptr = 0;
+ if (xform->context)
+ zip_addr.u64 = rte_malloc_virt2iova(xform->context);
+ else
+ zip_addr.u64 = 0;
+
+ instr->w2.cptr = zip_addr.zda.addr;
+ cptr_msb = zip_addr.zda.addr_msb;
/* word 3 */
- instr->w3.hlen = 0;
- instr->w3.hptr = 0;
+ instr->w3.hlen = xform->hlen;
+ if (xform->history_window)
+ zip_addr.u64 = rte_malloc_virt2iova(xform->history_window);
+ else
+ zip_addr.u64 = 0;
+
+ instr->w3.hptr = zip_addr.zda.addr;
+ hptr_msb = zip_addr.zda.addr_msb;
/* word 4 */
if (sr->src.filled_sgls == 1) {
@@ -568,7 +807,9 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
instr->w6.rptr = zip_addr.zda.addr;
rptr_msb = zip_addr.zda.addr_msb;
- if (iptr_msb != optr_msb || iptr_msb != rptr_msb) {
+ if (unlikely(iptr_msb != optr_msb || iptr_msb != rptr_msb ||
+ (xform->history_window && (iptr_msb != hptr_msb)) ||
+ (xform->context && (iptr_msb != cptr_msb)))) {
NITROX_LOG(ERR, "addr_msb is not same for all addresses\n");
return -ENOTSUP;
}
@@ -577,32 +818,20 @@ static int process_zip_stateless(struct nitrox_softreq *sr)
instr->w7.addr_msb = iptr_msb;
instr->w7.grp = 0;
+ nitrox_dump_zip_instr(instr, NULL, sr->src.sgl, sr->dst.sgl);
+ nitrox_dump_databuf("IN", sr->op->m_src, sr->op->src.offset,
+ sr->op->src.length);
nitrox_zip_instr_to_b64(sr);
return 0;
}
-static int process_zip_request(struct nitrox_softreq *sr)
-{
- int err;
-
- switch (sr->op->op_type) {
- case RTE_COMP_OP_STATELESS:
- err = process_zip_stateless(sr);
- break;
- default:
- err = -EINVAL;
- break;
- }
-
- return err;
-}
-
int
nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr)
{
int err;
sr->op = op;
+ sr->op->status = RTE_COMP_OP_STATUS_NOT_PROCESSED;
err = process_zip_request(sr);
if (unlikely(err))
goto err_exit;
@@ -628,55 +857,239 @@ static struct nitrox_zip_result zip_result_to_cpu64(struct nitrox_zip_result *r)
return out_res;
}
-int
-nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op)
+static int post_process_zip_stateless(struct nitrox_softreq *sr,
+ struct nitrox_comp_xform *xform,
+ struct nitrox_zip_result *zip_res)
{
- struct nitrox_zip_result zip_res;
int output_unused_bytes;
- int err = 0;
-
- zip_res = zip_result_to_cpu64(&sr->zip_res);
- if (zip_res.w2.compcode == NITROX_CC_NOTDONE) {
- if (rte_get_timer_cycles() >= sr->timeout) {
- NITROX_LOG(ERR, "Op timedout\n");
- sr->op->status = RTE_COMP_OP_STATUS_ERROR;
- err = -ETIMEDOUT;
- goto exit;
- } else {
- return -EAGAIN;
- }
- }
- if (unlikely(zip_res.w2.compcode != NITROX_CC_SUCCESS)) {
+ if (unlikely(zip_res->w2.compcode != NITROX_CC_SUCCESS)) {
struct rte_comp_op *op = sr->op;
- NITROX_LOG(ERR, "Op dequeue error 0x%x\n",
- zip_res.w2.compcode);
- if (zip_res.w2.compcode == NITROX_CC_STOP ||
- zip_res.w2.compcode == NITROX_CC_DTRUNC)
+ NITROX_LOG(ERR, "Dequeue error 0x%x\n",
+ zip_res->w2.compcode);
+ if (zip_res->w2.compcode == NITROX_CC_STOP ||
+ zip_res->w2.compcode == NITROX_CC_DTRUNC)
op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
else
op->status = RTE_COMP_OP_STATUS_ERROR;
op->consumed = 0;
op->produced = 0;
- err = -EFAULT;
- goto exit;
+ return -EFAULT;
}
- output_unused_bytes = sr->dst.total_bytes - zip_res.w1.tbyteswritten;
- if (unlikely(sr->xform.op == NITROX_COMP_OP_DECOMPRESS &&
+ output_unused_bytes = sr->dst.total_bytes - zip_res->w1.tbyteswritten;
+ if (unlikely(xform->op == NITROX_COMP_OP_DECOMPRESS &&
output_unused_bytes < NITROX_ZIP_MAX_ONFSIZE)) {
NITROX_LOG(ERR, "TOL %d, Total bytes written %d\n",
- sr->dst.total_bytes, zip_res.w1.tbyteswritten);
+ sr->dst.total_bytes, zip_res->w1.tbyteswritten);
sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
sr->op->consumed = 0;
sr->op->produced = sr->dst.total_bytes - NITROX_ZIP_MAX_ONFSIZE;
- err = -EIO;
- goto exit;
+ return -EIO;
+ }
+
+ if (xform->chksum_type == NITROX_CHKSUM_TYPE_CRC32)
+ sr->op->output_chksum = zip_res->w0.crc32;
+ else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32)
+ sr->op->output_chksum = zip_res->w0.adler32;
+
+ sr->op->consumed = RTE_MIN(sr->op->src.length,
+ (uint32_t)zip_res->w1.tbytesread);
+ sr->op->produced = zip_res->w1.tbyteswritten;
+ sr->op->status = RTE_COMP_OP_STATUS_SUCCESS;
+ return 0;
+}
+
+static int update_history(struct rte_mbuf *mbuf, uint32_t off, uint16_t datalen,
+ uint8_t *dst)
+{
+ struct rte_mbuf *m;
+ uint32_t mlen;
+ uint16_t copied = 0;
+
+ for (m = mbuf; m && off > rte_pktmbuf_data_len(m); m = m->next)
+ off -= rte_pktmbuf_data_len(m);
+
+ if (unlikely(!m)) {
+ NITROX_LOG(ERR, "Failed to update history. Invalid mbuf\n");
+ return -EINVAL;
+ }
+
+ mlen = rte_pktmbuf_data_len(m) - off;
+ if (datalen <= mlen)
+ mlen = datalen;
+
+ memcpy(&dst[copied], rte_pktmbuf_mtod_offset(m, char *, off), mlen);
+ copied += mlen;
+ datalen -= mlen;
+ for (m = m->next; m && datalen; m = m->next) {
+ mlen = rte_pktmbuf_data_len(m) < datalen ?
+ rte_pktmbuf_data_len(m) : datalen;
+ memcpy(&dst[copied], rte_pktmbuf_mtod(m, char *), mlen);
+ copied += mlen;
+ datalen -= mlen;
+ }
+
+ if (unlikely(datalen != 0)) {
+ NITROX_LOG(ERR, "Failed to update history. Invalid datalen\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void reset_nitrox_xform(struct nitrox_comp_xform *xform)
+{
+ xform->hlen = 0;
+ xform->exn = 0;
+ xform->exbits = 0;
+ xform->bf = true;
+}
+
+static int post_process_zip_stateful(struct nitrox_softreq *sr,
+ struct nitrox_comp_xform *xform,
+ struct nitrox_zip_result *zip_res)
+{
+ uint32_t bytesread = 0;
+ uint32_t chksum = 0;
+
+ if (unlikely(zip_res->w2.compcode == NITROX_CC_DTRUNC)) {
+ sr->op->consumed = 0;
+ sr->op->produced = 0;
+ xform->hlen = 0;
+ sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ NITROX_LOG(ERR, "Dequeue compress DTRUNC error\n");
+ return 0;
+ } else if (unlikely(zip_res->w2.compcode == NITROX_CC_STOP)) {
+ sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ NITROX_LOG(NOTICE, "Dequeue decompress dynamic STOP\n");
+ } else if (zip_res->w2.compcode == NITROX_CC_SUCCESS) {
+ sr->op->status = RTE_COMP_OP_STATUS_SUCCESS;
+ } else {
+ xform->hlen = 0;
+ xform->exn = 0;
+ xform->exbits = 0;
+ xform->bf = true;
+ sr->op->status = RTE_COMP_OP_STATUS_ERROR;
+ NITROX_LOG(ERR, "Dequeue error 0x%x\n",
+ zip_res->w2.compcode);
+ return -EFAULT;
+ }
+
+ if (xform->op == NITROX_COMP_OP_COMPRESS) {
+ if (zip_res->w1.tbytesread < xform->hlen) {
+ NITROX_LOG(ERR, "Invalid bytesread\n");
+ reset_nitrox_xform(xform);
+ sr->op->status = RTE_COMP_OP_STATUS_ERROR;
+ return -EFAULT;
+ }
+
+ bytesread = zip_res->w1.tbytesread - xform->hlen;
+ } else {
+ bytesread = RTE_MIN(sr->op->src.length,
+ (uint32_t)zip_res->w1.tbytesread);
+ }
+
+ if ((xform->op == NITROX_COMP_OP_COMPRESS &&
+ (sr->op->flush_flag == RTE_COMP_FLUSH_NONE ||
+ sr->op->flush_flag == RTE_COMP_FLUSH_SYNC)) ||
+ (xform->op == NITROX_COMP_OP_DECOMPRESS && !zip_res->w2.ef)) {
+ struct rte_mbuf *mbuf;
+ uint32_t pktlen, m_off;
+ int err;
+
+ if (xform->op == NITROX_COMP_OP_COMPRESS) {
+ mbuf = sr->op->m_src;
+ pktlen = bytesread;
+ m_off = sr->op->src.offset;
+ } else {
+ mbuf = sr->op->m_dst;
+ pktlen = zip_res->w1.tbyteswritten;
+ m_off = sr->op->dst.offset;
+ }
+
+ if (pktlen >= xform->window_size) {
+ m_off += pktlen - xform->window_size;
+ err = update_history(mbuf, m_off, xform->window_size,
+ xform->history_window);
+ xform->hlen = xform->window_size;
+ } else if ((xform->hlen + pktlen) <= xform->window_size) {
+ err = update_history(mbuf, m_off, pktlen,
+ &xform->history_window[xform->hlen]);
+ xform->hlen += pktlen;
+ } else {
+ uint16_t shift_off, shift_len;
+
+ shift_off = pktlen + xform->hlen - xform->window_size;
+ shift_len = xform->hlen - shift_off;
+ memmove(xform->history_window,
+ &xform->history_window[shift_off],
+ shift_len);
+ err = update_history(mbuf, m_off, pktlen,
+ &xform->history_window[shift_len]);
+ xform->hlen = xform->window_size;
+
+ }
+
+ if (unlikely(err)) {
+ sr->op->status = RTE_COMP_OP_STATUS_ERROR;
+ return err;
+ }
+
+ if (xform->op == NITROX_COMP_OP_COMPRESS) {
+ xform->exn = zip_res->w2.exn;
+ xform->exbits = zip_res->w2.exbits;
+ }
+
+ xform->bf = false;
+ } else {
+ reset_nitrox_xform(xform);
}
- if (sr->xform.op == NITROX_COMP_OP_COMPRESS &&
+ if (xform->chksum_type == NITROX_CHKSUM_TYPE_CRC32)
+ chksum = zip_res->w0.crc32;
+ else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32)
+ chksum = zip_res->w0.adler32;
+
+ if (xform->bf)
+ sr->op->output_chksum = chksum;
+ else
+ xform->chksum = chksum;
+
+ sr->op->consumed = bytesread;
+ sr->op->produced = zip_res->w1.tbyteswritten;
+ return 0;
+}
+
+int
+nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op)
+{
+ struct nitrox_zip_result zip_res;
+ struct nitrox_comp_xform *xform;
+ int err = 0;
+
+ zip_res = zip_result_to_cpu64(&sr->zip_res);
+ if (zip_res.w2.compcode == NITROX_CC_NOTDONE) {
+ if (rte_get_timer_cycles() >= sr->timeout) {
+ NITROX_LOG(ERR, "Op timedout\n");
+ sr->op->status = RTE_COMP_OP_STATUS_ERROR;
+ err = -ETIMEDOUT;
+ goto exit;
+ } else {
+ return -EAGAIN;
+ }
+ }
+
+ xform = sr->op->private_xform;
+ if (sr->op->op_type == RTE_COMP_OP_STATELESS)
+ err = post_process_zip_stateless(sr, xform, &zip_res);
+ else
+ err = post_process_zip_stateful(sr, xform, &zip_res);
+
+ if (sr->op->status == RTE_COMP_OP_STATUS_SUCCESS &&
+ xform->op == NITROX_COMP_OP_COMPRESS &&
sr->op->flush_flag == RTE_COMP_FLUSH_FINAL &&
zip_res.w2.exn) {
uint32_t datalen = zip_res.w1.tbyteswritten;
@@ -696,17 +1109,11 @@ nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op)
*last_byte = zip_res.w2.exbits & 0xFF;
}
- sr->op->consumed = zip_res.w1.tbytesread;
- sr->op->produced = zip_res.w1.tbyteswritten;
- if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_CRC32)
- sr->op->output_chksum = zip_res.w0.crc32;
- else if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_ADLER32)
- sr->op->output_chksum = zip_res.w0.adler32;
-
- sr->op->status = RTE_COMP_OP_STATUS_SUCCESS;
- err = 0;
exit:
*op = sr->op;
+ nitrox_dump_zip_result(&sr->instr, &zip_res);
+ nitrox_dump_databuf("OUT after", sr->op->m_dst, sr->op->dst.offset,
+ sr->op->produced);
return err;
}
@@ -7,6 +7,7 @@
#define NITROX_DECOMP_CTX_SIZE 2048
#define NITROX_CONSTANTS_MAX_SEARCH_DEPTH 31744
+#define NITROX_DEFAULT_DEFLATE_SEARCH_DEPTH 32768
struct nitrox_qp;
struct nitrox_softreq;
@@ -41,13 +42,14 @@ struct nitrox_comp_xform {
enum nitrox_comp_algo algo;
enum nitrox_comp_level level;
enum nitrox_chksum_type chksum_type;
-};
-
-struct nitrox_comp_stream {
- struct nitrox_comp_xform xform;
- int window_size;
- char context[NITROX_DECOMP_CTX_SIZE] __rte_aligned(8);
- char history_window[NITROX_CONSTANTS_MAX_SEARCH_DEPTH] __rte_aligned(8);
+ uint8_t *context;
+ uint8_t *history_window;
+ uint32_t chksum;
+ uint16_t window_size;
+ uint16_t hlen;
+ uint8_t exn;
+ uint8_t exbits;
+ bool bf;
};
int nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr);