new file mode 100644
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#ifndef _CNXK_DMA_EVENT_DP_H_
+#define _CNXK_DMA_EVENT_DP_H_
+
+#include <stdint.h>
+
+#include <rte_common.h>
+#include <rte_eventdev.h>
+
+__rte_internal
+uint16_t cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events);
+
+__rte_internal
+uint16_t cn9k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events);
+
+__rte_internal
+uintptr_t cnxk_dma_adapter_dequeue(uintptr_t get_work1);
+
+#endif /* _CNXK_DMA_EVENT_DP_H_ */
@@ -40,6 +40,11 @@
*/
#define CNXK_DPI_REQ_CDATA 0xFF
+/* Set Completion data to 0xDEADBEEF when request submitted for SSO.
+ * This helps differentiate if the dequeue is called after cnxk enueue.
+ */
+#define CNXK_DPI_REQ_SSO_CDATA 0xDEADBEEF
+
union cnxk_dpi_instr_cmd {
uint64_t u;
struct cn9k_dpi_instr_cmd {
@@ -85,7 +90,9 @@ union cnxk_dpi_instr_cmd {
struct cnxk_dpi_compl_s {
uint64_t cdata;
- void *cb_data;
+ void *op;
+ uint16_t dev_id;
+ uint16_t vchan;
uint32_t wqecs;
};
@@ -5,6 +5,8 @@
#include <rte_vect.h>
#include "cnxk_dmadev.h"
+#include "cnxk_dma_event_dp.h"
+#include <rte_event_dma_adapter.h>
static __plt_always_inline void
__dpi_cpy_scalar(uint64_t *src, uint64_t *dst, uint8_t n)
@@ -434,3 +436,210 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge
return dpi_conf->desc_idx++;
}
+
+uint16_t
+cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
+{
+ union rte_event_dma_metadata *dma_mdata;
+ struct rte_event_dma_request *req_info;
+ const struct rte_dma_sge *src, *dst;
+ struct rte_event_dma_adapter_op *op;
+ struct cnxk_dpi_compl_s *comp_ptr;
+ struct cnxk_dpi_conf *dpi_conf;
+ struct cnxk_dpi_vf_s *dpivf;
+ struct rte_event *rsp_info;
+ uint16_t nb_src, nb_dst;
+ struct rte_dma_dev *dev;
+ uint64_t hdr[4];
+ uint16_t count;
+ int rc;
+
+ PLT_SET_USED(ws);
+
+ for (count = 0; count < nb_events; count++) {
+ op = ev[count].event_ptr;
+ dma_mdata = (union rte_event_dma_metadata *)((uint8_t *)op +
+ sizeof(struct rte_event_dma_adapter_op));
+ rsp_info = &dma_mdata->response_info;
+ req_info = &dma_mdata->request_info;
+ dev = rte_dma_pmd_dev_get(req_info->dma_dev_id);
+ dpivf = dev->data->dev_private;
+ dpi_conf = &dpivf->conf[req_info->vchan];
+
+ if (unlikely(((dpi_conf->c_desc.tail + 1) & dpi_conf->c_desc.max_cnt) ==
+ dpi_conf->c_desc.head))
+ return count;
+
+ comp_ptr = dpi_conf->c_desc.compl_ptr[dpi_conf->c_desc.tail];
+ CNXK_DPI_STRM_INC(dpi_conf->c_desc, tail);
+ comp_ptr->op = op;
+ comp_ptr->dev_id = req_info->dma_dev_id;
+ comp_ptr->vchan = req_info->vchan;
+ comp_ptr->cdata = CNXK_DPI_REQ_SSO_CDATA;
+
+ nb_src = op->nb_src & CNXK_DPI_MAX_POINTER;
+ nb_dst = op->nb_dst & CNXK_DPI_MAX_POINTER;
+
+ hdr[0] = dpi_conf->cmd.u | ((uint64_t)DPI_HDR_PT_WQP << 54);
+ hdr[0] |= (nb_dst << 6) | nb_src;
+ hdr[1] = ((uint64_t)comp_ptr);
+ hdr[2] = (RTE_EVENT_TYPE_DMADEV << 28 | (rsp_info->sub_event_type << 20) |
+ rsp_info->flow_id);
+ hdr[2] |= ((uint64_t)(rsp_info->sched_type & DPI_HDR_TT_MASK)) << 32;
+ hdr[2] |= ((uint64_t)(rsp_info->queue_id & DPI_HDR_GRP_MASK)) << 34;
+
+ src = &op->src_seg[0];
+ dst = &op->dst_seg[0];
+
+ rc = __dpi_queue_write_sg(dpivf, hdr, src, dst, nb_src, nb_dst);
+ if (unlikely(rc)) {
+ CNXK_DPI_STRM_DEC(dpi_conf->c_desc, tail);
+ return rc;
+ }
+
+ if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ rte_wmb();
+ plt_write64(dpi_conf->pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst),
+ dpivf->rdpi.rbase + DPI_VDMA_DBELL);
+ dpi_conf->stats.submitted += dpi_conf->pending + 1;
+ dpi_conf->pnum_words = 0;
+ dpi_conf->pending = 0;
+ } else {
+ dpi_conf->pnum_words += CNXK_DPI_CMD_LEN(nb_src, nb_dst);
+ dpi_conf->pending++;
+ }
+ }
+
+ return count;
+}
+
+uint16_t
+cn9k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
+{
+ union rte_event_dma_metadata *dma_mdata;
+ struct rte_event_dma_request *req_info;
+ const struct rte_dma_sge *fptr, *lptr;
+ struct rte_event_dma_adapter_op *op;
+ struct cnxk_dpi_compl_s *comp_ptr;
+ struct cnxk_dpi_conf *dpi_conf;
+ struct cnxk_dpi_vf_s *dpivf;
+ struct rte_event *rsp_info;
+ uint16_t nb_src, nb_dst;
+ struct rte_dma_dev *dev;
+ uint64_t hdr[4];
+ uint16_t count;
+ int rc;
+
+ PLT_SET_USED(ws);
+
+ for (count = 0; count < nb_events; count++) {
+ op = ev[count].event_ptr;
+ dma_mdata = (union rte_event_dma_metadata *)((uint8_t *)op +
+ sizeof(struct rte_event_dma_adapter_op));
+ rsp_info = &dma_mdata->response_info;
+ req_info = &dma_mdata->request_info;
+ dev = rte_dma_pmd_dev_get(req_info->dma_dev_id);
+ dpivf = dev->data->dev_private;
+ dpi_conf = &dpivf->conf[req_info->vchan];
+
+ if (unlikely(((dpi_conf->c_desc.tail + 1) & dpi_conf->c_desc.max_cnt) ==
+ dpi_conf->c_desc.head))
+ return count;
+
+ comp_ptr = dpi_conf->c_desc.compl_ptr[dpi_conf->c_desc.tail];
+ CNXK_DPI_STRM_INC(dpi_conf->c_desc, tail);
+ comp_ptr->op = op;
+ comp_ptr->dev_id = req_info->dma_dev_id;
+ comp_ptr->vchan = req_info->vchan;
+ comp_ptr->cdata = CNXK_DPI_REQ_SSO_CDATA;
+
+ hdr[1] = dpi_conf->cmd.u | ((uint64_t)DPI_HDR_PT_WQP << 36);
+ hdr[2] = (uint64_t)comp_ptr;
+
+ nb_src = op->nb_src & CNXK_DPI_MAX_POINTER;
+ nb_dst = op->nb_dst & CNXK_DPI_MAX_POINTER;
+ /*
+ * For inbound case, src pointers are last pointers.
+ * For all other cases, src pointers are first pointers.
+ */
+ if (((dpi_conf->cmd.u >> 48) & DPI_HDR_XTYPE_MASK) == DPI_XTYPE_INBOUND) {
+ fptr = &op->dst_seg[0];
+ lptr = &op->src_seg[0];
+ RTE_SWAP(nb_src, nb_dst);
+ } else {
+ fptr = &op->src_seg[0];
+ lptr = &op->dst_seg[0];
+ }
+
+ hdr[0] = ((uint64_t)nb_dst << 54) | (uint64_t)nb_src << 48;
+ hdr[0] |= (RTE_EVENT_TYPE_DMADEV << 28 | (rsp_info->sub_event_type << 20) |
+ rsp_info->flow_id);
+ hdr[0] |= ((uint64_t)(rsp_info->sched_type & DPI_HDR_TT_MASK)) << 32;
+ hdr[0] |= ((uint64_t)(rsp_info->queue_id & DPI_HDR_GRP_MASK)) << 34;
+
+ rc = __dpi_queue_write_sg(dpivf, hdr, fptr, lptr, nb_src, nb_dst);
+ if (unlikely(rc)) {
+ CNXK_DPI_STRM_DEC(dpi_conf->c_desc, tail);
+ return rc;
+ }
+
+ if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) {
+ rte_wmb();
+ plt_write64(dpi_conf->pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst),
+ dpivf->rdpi.rbase + DPI_VDMA_DBELL);
+ dpi_conf->stats.submitted += dpi_conf->pending + 1;
+ dpi_conf->pnum_words = 0;
+ dpi_conf->pending = 0;
+ } else {
+ dpi_conf->pnum_words += CNXK_DPI_CMD_LEN(nb_src, nb_dst);
+ dpi_conf->pending++;
+ }
+ }
+
+ return count;
+}
+
+uintptr_t
+cnxk_dma_adapter_dequeue(uintptr_t get_work1)
+{
+ struct rte_event_dma_adapter_op *op;
+ struct cnxk_dpi_compl_s *comp_ptr;
+ struct cnxk_dpi_conf *dpi_conf;
+ struct cnxk_dpi_vf_s *dpivf;
+ struct rte_dma_dev *dev;
+ uint8_t *wqecs;
+
+ comp_ptr = (struct cnxk_dpi_compl_s *)get_work1;
+
+ /* Dequeue can be called without calling cnx_enqueue in case of
+ * dma_adapter. When its called from adapter, dma op will not be
+ * embedded in completion pointer. In those cases return op.
+ */
+ if (comp_ptr->cdata != CNXK_DPI_REQ_SSO_CDATA)
+ return (uintptr_t)comp_ptr;
+
+ dev = rte_dma_pmd_dev_get(comp_ptr->dev_id);
+ dpivf = dev->data->dev_private;
+ dpi_conf = &dpivf->conf[comp_ptr->vchan];
+
+ wqecs = (uint8_t *)&comp_ptr->wqecs;
+ if (__atomic_load_n(wqecs, __ATOMIC_RELAXED) != 0)
+ dpi_conf->stats.errors++;
+
+ op = (struct rte_event_dma_adapter_op *)comp_ptr->op;
+
+ /* We are done here. Reset completion buffer.*/
+ comp_ptr->wqecs = ~0;
+ comp_ptr->op = NULL;
+ comp_ptr->dev_id = ~0;
+ comp_ptr->vchan = ~0;
+ comp_ptr->cdata = CNXK_DPI_REQ_CDATA;
+
+ CNXK_DPI_STRM_INC(dpi_conf->c_desc, head);
+ /* Take into account errors also. This is similar to
+ * cnxk_dmadev_completed_status().
+ */
+ dpi_conf->stats.completed++;
+
+ return (uintptr_t)op;
+}
@@ -8,6 +8,10 @@ foreach flag: error_cflags
endif
endforeach
-deps += ['bus_pci', 'common_cnxk', 'dmadev']
+driver_sdk_headers = files(
+ 'cnxk_dma_event_dp.h',
+)
+
+deps += ['bus_pci', 'common_cnxk', 'dmadev', 'eventdev']
sources = files('cnxk_dmadev.c', 'cnxk_dmadev_fp.c')
require_iova_in_mbuf = false
new file mode 100644
@@ -0,0 +1,9 @@
+INTERNAL {
+ global:
+
+ cn10k_dma_adapter_enqueue;
+ cn9k_dma_adapter_enqueue;
+ cnxk_dma_adapter_dequeue;
+
+ local: *;
+};