@@ -32,6 +32,8 @@ struct ionic_dev_bar {
__rte_internal
void ionic_uio_scan_mnet_devices(void);
+__rte_internal
+void ionic_uio_scan_mcrypt_devices(void);
__rte_internal
void ionic_uio_get_rsrc(const char *name, int idx, struct ionic_dev_bar *bar);
@@ -23,10 +23,12 @@
#define IONIC_MDEV_UNK "mdev_unknown"
#define IONIC_MNIC "cpu_mnic"
+#define IONIC_MCRYPT "cpu_mcrypt"
#define IONIC_MAX_NAME_LEN 20
#define IONIC_MAX_MNETS 5
-#define IONIC_MAX_DEVICES (IONIC_MAX_MNETS)
+#define IONIC_MAX_MCPTS 1
+#define IONIC_MAX_DEVICES (IONIC_MAX_MNETS + IONIC_MAX_MCPTS)
#define IONIC_MAX_U16_IDX 0xFFFF
#define IONIC_UIO_MAX_TRIES 32
@@ -49,6 +51,7 @@ struct ionic_map_tbl ionic_mdev_map[IONIC_MAX_DEVICES] = {
{ "net_ionic2", 2, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
{ "net_ionic3", 3, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
{ "net_ionic4", 4, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
+ { "crypto_ionic0", 5, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
};
struct uio_name {
@@ -137,6 +140,49 @@ ionic_uio_scan_mnet_devices(void)
}
}
+void
+ionic_uio_scan_mcrypt_devices(void)
+{
+ struct ionic_map_tbl *map;
+ char devname[IONIC_MAX_NAME_LEN];
+ struct uio_name name_cache[IONIC_MAX_DEVICES];
+ bool done;
+ int mdev_idx = 0;
+ int uio_idx;
+ int i;
+ static bool scan_done;
+
+ if (scan_done)
+ return;
+
+ scan_done = true;
+
+ uio_fill_name_cache(name_cache, IONIC_MCRYPT);
+
+ for (i = IONIC_MAX_MNETS; i < IONIC_MAX_DEVICES; i++) {
+ done = false;
+
+ while (!done) {
+ if (mdev_idx > IONIC_MAX_MDEV_SCAN)
+ break;
+
+ /* Look for a matching mcrypt */
+ snprintf(devname, IONIC_MAX_NAME_LEN,
+ IONIC_MCRYPT "%d", mdev_idx);
+ uio_idx = uio_get_idx_for_devname(name_cache, devname);
+ if (uio_idx >= 0) {
+ map = &ionic_mdev_map[i];
+ map->uio_idx = (uint16_t)uio_idx;
+ strlcpy(map->mdev_name, devname,
+ IONIC_MAX_NAME_LEN);
+ done = true;
+ }
+
+ mdev_idx++;
+ }
+ }
+}
+
static int
uio_get_multi_dev_uionum(const char *name)
{
@@ -2,6 +2,7 @@ INTERNAL {
global:
ionic_uio_scan_mnet_devices;
+ ionic_uio_scan_mcrypt_devices;
ionic_uio_get_rsrc;
ionic_uio_rel_rsrc;
@@ -20,6 +20,11 @@
#include "ionic_crypto_if.h"
#include "ionic_regs.h"
+/* Devargs */
+/* NONE */
+
+#define IOCPT_MAX_RING_DESC 32768
+#define IOCPT_MIN_RING_DESC 16
#define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
#define IOCPT_CRYPTOQ_WAIT 10 /* 1s */
@@ -31,6 +36,64 @@ extern int iocpt_logtype;
#define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>")
+static inline void iocpt_struct_size_checks(void)
+{
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8);
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_intr) != 32);
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_intr_status) != 8);
+
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_regs) != 4096);
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_info_regs) != 2048);
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_cmd_regs) != 2048);
+
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_comp) != 16);
+
+ /* Device commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_comp) != 16);
+
+ /* LIF commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_reset_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_comp) != 16);
+
+ /* Queue commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_control_cmd) != 64);
+
+ /* Crypto */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_desc) != 32);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_sg_desc) != 256);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_comp) != 16);
+}
+
+struct iocpt_dev_bars {
+ struct ionic_dev_bar bar[IONIC_BARS_MAX];
+ uint32_t num_bars;
+};
+
+/* Queue watchdog */
+#define IOCPT_Q_WDOG_SESS_IDX 0
+#define IOCPT_Q_WDOG_KEY_LEN 16
+#define IOCPT_Q_WDOG_IV_LEN 12
+#define IOCPT_Q_WDOG_PLD_LEN 4
+#define IOCPT_Q_WDOG_TAG_LEN 16
+#define IOCPT_Q_WDOG_OP_TYPE RTE_CRYPTO_OP_TYPE_UNDEFINED
+
struct iocpt_qtype_info {
uint8_t version;
uint8_t supported;
@@ -107,8 +170,10 @@ struct iocpt_admin_q {
struct iocpt_dev {
const char *name;
char fw_version[IOCPT_FWVERS_BUFLEN];
+ struct iocpt_dev_bars bars;
struct iocpt_identity ident;
+ const struct iocpt_dev_intf *intf;
void *bus_dev;
struct rte_cryptodev *crypto_dev;
@@ -129,6 +194,8 @@ struct iocpt_dev {
struct iocpt_admin_q *adminq;
+ struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
+
uint64_t features;
uint32_t hw_features;
@@ -143,6 +210,20 @@ struct iocpt_dev {
struct rte_cryptodev_stats stats_base;
};
+struct iocpt_dev_intf {
+ int (*setup_bars)(struct iocpt_dev *dev);
+ void (*unmap_bars)(struct iocpt_dev *dev);
+};
+
+static inline int
+iocpt_setup_bars(struct iocpt_dev *dev)
+{
+ if (dev->intf->setup_bars == NULL)
+ return -EINVAL;
+
+ return (*dev->intf->setup_bars)(dev);
+}
+
/** iocpt_admin_ctx - Admin command context.
* @pending_work: Flag that indicates a completion.
* @cmd: Admin command (64B) to be copied to the queue.
@@ -154,6 +235,14 @@ struct iocpt_admin_ctx {
union iocpt_adminq_comp comp;
};
+int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
+ struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
+ uint8_t driver_id, uint8_t socket_id);
+int iocpt_remove(struct rte_device *rte_dev);
+
+void iocpt_configure(struct iocpt_dev *dev);
+void iocpt_deinit(struct iocpt_dev *dev);
+
int iocpt_dev_identify(struct iocpt_dev *dev);
int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa);
int iocpt_dev_adminq_init(struct iocpt_dev *dev);
@@ -12,6 +12,32 @@
int iocpt_logtype;
+static int
+iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)
+{
+ if (!rte_is_power_of_2(num_descs) ||
+ num_descs < IOCPT_MIN_RING_DESC ||
+ num_descs > IOCPT_MAX_RING_DESC) {
+ IOCPT_PRINT(ERR, "%u descriptors (min: %u max: %u)",
+ num_descs, IOCPT_MIN_RING_DESC, IOCPT_MAX_RING_DESC);
+ return -EINVAL;
+ }
+
+ cq->num_descs = num_descs;
+ cq->size_mask = num_descs - 1;
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ return 0;
+}
+
+static void
+iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)
+{
+ cq->base = base;
+ cq->base_pa = base_pa;
+}
+
uint32_t
iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
iocpt_cq_cb cb, void *cb_arg)
@@ -33,8 +59,520 @@ iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
return work_done;
}
+static int
+iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index,
+ uint16_t num_descs, uint16_t num_segs, uint32_t socket_id)
+{
+ uint32_t ring_size;
+
+ if (!rte_is_power_of_2(num_descs))
+ return -EINVAL;
+
+ ring_size = rte_log2_u32(num_descs);
+ if (ring_size < 2 || ring_size > 16)
+ return -EINVAL;
+
+ q->type = type;
+ q->index = index;
+ q->num_descs = num_descs;
+ q->num_segs = num_segs;
+ q->size_mask = num_descs - 1;
+ q->head_idx = 0;
+ q->tail_idx = 0;
+
+ q->info = rte_calloc_socket("iocpt",
+ num_descs * num_segs, sizeof(void *),
+ rte_mem_page_size(), socket_id);
+ if (q->info == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue info");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void
+iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->base = base;
+ q->base_pa = base_pa;
+}
+
+static void
+iocpt_q_sg_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->sg_base = base;
+ q->sg_base_pa = base_pa;
+}
+
+static void
+iocpt_q_free(struct iocpt_queue *q)
+{
+ if (q->info != NULL) {
+ rte_free(q->info);
+ q->info = NULL;
+ }
+}
+
+static const struct rte_memzone *
+iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
+ unsigned int align, int socket_id)
+{
+ char zone_name[RTE_MEMZONE_NAMESIZE];
+ const struct rte_memzone *mz;
+ int err;
+
+ err = snprintf(zone_name, sizeof(zone_name),
+ "iocpt_%s_%u", type_name, qid);
+ if (err >= RTE_MEMZONE_NAMESIZE) {
+ IOCPT_PRINT(ERR, "Name %s too long", type_name);
+ return NULL;
+ }
+
+ mz = rte_memzone_lookup(zone_name);
+ if (mz != NULL)
+ return mz;
+
+ return rte_memzone_reserve_aligned(zone_name, size, socket_id,
+ RTE_MEMZONE_IOVA_CONTIG, align);
+}
+
+static int
+iocpt_commonq_alloc(struct iocpt_dev *dev,
+ uint8_t type,
+ size_t struct_size,
+ uint32_t socket_id,
+ uint32_t index,
+ const char *type_name,
+ uint16_t flags,
+ uint16_t num_descs,
+ uint16_t num_segs,
+ uint16_t desc_size,
+ uint16_t cq_desc_size,
+ uint16_t sg_desc_size,
+ struct iocpt_common_q **comq)
+{
+ struct iocpt_common_q *new;
+ uint32_t q_size, cq_size, sg_size, total_size;
+ void *q_base, *cq_base, *sg_base;
+ rte_iova_t q_base_pa = 0;
+ rte_iova_t cq_base_pa = 0;
+ rte_iova_t sg_base_pa = 0;
+ size_t page_size = rte_mem_page_size();
+ int err;
+
+ *comq = NULL;
+
+ q_size = num_descs * desc_size;
+ cq_size = num_descs * cq_desc_size;
+ sg_size = num_descs * sg_desc_size;
+
+ /*
+ * Note: aligning q_size/cq_size is not enough due to cq_base address
+ * aligning as q_base could be not aligned to the page.
+ * Adding page_size.
+ */
+ total_size = RTE_ALIGN(q_size, page_size) +
+ RTE_ALIGN(cq_size, page_size) + page_size;
+ if (flags & IOCPT_Q_F_SG)
+ total_size += RTE_ALIGN(sg_size, page_size) + page_size;
+
+ new = rte_zmalloc_socket("iocpt", struct_size,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (new == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue structure");
+ return -ENOMEM;
+ }
+
+ new->dev = dev;
+
+ err = iocpt_q_init(&new->q, type, index, num_descs, num_segs,
+ socket_id);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Queue initialization failed");
+ goto err_free_q;
+ }
+
+ err = iocpt_cq_init(&new->cq, num_descs);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Completion queue initialization failed");
+ goto err_deinit_q;
+ }
+
+ new->base_z = iocpt_dma_zone_reserve(type_name, index, total_size,
+ IONIC_ALIGN, socket_id);
+ if (new->base_z == NULL) {
+ IOCPT_PRINT(ERR, "Cannot reserve queue DMA memory");
+ err = -ENOMEM;
+ goto err_deinit_cq;
+ }
+
+ new->base = new->base_z->addr;
+ new->base_pa = new->base_z->iova;
+
+ q_base = new->base;
+ q_base_pa = new->base_pa;
+ iocpt_q_map(&new->q, q_base, q_base_pa);
+
+ cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, page_size);
+ cq_base_pa = RTE_ALIGN(q_base_pa + q_size, page_size);
+ iocpt_cq_map(&new->cq, cq_base, cq_base_pa);
+
+ if (flags & IOCPT_Q_F_SG) {
+ sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
+ page_size);
+ sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, page_size);
+ iocpt_q_sg_map(&new->q, sg_base, sg_base_pa);
+ }
+
+ IOCPT_PRINT(DEBUG, "q_base_pa %#jx cq_base_pa %#jx sg_base_pa %#jx",
+ q_base_pa, cq_base_pa, sg_base_pa);
+
+ *comq = new;
+
+ return 0;
+
+err_deinit_cq:
+err_deinit_q:
+ iocpt_q_free(&new->q);
+err_free_q:
+ rte_free(new);
+ return err;
+}
+
struct ionic_doorbell *
iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)
{
return dev->db_pages + q->hw_type;
}
+
+static int
+iocpt_adminq_alloc(struct iocpt_dev *dev)
+{
+ struct iocpt_admin_q *aq;
+ uint16_t num_descs = IOCPT_ADMINQ_LENGTH;
+ uint16_t flags = 0;
+ int err;
+
+ err = iocpt_commonq_alloc(dev,
+ IOCPT_QTYPE_ADMINQ,
+ sizeof(struct iocpt_admin_q),
+ rte_socket_id(),
+ 0,
+ "admin",
+ flags,
+ num_descs,
+ 1,
+ sizeof(struct iocpt_admin_cmd),
+ sizeof(struct iocpt_admin_comp),
+ 0,
+ (struct iocpt_common_q **)&aq);
+ if (err != 0)
+ return err;
+
+ aq->flags = flags;
+
+ dev->adminq = aq;
+
+ return 0;
+}
+
+static int
+iocpt_adminq_init(struct iocpt_dev *dev)
+{
+ return iocpt_dev_adminq_init(dev);
+}
+
+static void
+iocpt_adminq_deinit(struct iocpt_dev *dev)
+{
+ dev->adminq->flags &= ~IOCPT_Q_F_INITED;
+}
+
+static void
+iocpt_adminq_free(struct iocpt_admin_q *aq)
+{
+ if (aq->base_z != NULL) {
+ rte_memzone_free(aq->base_z);
+ aq->base_z = NULL;
+ aq->base = NULL;
+ aq->base_pa = 0;
+ }
+
+ iocpt_q_free(&aq->q);
+
+ rte_free(aq);
+}
+
+static int
+iocpt_alloc_objs(struct iocpt_dev *dev)
+{
+ uint32_t bmsize, i;
+ uint8_t *bm;
+ int err;
+
+ IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+
+ rte_spinlock_init(&dev->adminq_lock);
+ rte_spinlock_init(&dev->adminq_service_lock);
+
+ err = iocpt_adminq_alloc(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot allocate admin queue");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
+ dev->info_z = iocpt_dma_zone_reserve("info", 0, dev->info_sz,
+ IONIC_ALIGN, dev->socket_id);
+ if (dev->info_z == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate dev info memory");
+ err = -ENOMEM;
+ goto err_free_adminq;
+ }
+
+ dev->info = dev->info_z->addr;
+ dev->info_pa = dev->info_z->iova;
+
+ bmsize = rte_bitmap_get_memory_footprint(dev->max_sessions);
+ bm = rte_malloc_socket("iocpt", bmsize,
+ RTE_CACHE_LINE_SIZE, dev->socket_id);
+ if (bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate %uB bitmap memory", bmsize);
+ err = -ENOMEM;
+ goto err_free_dmazone;
+ }
+
+ dev->sess_bm = rte_bitmap_init(dev->max_sessions, bm, bmsize);
+ if (dev->sess_bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot initialize bitmap");
+ err = -EFAULT;
+ goto err_free_bm;
+ }
+ for (i = 0; i < dev->max_sessions; i++)
+ rte_bitmap_set(dev->sess_bm, i);
+
+ return 0;
+
+err_free_bm:
+ rte_free(bm);
+err_free_dmazone:
+ rte_memzone_free(dev->info_z);
+ dev->info_z = NULL;
+ dev->info = NULL;
+ dev->info_pa = 0;
+err_free_adminq:
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
+err_out:
+ return err;
+}
+
+static int
+iocpt_init(struct iocpt_dev *dev)
+{
+ int err;
+
+ memset(&dev->stats_base, 0, sizeof(dev->stats_base));
+
+ /* Uses dev_cmds */
+ err = iocpt_dev_init(dev, dev->info_pa);
+ if (err != 0)
+ return err;
+
+ err = iocpt_adminq_init(dev);
+ if (err != 0)
+ return err;
+
+ dev->state |= IOCPT_DEV_F_INITED;
+
+ return 0;
+}
+
+void
+iocpt_configure(struct iocpt_dev *dev)
+{
+ RTE_SET_USED(dev);
+}
+
+void
+iocpt_deinit(struct iocpt_dev *dev)
+{
+ IOCPT_PRINT_CALL();
+
+ if (!(dev->state & IOCPT_DEV_F_INITED))
+ return;
+
+ iocpt_adminq_deinit(dev);
+
+ dev->state &= ~IOCPT_DEV_F_INITED;
+}
+
+static void
+iocpt_free_objs(struct iocpt_dev *dev)
+{
+ IOCPT_PRINT_CALL();
+
+ if (dev->sess_bm != NULL) {
+ rte_bitmap_free(dev->sess_bm);
+ rte_free(dev->sess_bm);
+ dev->sess_bm = NULL;
+ }
+
+ if (dev->adminq != NULL) {
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
+ }
+
+ if (dev->info != NULL) {
+ rte_memzone_free(dev->info_z);
+ dev->info_z = NULL;
+ dev->info = NULL;
+ dev->info_pa = 0;
+ }
+}
+
+static int
+iocpt_devargs(struct rte_devargs *devargs, struct iocpt_dev *dev)
+{
+ RTE_SET_USED(devargs);
+ RTE_SET_USED(dev);
+
+ return 0;
+}
+
+int
+iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
+ struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
+ uint8_t driver_id, uint8_t socket_id)
+{
+ struct rte_cryptodev_pmd_init_params init_params = {
+ "iocpt",
+ sizeof(struct iocpt_dev),
+ socket_id,
+ RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
+ };
+ struct rte_cryptodev *cdev;
+ struct iocpt_dev *dev;
+ uint32_t i, sig;
+ int err;
+
+ /* Check structs (trigger error at compilation time) */
+ iocpt_struct_size_checks();
+
+ /* Multi-process not supported */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ err = -EPERM;
+ goto err;
+ }
+
+ cdev = rte_cryptodev_pmd_create(rte_dev->name, rte_dev, &init_params);
+ if (cdev == NULL) {
+ IOCPT_PRINT(ERR, "OOM");
+ err = -ENOMEM;
+ goto err;
+ }
+
+ dev = cdev->data->dev_private;
+ dev->crypto_dev = cdev;
+ dev->bus_dev = bus_dev;
+ dev->intf = intf;
+ dev->driver_id = driver_id;
+ dev->socket_id = socket_id;
+
+ for (i = 0; i < bars->num_bars; i++) {
+ struct ionic_dev_bar *bar = &bars->bar[i];
+
+ IOCPT_PRINT(DEBUG,
+ "bar[%u] = { .va = %p, .pa = %#jx, .len = %lu }",
+ i, bar->vaddr, bar->bus_addr, bar->len);
+ if (bar->vaddr == NULL) {
+ IOCPT_PRINT(ERR, "Null bar found, aborting");
+ err = -EFAULT;
+ goto err_destroy_crypto_dev;
+ }
+
+ dev->bars.bar[i].vaddr = bar->vaddr;
+ dev->bars.bar[i].bus_addr = bar->bus_addr;
+ dev->bars.bar[i].len = bar->len;
+ }
+ dev->bars.num_bars = bars->num_bars;
+
+ err = iocpt_devargs(rte_dev->devargs, dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot parse device arguments");
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_setup_bars(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot setup BARs: %d, aborting", err);
+ goto err_destroy_crypto_dev;
+ }
+
+ sig = ioread32(&dev->dev_info->signature);
+ if (sig != IOCPT_DEV_INFO_SIGNATURE) {
+ IOCPT_PRINT(ERR, "Incompatible firmware signature %#x", sig);
+ err = -EFAULT;
+ goto err_destroy_crypto_dev;
+ }
+
+ for (i = 0; i < IOCPT_FWVERS_BUFLEN; i++)
+ dev->fw_version[i] = ioread8(&dev->dev_info->fw_version[i]);
+ dev->fw_version[IOCPT_FWVERS_BUFLEN - 1] = '\0';
+ IOCPT_PRINT(DEBUG, "%s firmware: %s", dev->name, dev->fw_version);
+
+ err = iocpt_dev_identify(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot identify device: %d, aborting",
+ err);
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_alloc_objs(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot alloc device objects: %d", err);
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_init(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot init device: %d, aborting", err);
+ goto err_free_objs;
+ }
+
+ return 0;
+
+err_free_objs:
+ iocpt_free_objs(dev);
+err_destroy_crypto_dev:
+ rte_cryptodev_pmd_destroy(cdev);
+err:
+ return err;
+}
+
+int
+iocpt_remove(struct rte_device *rte_dev)
+{
+ struct rte_cryptodev *cdev;
+ struct iocpt_dev *dev;
+
+ cdev = rte_cryptodev_pmd_get_named_dev(rte_dev->name);
+ if (cdev == NULL) {
+ IOCPT_PRINT(DEBUG, "Cannot find device %s", rte_dev->name);
+ return -ENODEV;
+ }
+
+ dev = cdev->data->dev_private;
+
+ iocpt_deinit(dev);
+
+ iocpt_dev_reset(dev);
+
+ iocpt_free_objs(dev);
+
+ rte_cryptodev_pmd_destroy(cdev);
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <stdint.h>
+#include <errno.h>
+
+#include <rte_errno.h>
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_eal.h>
+#include <bus_vdev_driver.h>
+#include <rte_dev.h>
+#include <rte_string_fns.h>
+
+#include "ionic_crypto.h"
+
+#define IOCPT_VDEV_DEV_BAR 0
+#define IOCPT_VDEV_INTR_CTL_BAR 1
+#define IOCPT_VDEV_INTR_CFG_BAR 2
+#define IOCPT_VDEV_DB_BAR 3
+#define IOCPT_VDEV_BARS_MAX 4
+
+#define IOCPT_VDEV_DEV_INFO_REGS_OFFSET 0x0000
+#define IOCPT_VDEV_DEV_CMD_REGS_OFFSET 0x0800
+
+#define IOCPT_VDEV_FW_WAIT_US 1000 /* 1ms */
+#define IOCPT_VDEV_FW_WAIT_MAX 5000 /* 5s */
+
+static int
+iocpt_vdev_setup_bars(struct iocpt_dev *dev)
+{
+ struct iocpt_dev_bars *bars = &dev->bars;
+ uint8_t *bar0_base;
+ uint32_t fw_waits = 0;
+ uint8_t fw;
+
+ IOCPT_PRINT_CALL();
+
+ /* BAR0: dev_cmd */
+ bar0_base = bars->bar[IOCPT_VDEV_DEV_BAR].vaddr;
+ dev->dev_info = (union iocpt_dev_info_regs *)
+ &bar0_base[IOCPT_VDEV_DEV_INFO_REGS_OFFSET];
+ dev->dev_cmd = (union iocpt_dev_cmd_regs *)
+ &bar0_base[IOCPT_VDEV_DEV_CMD_REGS_OFFSET];
+
+ /* BAR1: interrupts */
+ dev->intr_ctrl = (void *)bars->bar[IOCPT_VDEV_INTR_CTL_BAR].vaddr;
+
+ /* BAR3: doorbells */
+ dev->db_pages = (void *)bars->bar[IOCPT_VDEV_DB_BAR].vaddr;
+
+ /* Wait for the FW to indicate readiness */
+ while (1) {
+ fw = ioread8(&dev->dev_info->fw_status);
+ if ((fw & IOCPT_FW_STS_F_RUNNING) != 0)
+ break;
+
+ if (fw_waits > IOCPT_VDEV_FW_WAIT_MAX) {
+ IOCPT_PRINT(ERR, "Firmware readiness bit not set");
+ return -ETIMEDOUT;
+ }
+
+ fw_waits++;
+ rte_delay_us_block(IOCPT_VDEV_FW_WAIT_US);
+ }
+ IOCPT_PRINT(DEBUG, "Firmware ready (%u waits)", fw_waits);
+
+ dev->name = rte_vdev_device_name(dev->bus_dev);
+
+ return 0;
+}
+
+static void
+iocpt_vdev_unmap_bars(struct iocpt_dev *dev)
+{
+ struct iocpt_dev_bars *bars = &dev->bars;
+ uint32_t i;
+
+ for (i = 0; i < IOCPT_VDEV_BARS_MAX; i++)
+ ionic_uio_rel_rsrc(dev->name, i, &bars->bar[i]);
+}
+
+static uint8_t iocpt_vdev_driver_id;
+static const struct iocpt_dev_intf iocpt_vdev_intf = {
+ .setup_bars = iocpt_vdev_setup_bars,
+ .unmap_bars = iocpt_vdev_unmap_bars,
+};
+
+static int
+iocpt_vdev_probe(struct rte_vdev_device *vdev)
+{
+ struct iocpt_dev_bars bars = {};
+ const char *name = rte_vdev_device_name(vdev);
+ unsigned int i;
+
+ IOCPT_PRINT(NOTICE, "Initializing device %s%s", name,
+ rte_eal_process_type() == RTE_PROC_SECONDARY ?
+ " [SECONDARY]" : "");
+
+ ionic_uio_scan_mcrypt_devices();
+
+ for (i = 0; i < IOCPT_VDEV_BARS_MAX; i++)
+ ionic_uio_get_rsrc(name, i, &bars.bar[i]);
+
+ bars.num_bars = IOCPT_VDEV_BARS_MAX;
+
+ return iocpt_probe((void *)vdev, &vdev->device,
+ &bars, &iocpt_vdev_intf,
+ iocpt_vdev_driver_id, rte_socket_id());
+}
+
+static int
+iocpt_vdev_remove(struct rte_vdev_device *vdev)
+{
+ return iocpt_remove(&vdev->device);
+}
+
+static struct rte_vdev_driver rte_vdev_iocpt_pmd = {
+ .probe = iocpt_vdev_probe,
+ .remove = iocpt_vdev_remove,
+};
+
+static struct cryptodev_driver rte_vdev_iocpt_drv;
+
+RTE_PMD_REGISTER_VDEV(crypto_ionic, rte_vdev_iocpt_pmd);
+RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_vdev_iocpt_drv, rte_vdev_iocpt_pmd.driver,
+ iocpt_vdev_driver_id);
@@ -1,11 +1,13 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2021-2024 Advanced Micro Devices, Inc.
+deps += ['bus_vdev']
deps += ['common_ionic']
sources = files(
'ionic_crypto_cmds.c',
'ionic_crypto_main.c',
+ 'ionic_crypto_vdev.c',
)
name = 'ionic_crypto'