[v5,1/5] drivers/raw: introduce AFU raw device driver

  Add multi-function AFU raw device driver to manage various AFU
(Acceleration Function Unit) in FPGA.
This driver implements common AFU raw device interfaces and
exposes them to application as standard raw device APIs.
Normal application can operate specified AFU as below,
1. call rte_rawdev_pmd_get_named_dev() to find AFU raw device.
2. call rte_rawdev_configure() to initialize AFU raw device.
3. call rte_rawdev_selftest() to test function of AFU.

Signed-off-by: Wei Huang <wei.huang@intel.com>
---
v2: fix typo
---
v3: fix build error in FreeBSD13-64, UB2004-32 and UB2204-32
---
v4: fix coding style issue and build error in FreeBSD13-64
---
v5: split patch into several patches
---
 drivers/raw/afu_mf/afu_mf_rawdev.c | 425 +++++++++++++++++++++++++++++++++++++
 drivers/raw/afu_mf/afu_mf_rawdev.h |  71 +++++++
 drivers/raw/afu_mf/meson.build     |   5 +
 drivers/raw/afu_mf/version.map     |   3 +
 drivers/raw/meson.build            |   1 +
 5 files changed, 505 insertions(+)
 create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.c
 create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.h
 create mode 100644 drivers/raw/afu_mf/meson.build
 create mode 100644 drivers/raw/afu_mf/version.map
  

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Friday, May 27, 2022 1:37 PM
> To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>
> Subject: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> 
> Add multi-function AFU raw device driver to manage various AFU (Acceleration
> Function Unit) in FPGA.
> This driver implements common AFU raw device interfaces and exposes them to
> application as standard raw device APIs.
> Normal application can operate specified AFU as below, 1. call
> rte_rawdev_pmd_get_named_dev() to find AFU raw device.
> 2. call rte_rawdev_configure() to initialize AFU raw device.
> 3. call rte_rawdev_selftest() to test function of AFU.
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> ---
> v2: fix typo
> ---
> v3: fix build error in FreeBSD13-64, UB2004-32 and UB2204-32
> ---
> v4: fix coding style issue and build error in FreeBSD13-64
> ---
> v5: split patch into several patches
> ---
>  drivers/raw/afu_mf/afu_mf_rawdev.c | 425
> +++++++++++++++++++++++++++++++++++++
>  drivers/raw/afu_mf/afu_mf_rawdev.h |  71 +++++++
>  drivers/raw/afu_mf/meson.build     |   5 +
>  drivers/raw/afu_mf/version.map     |   3 +
>  drivers/raw/meson.build            |   1 +
>  5 files changed, 505 insertions(+)
>  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.c
>  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.h
>  create mode 100644 drivers/raw/afu_mf/meson.build  create mode 100644
> drivers/raw/afu_mf/version.map

I am thinking that just put those afu* c/h files into drivers/raw/ifpga/ folder is better? Because this AFU PMD driver is work on ifpga.
And the file name of "afu_mf_rawdev.c" change to "afu_pmd_driver.c"?

> 
> diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.c
> b/drivers/raw/afu_mf/afu_mf_rawdev.c
> new file mode 100644
> index 0000000..5be372a
> --- /dev/null
> +++ b/drivers/raw/afu_mf/afu_mf_rawdev.c
> @@ -0,0 +1,425 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2022 Intel Corporation
> + */
> +
> +#include <errno.h>
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <poll.h>
> +#include <sys/eventfd.h>
> +
> +#include <rte_eal.h>
> +#include <rte_malloc.h>
> +#include <rte_memzone.h>
> +#include <rte_rawdev_pmd.h>
> +
> +#include "afu_mf_rawdev.h"
> +
> +#define AFU_MF_PMD_RAWDEV_NAME rawdev_afu_mf
> +
> +static const struct rte_afu_uuid afu_uuid_map[] = {
> +	{ 0, 0 /* sentinel */ }
> +};
> +
> +static struct afu_mf_drv *afu_table[] = {
> +	NULL
> +};
> +
> +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev) {
> +	int32_t x = 0;
> +
> +	if (!dev || !dev->shared)
> +		return -ENODEV;
> +
> +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> +
> +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock, &x,
> 1,
> +				1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)
> == 0))
> +		return -EBUSY;
> +
> +	return 0;
> +}
> +
> +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev) {
> +	if (!dev || !dev->shared)
> +		return;
> +
> +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE); }
> +
> +static int afu_mf_rawdev_configure(const struct rte_rawdev *rawdev,
> +	rte_rawdev_obj_t config, size_t config_size) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return -ENODEV;
> +
> +	if (dev->ops && dev->ops->config)
> +		ret = (*dev->ops->config)(dev, config, config_size);
> +
> +	return ret;
> +}
> +
> +static int afu_mf_rawdev_start(struct rte_rawdev *rawdev) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return -ENODEV;
> +
> +	ret = afu_mf_trylock(dev);
> +	if (ret) {
> +		AFU_MF_PMD_WARN("AFU is busy, please start it later");
> +		return ret;
> +	}
> +
> +	if (dev->ops && dev->ops->start)
> +		ret = (*dev->ops->start)(dev);
> +
> +	afu_mf_unlock(dev);
> +
> +	return ret;
> +}
> +
> +static void afu_mf_rawdev_stop(struct rte_rawdev *rawdev) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return;
> +
> +	ret = afu_mf_trylock(dev);
> +	if (ret) {
> +		AFU_MF_PMD_WARN("AFU is busy, please stop it later");
> +		return;
> +	}
> +
> +	if (dev->ops && dev->ops->stop)
> +		ret = (*dev->ops->stop)(dev);
> +
> +	afu_mf_unlock(dev);
> +}
> +
> +static int afu_mf_rawdev_close(struct rte_rawdev *rawdev) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return -ENODEV;
> +
> +	if (dev->ops && dev->ops->close)
> +		ret = (*dev->ops->close)(dev);
> +
> +	return ret;
> +}
> +
> +static int afu_mf_rawdev_reset(struct rte_rawdev *rawdev) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return -ENODEV;
> +
> +	ret = afu_mf_trylock(dev);
> +	if (ret) {
> +		AFU_MF_PMD_WARN("AFU is busy, please reset it later");
> +		return ret;
> +	}
> +
> +	if (dev->ops && dev->ops->reset)
> +		ret = (*dev->ops->reset)(dev);
> +
> +	afu_mf_unlock(dev);
> +
> +	return ret;
> +}
> +
> +static int afu_mf_rawdev_selftest(uint16_t dev_id) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	if (!rte_rawdev_pmd_is_valid_dev(dev_id))
> +		return -ENODEV;
> +
> +	dev = afu_mf_rawdev_get_priv(&rte_rawdevs[dev_id]);
> +	if (!dev)
> +		return -ENOENT;
> +
> +	ret = afu_mf_trylock(dev);
> +	if (ret) {
> +		AFU_MF_PMD_WARN("AFU is busy, please test it later");
> +		return ret;
> +	}
> +
> +	if (dev->ops && dev->ops->test)
> +		ret = (*dev->ops->test)(dev);
> +
> +	afu_mf_unlock(dev);
> +
> +	return ret;
> +}
> +
> +static int afu_mf_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) {
> +	struct afu_mf_rawdev *dev = NULL;
> +	int ret = 0;
> +
> +	AFU_MF_PMD_FUNC_TRACE();
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		return -ENODEV;
> +
> +	if (dev->ops && dev->ops->dump)
> +		ret = (*dev->ops->dump)(dev, f);
> +
> +	return ret;
> +}
> +
> +static const struct rte_rawdev_ops afu_mf_rawdev_ops = {
> +	.dev_info_get = NULL,
> +	.dev_configure = afu_mf_rawdev_configure,
> +	.dev_start = afu_mf_rawdev_start,
> +	.dev_stop = afu_mf_rawdev_stop,
> +	.dev_close = afu_mf_rawdev_close,
> +	.dev_reset = afu_mf_rawdev_reset,
> +
> +	.queue_def_conf = NULL,
> +	.queue_setup = NULL,
> +	.queue_release = NULL,
> +	.queue_count = NULL,
> +
> +	.attr_get = NULL,
> +	.attr_set = NULL,
> +
> +	.enqueue_bufs = NULL,
> +	.dequeue_bufs = NULL,
> +
> +	.dump = afu_mf_rawdev_dump,
> +
> +	.xstats_get = NULL,
> +	.xstats_get_names = NULL,
> +	.xstats_get_by_name = NULL,
> +	.xstats_reset = NULL,
> +
> +	.firmware_status_get = NULL,
> +	.firmware_version_get = NULL,
> +	.firmware_load = NULL,
> +	.firmware_unload = NULL,
> +
> +	.dev_selftest = afu_mf_rawdev_selftest, };
> +
> +static int
> +afu_mf_shared_alloc(const char *name, struct afu_mf_shared **data,
> +	int socket_id)
> +{
> +	const struct rte_memzone *mz;
> +	char mz_name[RTE_MEMZONE_NAMESIZE];
> +	struct afu_mf_shared *ptr = NULL;
> +	int init_mz = 0;
> +
> +	if (!name || !data)
> +		return -EINVAL;
> +
> +	/* name format is afu_?|??:??.? which is unique */
> +	snprintf(mz_name, sizeof(mz_name), "%s", name);
> +
> +	mz = rte_memzone_lookup(mz_name);
> +	if (!mz) {
> +		mz = rte_memzone_reserve(mz_name,
> +				sizeof(struct afu_mf_shared),
> +				socket_id, 0);
> +		init_mz = 1;
> +	}
> +
> +	if (!mz) {
> +		AFU_MF_PMD_ERR("Allocate memory zone %s failed!",
> +			mz_name);
> +		return -ENOMEM;
> +	}
> +
> +	ptr = (struct afu_mf_shared *)mz->addr;
> +
> +	if (init_mz)  /* initialize memory zone on the first time */
> +		ptr->lock = 0;
> +
> +	*data = ptr;
> +
> +	return 0;
> +}
> +
> +static int afu_mf_rawdev_name_get(struct rte_afu_device *afu_dev, char
> *name,
> +	size_t size)
> +{
> +	int n = 0;
> +
> +	if (!afu_dev || !name || !size)
> +		return -EINVAL;
> +
> +	n = snprintf(name, size, "afu_%s", afu_dev->device.name);
> +	if (n >= (int)size) {
> +		AFU_MF_PMD_ERR("Name of AFU device is too long!");
> +		return -ENAMETOOLONG;
> +	}
> +
> +	return 0;
> +}
> +
> +static struct afu_mf_ops *afu_mf_ops_get(struct rte_afu_uuid *afu_id) {
> +	struct afu_mf_drv *entry = NULL;
> +	int i = 0;
> +
> +	if (!afu_id)
> +		return NULL;
> +
> +	while ((entry = afu_table[i++])) {
> +		if ((entry->uuid.uuid_low == afu_id->uuid_low) &&
> +			(entry->uuid.uuid_high == afu_id->uuid_high))
> +			break;
> +	}
> +
> +	return entry ? entry->ops : NULL;
> +}
> +
> +static int afu_mf_rawdev_create(struct rte_afu_device *afu_dev, int
> +socket_id) {
> +	struct rte_rawdev *rawdev = NULL;
> +	struct afu_mf_rawdev *dev = NULL;
> +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> +	int ret = 0;
> +
> +	if (!afu_dev)
> +		return -EINVAL;
> +
> +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> +	if (ret)
> +		return ret;
> +
> +	AFU_MF_PMD_INFO("Create raw device %s on NUMA node %d",
> +		name, socket_id);
> +
> +	/* Allocate device structure */
> +	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> afu_mf_rawdev),
> +				socket_id);
> +	if (!rawdev) {
> +		AFU_MF_PMD_ERR("Unable to allocate raw device");
> +		return -ENOMEM;
> +	}
> +
> +	rawdev->dev_ops = &afu_mf_rawdev_ops;
> +	rawdev->device = &afu_dev->device;
> +	rawdev->driver_name = afu_dev->driver->driver.name;
> +
> +	dev = afu_mf_rawdev_get_priv(rawdev);
> +	if (!dev)
> +		goto cleanup;
> +
> +	dev->rawdev = rawdev;
> +	dev->port = afu_dev->id.port;
> +	dev->addr = afu_dev->mem_resource[0].addr;
> +	dev->ops = afu_mf_ops_get(&afu_dev->id.uuid);
> +	if (dev->ops == NULL) {
> +		AFU_MF_PMD_ERR("Unsupported AFU device");
> +		goto cleanup;
> +	}
> +
> +	if (dev->ops->init) {
> +		ret = (*dev->ops->init)(dev);
> +		if (ret) {
> +			AFU_MF_PMD_ERR("Failed to init %s", name);
> +			goto cleanup;
> +		}
> +	}
> +
> +	ret = afu_mf_shared_alloc(name, &dev->shared, socket_id);
> +	if (ret)
> +		goto cleanup;
> +
> +	return ret;
> +
> +cleanup:
> +	rte_rawdev_pmd_release(rawdev);
> +	return ret;
> +}
> +
> +static int afu_mf_rawdev_destroy(struct rte_afu_device *afu_dev) {
> +	struct rte_rawdev *rawdev = NULL;
> +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> +	int ret = 0;
> +
> +	if (!afu_dev)
> +		return -EINVAL;
> +
> +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> +	if (ret)
> +		return ret;
> +
> +	AFU_MF_PMD_INFO("Destroy raw device %s", name);
> +
> +	rawdev = rte_rawdev_pmd_get_named_dev(name);
> +	if (!rawdev) {
> +		AFU_MF_PMD_ERR("Raw device %s not found", name);
> +		return -EINVAL;
> +	}
> +
> +	/* rte_rawdev_close is called by pmd_release */
> +	ret = rte_rawdev_pmd_release(rawdev);
> +	if (ret)
> +		AFU_MF_PMD_DEBUG("Device cleanup failed");
> +
> +	return 0;
> +}
> +
> +static int afu_mf_rawdev_probe(struct rte_afu_device *afu_dev) {
> +	AFU_MF_PMD_FUNC_TRACE();
> +	return afu_mf_rawdev_create(afu_dev, rte_socket_id()); }
> +
> +static int afu_mf_rawdev_remove(struct rte_afu_device *afu_dev) {
> +	AFU_MF_PMD_FUNC_TRACE();
> +	return afu_mf_rawdev_destroy(afu_dev); }
> +
> +static struct rte_afu_driver afu_mf_pmd_drv = {
> +	.id_table = afu_uuid_map,
> +	.probe = afu_mf_rawdev_probe,
> +	.remove = afu_mf_rawdev_remove
> +};
> +
> +RTE_PMD_REGISTER_AFU(AFU_MF_PMD_RAWDEV_NAME, afu_mf_pmd_drv);
> +RTE_LOG_REGISTER_DEFAULT(afu_mf_pmd_logtype, NOTICE);
> diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.h
> b/drivers/raw/afu_mf/afu_mf_rawdev.h
> new file mode 100644
> index 0000000..df6715c
> --- /dev/null
> +++ b/drivers/raw/afu_mf/afu_mf_rawdev.h
> @@ -0,0 +1,71 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2022 Intel Corporation
> + */
> +
> +#ifndef __AFU_MF_RAWDEV_H__
> +#define __AFU_MF_RAWDEV_H__
> +
> +#include <stdint.h>
> +#include <stdio.h>
> +#include <unistd.h>
> +
> +#include <rte_cycles.h>
> +#include <rte_bus_ifpga.h>
> +#include <rte_rawdev.h>
> +
> +extern int afu_mf_pmd_logtype;
> +
> +#define AFU_MF_PMD_LOG(level, fmt, args...) \
> +	rte_log(RTE_LOG_ ## level, afu_mf_pmd_logtype, "%s(): " fmt "\n", \
> +		__func__, ##args)
> +
> +#define AFU_MF_PMD_FUNC_TRACE() AFU_MF_PMD_LOG(DEBUG, ">>")
> +
> +#define AFU_MF_PMD_DEBUG(fmt, args...) \
> +	AFU_MF_PMD_LOG(DEBUG, fmt, ## args)
> +#define AFU_MF_PMD_INFO(fmt, args...) \
> +	AFU_MF_PMD_LOG(INFO, fmt, ## args)
> +#define AFU_MF_PMD_ERR(fmt, args...) \
> +	AFU_MF_PMD_LOG(ERR, fmt, ## args)
> +#define AFU_MF_PMD_WARN(fmt, args...) \
> +	AFU_MF_PMD_LOG(WARNING, fmt, ## args)
> +
> +struct afu_mf_rawdev;
> +
> +struct afu_mf_ops {
> +	int (*init)(struct afu_mf_rawdev *dev);
> +	int (*config)(struct afu_mf_rawdev *dev, void *config,
> +		size_t config_size);
> +	int (*start)(struct afu_mf_rawdev *dev);
> +	int (*stop)(struct afu_mf_rawdev *dev);
> +	int (*test)(struct afu_mf_rawdev *dev);
> +	int (*close)(struct afu_mf_rawdev *dev);
> +	int (*reset)(struct afu_mf_rawdev *dev);
> +	int (*dump)(struct afu_mf_rawdev *dev, FILE *f); };
> +
> +struct afu_mf_drv {
> +	struct rte_afu_uuid uuid;
> +	struct afu_mf_ops *ops;
> +};
> +
> +struct afu_mf_shared {
> +	int32_t lock;
> +};
> +
> +struct afu_mf_rawdev {
> +	struct rte_rawdev *rawdev;  /* point to parent raw device */
> +	struct afu_mf_shared *shared;  /* shared data for multi-process */
> +	struct afu_mf_ops *ops;  /* device operation functions */
> +	int port;  /* index of port the AFU attached */
> +	void *addr;  /* base address of AFU registers */
> +	void *priv;  /* private driver data */ };
> +
> +static inline struct afu_mf_rawdev *
> +afu_mf_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> +	return rawdev ? (struct afu_mf_rawdev *)rawdev->dev_private : NULL; }
> +
> +#endif /* __AFU_MF_RAWDEV_H__ */
> diff --git a/drivers/raw/afu_mf/meson.build b/drivers/raw/afu_mf/meson.build
> new file mode 100644 index 0000000..80526a2
> --- /dev/null
> +++ b/drivers/raw/afu_mf/meson.build
> @@ -0,0 +1,5 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2022 Intel
> +Corporation
> +
> +deps += ['rawdev', 'bus_pci', 'bus_ifpga'] sources =
> +files('afu_mf_rawdev.c')
> diff --git a/drivers/raw/afu_mf/version.map b/drivers/raw/afu_mf/version.map
> new file mode 100644 index 0000000..c2e0723
> --- /dev/null
> +++ b/drivers/raw/afu_mf/version.map
> @@ -0,0 +1,3 @@
> +DPDK_22 {
> +	local: *;
> +};
> diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> 05e7de1..c3627f7 100644
> --- a/drivers/raw/meson.build
> +++ b/drivers/raw/meson.build
> @@ -6,6 +6,7 @@ if is_windows
>  endif
> 
>  drivers = [
> +        'afu_mf',
>          'cnxk_bphy',
>          'cnxk_gpio',
>          'dpaa2_cmdif',
> --
> 1.8.3.1
  

> -----Original Message-----
> From: Zhang, Tianfei <tianfei.zhang@intel.com>
> Sent: Monday, June 6, 2022 9:53 AM
> To: Huang, Wei <wei.huang@intel.com>; dev@dpdk.org;
> thomas@monjalon.net; nipun.gupta@nxp.com; hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>
> Subject: RE: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> 
> 
> 
> > -----Original Message-----
> > From: Huang, Wei <wei.huang@intel.com>
> > Sent: Friday, May 27, 2022 1:37 PM
> > To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> > hemant.agrawal@nxp.com
> > Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> > <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang,
> > Wei <wei.huang@intel.com>
> > Subject: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> >
> > Add multi-function AFU raw device driver to manage various AFU
> > (Acceleration Function Unit) in FPGA.
> > This driver implements common AFU raw device interfaces and exposes
> > them to application as standard raw device APIs.
> > Normal application can operate specified AFU as below, 1. call
> > rte_rawdev_pmd_get_named_dev() to find AFU raw device.
> > 2. call rte_rawdev_configure() to initialize AFU raw device.
> > 3. call rte_rawdev_selftest() to test function of AFU.
> >
> > Signed-off-by: Wei Huang <wei.huang@intel.com>
> > ---
> > v2: fix typo
> > ---
> > v3: fix build error in FreeBSD13-64, UB2004-32 and UB2204-32
> > ---
> > v4: fix coding style issue and build error in FreeBSD13-64
> > ---
> > v5: split patch into several patches
> > ---
> >  drivers/raw/afu_mf/afu_mf_rawdev.c | 425
> > +++++++++++++++++++++++++++++++++++++
> >  drivers/raw/afu_mf/afu_mf_rawdev.h |  71 +++++++
> >  drivers/raw/afu_mf/meson.build     |   5 +
> >  drivers/raw/afu_mf/version.map     |   3 +
> >  drivers/raw/meson.build            |   1 +
> >  5 files changed, 505 insertions(+)
> >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.c
> >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.h
> >  create mode 100644 drivers/raw/afu_mf/meson.build  create mode 100644
> > drivers/raw/afu_mf/version.map
> 
> I am thinking that just put those afu* c/h files into drivers/raw/ifpga/ folder is
> better? Because this AFU PMD driver is work on ifpga.
> And the file name of "afu_mf_rawdev.c" change to "afu_pmd_driver.c"?

Ohm, the file name of "afu_mf_rawdev.c" change to "afu_pmd_core.c"?
Because this file is the common code of AFU PMD driver, and specific function are in separate c file, like afu_pmd_n3000.c

> 
> >
> > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.c
> > b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > new file mode 100644
> > index 0000000..5be372a
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > @@ -0,0 +1,425 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2022 Intel Corporation
> > + */
> > +
> > +#include <errno.h>
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <unistd.h>
> > +#include <fcntl.h>
> > +#include <poll.h>
> > +#include <sys/eventfd.h>
> > +
> > +#include <rte_eal.h>
> > +#include <rte_malloc.h>
> > +#include <rte_memzone.h>
> > +#include <rte_rawdev_pmd.h>
> > +
> > +#include "afu_mf_rawdev.h"
> > +
> > +#define AFU_MF_PMD_RAWDEV_NAME rawdev_afu_mf
> > +
> > +static const struct rte_afu_uuid afu_uuid_map[] = {
> > +	{ 0, 0 /* sentinel */ }
> > +};
> > +
> > +static struct afu_mf_drv *afu_table[] = {
> > +	NULL
> > +};
> > +
> > +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev) {
> > +	int32_t x = 0;
> > +
> > +	if (!dev || !dev->shared)
> > +		return -ENODEV;
> > +
> > +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> > +
> > +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock, &x,
> > 1,
> > +				1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)
> > == 0))
> > +		return -EBUSY;
> > +
> > +	return 0;
> > +}
> > +
> > +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev) {
> > +	if (!dev || !dev->shared)
> > +		return;
> > +
> > +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE); }
> > +
> > +static int afu_mf_rawdev_configure(const struct rte_rawdev *rawdev,
> > +	rte_rawdev_obj_t config, size_t config_size) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->config)
> > +		ret = (*dev->ops->config)(dev, config, config_size);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_start(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please start it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->start)
> > +		ret = (*dev->ops->start)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static void afu_mf_rawdev_stop(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please stop it later");
> > +		return;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->stop)
> > +		ret = (*dev->ops->stop)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +}
> > +
> > +static int afu_mf_rawdev_close(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->close)
> > +		ret = (*dev->ops->close)(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_reset(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please reset it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->reset)
> > +		ret = (*dev->ops->reset)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_selftest(uint16_t dev_id) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	if (!rte_rawdev_pmd_is_valid_dev(dev_id))
> > +		return -ENODEV;
> > +
> > +	dev = afu_mf_rawdev_get_priv(&rte_rawdevs[dev_id]);
> > +	if (!dev)
> > +		return -ENOENT;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please test it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->test)
> > +		ret = (*dev->ops->test)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->dump)
> > +		ret = (*dev->ops->dump)(dev, f);
> > +
> > +	return ret;
> > +}
> > +
> > +static const struct rte_rawdev_ops afu_mf_rawdev_ops = {
> > +	.dev_info_get = NULL,
> > +	.dev_configure = afu_mf_rawdev_configure,
> > +	.dev_start = afu_mf_rawdev_start,
> > +	.dev_stop = afu_mf_rawdev_stop,
> > +	.dev_close = afu_mf_rawdev_close,
> > +	.dev_reset = afu_mf_rawdev_reset,
> > +
> > +	.queue_def_conf = NULL,
> > +	.queue_setup = NULL,
> > +	.queue_release = NULL,
> > +	.queue_count = NULL,
> > +
> > +	.attr_get = NULL,
> > +	.attr_set = NULL,
> > +
> > +	.enqueue_bufs = NULL,
> > +	.dequeue_bufs = NULL,
> > +
> > +	.dump = afu_mf_rawdev_dump,
> > +
> > +	.xstats_get = NULL,
> > +	.xstats_get_names = NULL,
> > +	.xstats_get_by_name = NULL,
> > +	.xstats_reset = NULL,
> > +
> > +	.firmware_status_get = NULL,
> > +	.firmware_version_get = NULL,
> > +	.firmware_load = NULL,
> > +	.firmware_unload = NULL,
> > +
> > +	.dev_selftest = afu_mf_rawdev_selftest, };
> > +
> > +static int
> > +afu_mf_shared_alloc(const char *name, struct afu_mf_shared **data,
> > +	int socket_id)
> > +{
> > +	const struct rte_memzone *mz;
> > +	char mz_name[RTE_MEMZONE_NAMESIZE];
> > +	struct afu_mf_shared *ptr = NULL;
> > +	int init_mz = 0;
> > +
> > +	if (!name || !data)
> > +		return -EINVAL;
> > +
> > +	/* name format is afu_?|??:??.? which is unique */
> > +	snprintf(mz_name, sizeof(mz_name), "%s", name);
> > +
> > +	mz = rte_memzone_lookup(mz_name);
> > +	if (!mz) {
> > +		mz = rte_memzone_reserve(mz_name,
> > +				sizeof(struct afu_mf_shared),
> > +				socket_id, 0);
> > +		init_mz = 1;
> > +	}
> > +
> > +	if (!mz) {
> > +		AFU_MF_PMD_ERR("Allocate memory zone %s failed!",
> > +			mz_name);
> > +		return -ENOMEM;
> > +	}
> > +
> > +	ptr = (struct afu_mf_shared *)mz->addr;
> > +
> > +	if (init_mz)  /* initialize memory zone on the first time */
> > +		ptr->lock = 0;
> > +
> > +	*data = ptr;
> > +
> > +	return 0;
> > +}
> > +
> > +static int afu_mf_rawdev_name_get(struct rte_afu_device *afu_dev,
> > +char
> > *name,
> > +	size_t size)
> > +{
> > +	int n = 0;
> > +
> > +	if (!afu_dev || !name || !size)
> > +		return -EINVAL;
> > +
> > +	n = snprintf(name, size, "afu_%s", afu_dev->device.name);
> > +	if (n >= (int)size) {
> > +		AFU_MF_PMD_ERR("Name of AFU device is too long!");
> > +		return -ENAMETOOLONG;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static struct afu_mf_ops *afu_mf_ops_get(struct rte_afu_uuid *afu_id) {
> > +	struct afu_mf_drv *entry = NULL;
> > +	int i = 0;
> > +
> > +	if (!afu_id)
> > +		return NULL;
> > +
> > +	while ((entry = afu_table[i++])) {
> > +		if ((entry->uuid.uuid_low == afu_id->uuid_low) &&
> > +			(entry->uuid.uuid_high == afu_id->uuid_high))
> > +			break;
> > +	}
> > +
> > +	return entry ? entry->ops : NULL;
> > +}
> > +
> > +static int afu_mf_rawdev_create(struct rte_afu_device *afu_dev, int
> > +socket_id) {
> > +	struct rte_rawdev *rawdev = NULL;
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > +	int ret = 0;
> > +
> > +	if (!afu_dev)
> > +		return -EINVAL;
> > +
> > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > +	if (ret)
> > +		return ret;
> > +
> > +	AFU_MF_PMD_INFO("Create raw device %s on NUMA node %d",
> > +		name, socket_id);
> > +
> > +	/* Allocate device structure */
> > +	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> > afu_mf_rawdev),
> > +				socket_id);
> > +	if (!rawdev) {
> > +		AFU_MF_PMD_ERR("Unable to allocate raw device");
> > +		return -ENOMEM;
> > +	}
> > +
> > +	rawdev->dev_ops = &afu_mf_rawdev_ops;
> > +	rawdev->device = &afu_dev->device;
> > +	rawdev->driver_name = afu_dev->driver->driver.name;
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		goto cleanup;
> > +
> > +	dev->rawdev = rawdev;
> > +	dev->port = afu_dev->id.port;
> > +	dev->addr = afu_dev->mem_resource[0].addr;
> > +	dev->ops = afu_mf_ops_get(&afu_dev->id.uuid);
> > +	if (dev->ops == NULL) {
> > +		AFU_MF_PMD_ERR("Unsupported AFU device");
> > +		goto cleanup;
> > +	}
> > +
> > +	if (dev->ops->init) {
> > +		ret = (*dev->ops->init)(dev);
> > +		if (ret) {
> > +			AFU_MF_PMD_ERR("Failed to init %s", name);
> > +			goto cleanup;
> > +		}
> > +	}
> > +
> > +	ret = afu_mf_shared_alloc(name, &dev->shared, socket_id);
> > +	if (ret)
> > +		goto cleanup;
> > +
> > +	return ret;
> > +
> > +cleanup:
> > +	rte_rawdev_pmd_release(rawdev);
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_destroy(struct rte_afu_device *afu_dev) {
> > +	struct rte_rawdev *rawdev = NULL;
> > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > +	int ret = 0;
> > +
> > +	if (!afu_dev)
> > +		return -EINVAL;
> > +
> > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > +	if (ret)
> > +		return ret;
> > +
> > +	AFU_MF_PMD_INFO("Destroy raw device %s", name);
> > +
> > +	rawdev = rte_rawdev_pmd_get_named_dev(name);
> > +	if (!rawdev) {
> > +		AFU_MF_PMD_ERR("Raw device %s not found", name);
> > +		return -EINVAL;
> > +	}
> > +
> > +	/* rte_rawdev_close is called by pmd_release */
> > +	ret = rte_rawdev_pmd_release(rawdev);
> > +	if (ret)
> > +		AFU_MF_PMD_DEBUG("Device cleanup failed");
> > +
> > +	return 0;
> > +}
> > +
> > +static int afu_mf_rawdev_probe(struct rte_afu_device *afu_dev) {
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +	return afu_mf_rawdev_create(afu_dev, rte_socket_id()); }
> > +
> > +static int afu_mf_rawdev_remove(struct rte_afu_device *afu_dev) {
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +	return afu_mf_rawdev_destroy(afu_dev); }
> > +
> > +static struct rte_afu_driver afu_mf_pmd_drv = {
> > +	.id_table = afu_uuid_map,
> > +	.probe = afu_mf_rawdev_probe,
> > +	.remove = afu_mf_rawdev_remove
> > +};
> > +
> > +RTE_PMD_REGISTER_AFU(AFU_MF_PMD_RAWDEV_NAME,
> afu_mf_pmd_drv);
> > +RTE_LOG_REGISTER_DEFAULT(afu_mf_pmd_logtype, NOTICE);
> > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.h
> > b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > new file mode 100644
> > index 0000000..df6715c
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > @@ -0,0 +1,71 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2022 Intel Corporation
> > + */
> > +
> > +#ifndef __AFU_MF_RAWDEV_H__
> > +#define __AFU_MF_RAWDEV_H__
> > +
> > +#include <stdint.h>
> > +#include <stdio.h>
> > +#include <unistd.h>
> > +
> > +#include <rte_cycles.h>
> > +#include <rte_bus_ifpga.h>
> > +#include <rte_rawdev.h>
> > +
> > +extern int afu_mf_pmd_logtype;
> > +
> > +#define AFU_MF_PMD_LOG(level, fmt, args...) \
> > +	rte_log(RTE_LOG_ ## level, afu_mf_pmd_logtype, "%s(): " fmt "\n", \
> > +		__func__, ##args)
> > +
> > +#define AFU_MF_PMD_FUNC_TRACE() AFU_MF_PMD_LOG(DEBUG, ">>")
> > +
> > +#define AFU_MF_PMD_DEBUG(fmt, args...) \
> > +	AFU_MF_PMD_LOG(DEBUG, fmt, ## args)
> > +#define AFU_MF_PMD_INFO(fmt, args...) \
> > +	AFU_MF_PMD_LOG(INFO, fmt, ## args)
> > +#define AFU_MF_PMD_ERR(fmt, args...) \
> > +	AFU_MF_PMD_LOG(ERR, fmt, ## args)
> > +#define AFU_MF_PMD_WARN(fmt, args...) \
> > +	AFU_MF_PMD_LOG(WARNING, fmt, ## args)
> > +
> > +struct afu_mf_rawdev;
> > +
> > +struct afu_mf_ops {
> > +	int (*init)(struct afu_mf_rawdev *dev);
> > +	int (*config)(struct afu_mf_rawdev *dev, void *config,
> > +		size_t config_size);
> > +	int (*start)(struct afu_mf_rawdev *dev);
> > +	int (*stop)(struct afu_mf_rawdev *dev);
> > +	int (*test)(struct afu_mf_rawdev *dev);
> > +	int (*close)(struct afu_mf_rawdev *dev);
> > +	int (*reset)(struct afu_mf_rawdev *dev);
> > +	int (*dump)(struct afu_mf_rawdev *dev, FILE *f); };
> > +
> > +struct afu_mf_drv {
> > +	struct rte_afu_uuid uuid;
> > +	struct afu_mf_ops *ops;
> > +};
> > +
> > +struct afu_mf_shared {
> > +	int32_t lock;
> > +};
> > +
> > +struct afu_mf_rawdev {
> > +	struct rte_rawdev *rawdev;  /* point to parent raw device */
> > +	struct afu_mf_shared *shared;  /* shared data for multi-process */
> > +	struct afu_mf_ops *ops;  /* device operation functions */
> > +	int port;  /* index of port the AFU attached */
> > +	void *addr;  /* base address of AFU registers */
> > +	void *priv;  /* private driver data */ };
> > +
> > +static inline struct afu_mf_rawdev *
> > +afu_mf_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> > +	return rawdev ? (struct afu_mf_rawdev *)rawdev->dev_private : NULL;
> > +}
> > +
> > +#endif /* __AFU_MF_RAWDEV_H__ */
> > diff --git a/drivers/raw/afu_mf/meson.build
> > b/drivers/raw/afu_mf/meson.build new file mode 100644 index
> > 0000000..80526a2
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/meson.build
> > @@ -0,0 +1,5 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2022 Intel
> > +Corporation
> > +
> > +deps += ['rawdev', 'bus_pci', 'bus_ifpga'] sources =
> > +files('afu_mf_rawdev.c')
> > diff --git a/drivers/raw/afu_mf/version.map
> > b/drivers/raw/afu_mf/version.map new file mode 100644 index
> > 0000000..c2e0723
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/version.map
> > @@ -0,0 +1,3 @@
> > +DPDK_22 {
> > +	local: *;
> > +};
> > diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> > 05e7de1..c3627f7 100644
> > --- a/drivers/raw/meson.build
> > +++ b/drivers/raw/meson.build
> > @@ -6,6 +6,7 @@ if is_windows
> >  endif
> >
> >  drivers = [
> > +        'afu_mf',
> >          'cnxk_bphy',
> >          'cnxk_gpio',
> >          'dpaa2_cmdif',
> > --
> > 1.8.3.1
  

On Fri, 27 May 2022 01:37:00 -0400
Wei Huang <wei.huang@intel.com> wrote:

> +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev)
> +{
> +	int32_t x = 0;
> +
> +	if (!dev || !dev->shared)
> +		return -ENODEV;
> +
> +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> +
> +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock, &x, 1,
> +				1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED) == 0))
> +		return -EBUSY;
> +
> +	return 0;
> +}
> +
> +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev)
> +{
> +	if (!dev || !dev->shared)
> +		return;
> +
> +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE);
> +}

What other subsystem is this shared with?
Is there a good reason it can't just use existing spinlock?
  

> -----Original Message-----
> From: Stephen Hemminger <stephen@networkplumber.org>
> Sent: Monday, June 6, 2022 23:38
> To: Huang, Wei <wei.huang@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com; stable@dpdk.org; Xu, Rosen
> <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Zhang, Qi
> Z <qi.z.zhang@intel.com>
> Subject: Re: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> 
> On Fri, 27 May 2022 01:37:00 -0400
> Wei Huang <wei.huang@intel.com> wrote:
> 
> > +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev) {
> > +	int32_t x = 0;
> > +
> > +	if (!dev || !dev->shared)
> > +		return -ENODEV;
> > +
> > +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> > +
> > +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock,
> &x, 1,
> > +				1, __ATOMIC_ACQUIRE,
> __ATOMIC_RELAXED) == 0))
> > +		return -EBUSY;
> > +
> > +	return 0;
> > +}
> > +
> > +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev) {
> > +	if (!dev || !dev->shared)
> > +		return;
> > +
> > +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE); }
> 
> What other subsystem is this shared with?
> Is there a good reason it can't just use existing spinlock?

This lock is used in multi-process situation, it prevent AFU from being operated by two DPDK process simultaneously.
I can use spinlock to replace it, thanks.
  

> -----Original Message-----
> From: Zhang, Tianfei <tianfei.zhang@intel.com>
> Sent: Monday, June 6, 2022 09:53
> To: Huang, Wei <wei.huang@intel.com>; dev@dpdk.org;
> thomas@monjalon.net; nipun.gupta@nxp.com; hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>
> Subject: RE: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> 
> 
> 
> > -----Original Message-----
> > From: Huang, Wei <wei.huang@intel.com>
> > Sent: Friday, May 27, 2022 1:37 PM
> > To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> > hemant.agrawal@nxp.com
> > Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> > <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang,
> > Wei <wei.huang@intel.com>
> > Subject: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> >
> > Add multi-function AFU raw device driver to manage various AFU
> > (Acceleration Function Unit) in FPGA.
> > This driver implements common AFU raw device interfaces and exposes
> > them to application as standard raw device APIs.
> > Normal application can operate specified AFU as below, 1. call
> > rte_rawdev_pmd_get_named_dev() to find AFU raw device.
> > 2. call rte_rawdev_configure() to initialize AFU raw device.
> > 3. call rte_rawdev_selftest() to test function of AFU.
> >
> > Signed-off-by: Wei Huang <wei.huang@intel.com>
> > ---
> > v2: fix typo
> > ---
> > v3: fix build error in FreeBSD13-64, UB2004-32 and UB2204-32
> > ---
> > v4: fix coding style issue and build error in FreeBSD13-64
> > ---
> > v5: split patch into several patches
> > ---
> >  drivers/raw/afu_mf/afu_mf_rawdev.c | 425
> > +++++++++++++++++++++++++++++++++++++
> >  drivers/raw/afu_mf/afu_mf_rawdev.h |  71 +++++++
> >  drivers/raw/afu_mf/meson.build     |   5 +
> >  drivers/raw/afu_mf/version.map     |   3 +
> >  drivers/raw/meson.build            |   1 +
> >  5 files changed, 505 insertions(+)
> >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.c
> >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.h
> >  create mode 100644 drivers/raw/afu_mf/meson.build  create mode
> 100644
> > drivers/raw/afu_mf/version.map
> 
> I am thinking that just put those afu* c/h files into drivers/raw/ifpga/ folder
> is better? Because this AFU PMD driver is work on ifpga.
> And the file name of "afu_mf_rawdev.c" change to "afu_pmd_driver.c"?
> 
agree
> >
> > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.c
> > b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > new file mode 100644
> > index 0000000..5be372a
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > @@ -0,0 +1,425 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2022 Intel Corporation
> > + */
> > +
> > +#include <errno.h>
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <unistd.h>
> > +#include <fcntl.h>
> > +#include <poll.h>
> > +#include <sys/eventfd.h>
> > +
> > +#include <rte_eal.h>
> > +#include <rte_malloc.h>
> > +#include <rte_memzone.h>
> > +#include <rte_rawdev_pmd.h>
> > +
> > +#include "afu_mf_rawdev.h"
> > +
> > +#define AFU_MF_PMD_RAWDEV_NAME rawdev_afu_mf
> > +
> > +static const struct rte_afu_uuid afu_uuid_map[] = {
> > +	{ 0, 0 /* sentinel */ }
> > +};
> > +
> > +static struct afu_mf_drv *afu_table[] = {
> > +	NULL
> > +};
> > +
> > +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev) {
> > +	int32_t x = 0;
> > +
> > +	if (!dev || !dev->shared)
> > +		return -ENODEV;
> > +
> > +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> > +
> > +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock,
> &x,
> > 1,
> > +				1, __ATOMIC_ACQUIRE,
> __ATOMIC_RELAXED)
> > == 0))
> > +		return -EBUSY;
> > +
> > +	return 0;
> > +}
> > +
> > +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev) {
> > +	if (!dev || !dev->shared)
> > +		return;
> > +
> > +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE); }
> > +
> > +static int afu_mf_rawdev_configure(const struct rte_rawdev *rawdev,
> > +	rte_rawdev_obj_t config, size_t config_size) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->config)
> > +		ret = (*dev->ops->config)(dev, config, config_size);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_start(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please start it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->start)
> > +		ret = (*dev->ops->start)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static void afu_mf_rawdev_stop(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please stop it later");
> > +		return;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->stop)
> > +		ret = (*dev->ops->stop)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +}
> > +
> > +static int afu_mf_rawdev_close(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->close)
> > +		ret = (*dev->ops->close)(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_reset(struct rte_rawdev *rawdev) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please reset it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->reset)
> > +		ret = (*dev->ops->reset)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_selftest(uint16_t dev_id) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	if (!rte_rawdev_pmd_is_valid_dev(dev_id))
> > +		return -ENODEV;
> > +
> > +	dev = afu_mf_rawdev_get_priv(&rte_rawdevs[dev_id]);
> > +	if (!dev)
> > +		return -ENOENT;
> > +
> > +	ret = afu_mf_trylock(dev);
> > +	if (ret) {
> > +		AFU_MF_PMD_WARN("AFU is busy, please test it later");
> > +		return ret;
> > +	}
> > +
> > +	if (dev->ops && dev->ops->test)
> > +		ret = (*dev->ops->test)(dev);
> > +
> > +	afu_mf_unlock(dev);
> > +
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) {
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	int ret = 0;
> > +
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		return -ENODEV;
> > +
> > +	if (dev->ops && dev->ops->dump)
> > +		ret = (*dev->ops->dump)(dev, f);
> > +
> > +	return ret;
> > +}
> > +
> > +static const struct rte_rawdev_ops afu_mf_rawdev_ops = {
> > +	.dev_info_get = NULL,
> > +	.dev_configure = afu_mf_rawdev_configure,
> > +	.dev_start = afu_mf_rawdev_start,
> > +	.dev_stop = afu_mf_rawdev_stop,
> > +	.dev_close = afu_mf_rawdev_close,
> > +	.dev_reset = afu_mf_rawdev_reset,
> > +
> > +	.queue_def_conf = NULL,
> > +	.queue_setup = NULL,
> > +	.queue_release = NULL,
> > +	.queue_count = NULL,
> > +
> > +	.attr_get = NULL,
> > +	.attr_set = NULL,
> > +
> > +	.enqueue_bufs = NULL,
> > +	.dequeue_bufs = NULL,
> > +
> > +	.dump = afu_mf_rawdev_dump,
> > +
> > +	.xstats_get = NULL,
> > +	.xstats_get_names = NULL,
> > +	.xstats_get_by_name = NULL,
> > +	.xstats_reset = NULL,
> > +
> > +	.firmware_status_get = NULL,
> > +	.firmware_version_get = NULL,
> > +	.firmware_load = NULL,
> > +	.firmware_unload = NULL,
> > +
> > +	.dev_selftest = afu_mf_rawdev_selftest, };
> > +
> > +static int
> > +afu_mf_shared_alloc(const char *name, struct afu_mf_shared **data,
> > +	int socket_id)
> > +{
> > +	const struct rte_memzone *mz;
> > +	char mz_name[RTE_MEMZONE_NAMESIZE];
> > +	struct afu_mf_shared *ptr = NULL;
> > +	int init_mz = 0;
> > +
> > +	if (!name || !data)
> > +		return -EINVAL;
> > +
> > +	/* name format is afu_?|??:??.? which is unique */
> > +	snprintf(mz_name, sizeof(mz_name), "%s", name);
> > +
> > +	mz = rte_memzone_lookup(mz_name);
> > +	if (!mz) {
> > +		mz = rte_memzone_reserve(mz_name,
> > +				sizeof(struct afu_mf_shared),
> > +				socket_id, 0);
> > +		init_mz = 1;
> > +	}
> > +
> > +	if (!mz) {
> > +		AFU_MF_PMD_ERR("Allocate memory zone %s failed!",
> > +			mz_name);
> > +		return -ENOMEM;
> > +	}
> > +
> > +	ptr = (struct afu_mf_shared *)mz->addr;
> > +
> > +	if (init_mz)  /* initialize memory zone on the first time */
> > +		ptr->lock = 0;
> > +
> > +	*data = ptr;
> > +
> > +	return 0;
> > +}
> > +
> > +static int afu_mf_rawdev_name_get(struct rte_afu_device *afu_dev,
> > +char
> > *name,
> > +	size_t size)
> > +{
> > +	int n = 0;
> > +
> > +	if (!afu_dev || !name || !size)
> > +		return -EINVAL;
> > +
> > +	n = snprintf(name, size, "afu_%s", afu_dev->device.name);
> > +	if (n >= (int)size) {
> > +		AFU_MF_PMD_ERR("Name of AFU device is too long!");
> > +		return -ENAMETOOLONG;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static struct afu_mf_ops *afu_mf_ops_get(struct rte_afu_uuid *afu_id) {
> > +	struct afu_mf_drv *entry = NULL;
> > +	int i = 0;
> > +
> > +	if (!afu_id)
> > +		return NULL;
> > +
> > +	while ((entry = afu_table[i++])) {
> > +		if ((entry->uuid.uuid_low == afu_id->uuid_low) &&
> > +			(entry->uuid.uuid_high == afu_id->uuid_high))
> > +			break;
> > +	}
> > +
> > +	return entry ? entry->ops : NULL;
> > +}
> > +
> > +static int afu_mf_rawdev_create(struct rte_afu_device *afu_dev, int
> > +socket_id) {
> > +	struct rte_rawdev *rawdev = NULL;
> > +	struct afu_mf_rawdev *dev = NULL;
> > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > +	int ret = 0;
> > +
> > +	if (!afu_dev)
> > +		return -EINVAL;
> > +
> > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > +	if (ret)
> > +		return ret;
> > +
> > +	AFU_MF_PMD_INFO("Create raw device %s on NUMA node %d",
> > +		name, socket_id);
> > +
> > +	/* Allocate device structure */
> > +	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> > afu_mf_rawdev),
> > +				socket_id);
> > +	if (!rawdev) {
> > +		AFU_MF_PMD_ERR("Unable to allocate raw device");
> > +		return -ENOMEM;
> > +	}
> > +
> > +	rawdev->dev_ops = &afu_mf_rawdev_ops;
> > +	rawdev->device = &afu_dev->device;
> > +	rawdev->driver_name = afu_dev->driver->driver.name;
> > +
> > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > +	if (!dev)
> > +		goto cleanup;
> > +
> > +	dev->rawdev = rawdev;
> > +	dev->port = afu_dev->id.port;
> > +	dev->addr = afu_dev->mem_resource[0].addr;
> > +	dev->ops = afu_mf_ops_get(&afu_dev->id.uuid);
> > +	if (dev->ops == NULL) {
> > +		AFU_MF_PMD_ERR("Unsupported AFU device");
> > +		goto cleanup;
> > +	}
> > +
> > +	if (dev->ops->init) {
> > +		ret = (*dev->ops->init)(dev);
> > +		if (ret) {
> > +			AFU_MF_PMD_ERR("Failed to init %s", name);
> > +			goto cleanup;
> > +		}
> > +	}
> > +
> > +	ret = afu_mf_shared_alloc(name, &dev->shared, socket_id);
> > +	if (ret)
> > +		goto cleanup;
> > +
> > +	return ret;
> > +
> > +cleanup:
> > +	rte_rawdev_pmd_release(rawdev);
> > +	return ret;
> > +}
> > +
> > +static int afu_mf_rawdev_destroy(struct rte_afu_device *afu_dev) {
> > +	struct rte_rawdev *rawdev = NULL;
> > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > +	int ret = 0;
> > +
> > +	if (!afu_dev)
> > +		return -EINVAL;
> > +
> > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > +	if (ret)
> > +		return ret;
> > +
> > +	AFU_MF_PMD_INFO("Destroy raw device %s", name);
> > +
> > +	rawdev = rte_rawdev_pmd_get_named_dev(name);
> > +	if (!rawdev) {
> > +		AFU_MF_PMD_ERR("Raw device %s not found", name);
> > +		return -EINVAL;
> > +	}
> > +
> > +	/* rte_rawdev_close is called by pmd_release */
> > +	ret = rte_rawdev_pmd_release(rawdev);
> > +	if (ret)
> > +		AFU_MF_PMD_DEBUG("Device cleanup failed");
> > +
> > +	return 0;
> > +}
> > +
> > +static int afu_mf_rawdev_probe(struct rte_afu_device *afu_dev) {
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +	return afu_mf_rawdev_create(afu_dev, rte_socket_id()); }
> > +
> > +static int afu_mf_rawdev_remove(struct rte_afu_device *afu_dev) {
> > +	AFU_MF_PMD_FUNC_TRACE();
> > +	return afu_mf_rawdev_destroy(afu_dev); }
> > +
> > +static struct rte_afu_driver afu_mf_pmd_drv = {
> > +	.id_table = afu_uuid_map,
> > +	.probe = afu_mf_rawdev_probe,
> > +	.remove = afu_mf_rawdev_remove
> > +};
> > +
> > +RTE_PMD_REGISTER_AFU(AFU_MF_PMD_RAWDEV_NAME,
> afu_mf_pmd_drv);
> > +RTE_LOG_REGISTER_DEFAULT(afu_mf_pmd_logtype, NOTICE);
> > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.h
> > b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > new file mode 100644
> > index 0000000..df6715c
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > @@ -0,0 +1,71 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2022 Intel Corporation
> > + */
> > +
> > +#ifndef __AFU_MF_RAWDEV_H__
> > +#define __AFU_MF_RAWDEV_H__
> > +
> > +#include <stdint.h>
> > +#include <stdio.h>
> > +#include <unistd.h>
> > +
> > +#include <rte_cycles.h>
> > +#include <rte_bus_ifpga.h>
> > +#include <rte_rawdev.h>
> > +
> > +extern int afu_mf_pmd_logtype;
> > +
> > +#define AFU_MF_PMD_LOG(level, fmt, args...) \
> > +	rte_log(RTE_LOG_ ## level, afu_mf_pmd_logtype, "%s(): " fmt "\n",
> \
> > +		__func__, ##args)
> > +
> > +#define AFU_MF_PMD_FUNC_TRACE() AFU_MF_PMD_LOG(DEBUG,
> ">>")
> > +
> > +#define AFU_MF_PMD_DEBUG(fmt, args...) \
> > +	AFU_MF_PMD_LOG(DEBUG, fmt, ## args)
> > +#define AFU_MF_PMD_INFO(fmt, args...) \
> > +	AFU_MF_PMD_LOG(INFO, fmt, ## args)
> > +#define AFU_MF_PMD_ERR(fmt, args...) \
> > +	AFU_MF_PMD_LOG(ERR, fmt, ## args)
> > +#define AFU_MF_PMD_WARN(fmt, args...) \
> > +	AFU_MF_PMD_LOG(WARNING, fmt, ## args)
> > +
> > +struct afu_mf_rawdev;
> > +
> > +struct afu_mf_ops {
> > +	int (*init)(struct afu_mf_rawdev *dev);
> > +	int (*config)(struct afu_mf_rawdev *dev, void *config,
> > +		size_t config_size);
> > +	int (*start)(struct afu_mf_rawdev *dev);
> > +	int (*stop)(struct afu_mf_rawdev *dev);
> > +	int (*test)(struct afu_mf_rawdev *dev);
> > +	int (*close)(struct afu_mf_rawdev *dev);
> > +	int (*reset)(struct afu_mf_rawdev *dev);
> > +	int (*dump)(struct afu_mf_rawdev *dev, FILE *f); };
> > +
> > +struct afu_mf_drv {
> > +	struct rte_afu_uuid uuid;
> > +	struct afu_mf_ops *ops;
> > +};
> > +
> > +struct afu_mf_shared {
> > +	int32_t lock;
> > +};
> > +
> > +struct afu_mf_rawdev {
> > +	struct rte_rawdev *rawdev;  /* point to parent raw device */
> > +	struct afu_mf_shared *shared;  /* shared data for multi-process */
> > +	struct afu_mf_ops *ops;  /* device operation functions */
> > +	int port;  /* index of port the AFU attached */
> > +	void *addr;  /* base address of AFU registers */
> > +	void *priv;  /* private driver data */ };
> > +
> > +static inline struct afu_mf_rawdev *
> > +afu_mf_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> > +	return rawdev ? (struct afu_mf_rawdev *)rawdev->dev_private :
> NULL;
> > +}
> > +
> > +#endif /* __AFU_MF_RAWDEV_H__ */
> > diff --git a/drivers/raw/afu_mf/meson.build
> > b/drivers/raw/afu_mf/meson.build new file mode 100644 index
> > 0000000..80526a2
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/meson.build
> > @@ -0,0 +1,5 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2022 Intel
> > +Corporation
> > +
> > +deps += ['rawdev', 'bus_pci', 'bus_ifpga'] sources =
> > +files('afu_mf_rawdev.c')
> > diff --git a/drivers/raw/afu_mf/version.map
> > b/drivers/raw/afu_mf/version.map new file mode 100644 index
> > 0000000..c2e0723
> > --- /dev/null
> > +++ b/drivers/raw/afu_mf/version.map
> > @@ -0,0 +1,3 @@
> > +DPDK_22 {
> > +	local: *;
> > +};
> > diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> > 05e7de1..c3627f7 100644
> > --- a/drivers/raw/meson.build
> > +++ b/drivers/raw/meson.build
> > @@ -6,6 +6,7 @@ if is_windows
> >  endif
> >
> >  drivers = [
> > +        'afu_mf',
> >          'cnxk_bphy',
> >          'cnxk_gpio',
> >          'dpaa2_cmdif',
> > --
> > 1.8.3.1
  

> -----Original Message-----
> From: Zhang, Tianfei <tianfei.zhang@intel.com>
> Sent: Monday, June 6, 2022 10:00
> To: Zhang, Tianfei <tianfei.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>; dev@dpdk.org; thomas@monjalon.net;
> nipun.gupta@nxp.com; hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>
> Subject: RE: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> 
> 
> 
> > -----Original Message-----
> > From: Zhang, Tianfei <tianfei.zhang@intel.com>
> > Sent: Monday, June 6, 2022 9:53 AM
> > To: Huang, Wei <wei.huang@intel.com>; dev@dpdk.org;
> > thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> > Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Qi Z
> > <qi.z.zhang@intel.com>
> > Subject: RE: [PATCH v5 1/5] drivers/raw: introduce AFU raw device
> > driver
> >
> >
> >
> > > -----Original Message-----
> > > From: Huang, Wei <wei.huang@intel.com>
> > > Sent: Friday, May 27, 2022 1:37 PM
> > > To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> > > hemant.agrawal@nxp.com
> > > Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> > > <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>;
> > > Huang, Wei <wei.huang@intel.com>
> > > Subject: [PATCH v5 1/5] drivers/raw: introduce AFU raw device driver
> > >
> > > Add multi-function AFU raw device driver to manage various AFU
> > > (Acceleration Function Unit) in FPGA.
> > > This driver implements common AFU raw device interfaces and exposes
> > > them to application as standard raw device APIs.
> > > Normal application can operate specified AFU as below, 1. call
> > > rte_rawdev_pmd_get_named_dev() to find AFU raw device.
> > > 2. call rte_rawdev_configure() to initialize AFU raw device.
> > > 3. call rte_rawdev_selftest() to test function of AFU.
> > >
> > > Signed-off-by: Wei Huang <wei.huang@intel.com>
> > > ---
> > > v2: fix typo
> > > ---
> > > v3: fix build error in FreeBSD13-64, UB2004-32 and UB2204-32
> > > ---
> > > v4: fix coding style issue and build error in FreeBSD13-64
> > > ---
> > > v5: split patch into several patches
> > > ---
> > >  drivers/raw/afu_mf/afu_mf_rawdev.c | 425
> > > +++++++++++++++++++++++++++++++++++++
> > >  drivers/raw/afu_mf/afu_mf_rawdev.h |  71 +++++++
> > >  drivers/raw/afu_mf/meson.build     |   5 +
> > >  drivers/raw/afu_mf/version.map     |   3 +
> > >  drivers/raw/meson.build            |   1 +
> > >  5 files changed, 505 insertions(+)
> > >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.c
> > >  create mode 100644 drivers/raw/afu_mf/afu_mf_rawdev.h
> > >  create mode 100644 drivers/raw/afu_mf/meson.build  create mode
> > > 100644 drivers/raw/afu_mf/version.map
> >
> > I am thinking that just put those afu* c/h files into
> > drivers/raw/ifpga/ folder is better? Because this AFU PMD driver is work on
> ifpga.
> > And the file name of "afu_mf_rawdev.c" change to "afu_pmd_driver.c"?
> 
> Ohm, the file name of "afu_mf_rawdev.c" change to "afu_pmd_core.c"?
> Because this file is the common code of AFU PMD driver, and specific
> function are in separate c file, like afu_pmd_n3000.c
> 
agree
> >
> > >
> > > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.c
> > > b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > > new file mode 100644
> > > index 0000000..5be372a
> > > --- /dev/null
> > > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.c
> > > @@ -0,0 +1,425 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright 2022 Intel Corporation  */
> > > +
> > > +#include <errno.h>
> > > +#include <stdio.h>
> > > +#include <stdint.h>
> > > +#include <stdlib.h>
> > > +#include <string.h>
> > > +#include <unistd.h>
> > > +#include <fcntl.h>
> > > +#include <poll.h>
> > > +#include <sys/eventfd.h>
> > > +
> > > +#include <rte_eal.h>
> > > +#include <rte_malloc.h>
> > > +#include <rte_memzone.h>
> > > +#include <rte_rawdev_pmd.h>
> > > +
> > > +#include "afu_mf_rawdev.h"
> > > +
> > > +#define AFU_MF_PMD_RAWDEV_NAME rawdev_afu_mf
> > > +
> > > +static const struct rte_afu_uuid afu_uuid_map[] = {
> > > +	{ 0, 0 /* sentinel */ }
> > > +};
> > > +
> > > +static struct afu_mf_drv *afu_table[] = {
> > > +	NULL
> > > +};
> > > +
> > > +static inline int afu_mf_trylock(struct afu_mf_rawdev *dev) {
> > > +	int32_t x = 0;
> > > +
> > > +	if (!dev || !dev->shared)
> > > +		return -ENODEV;
> > > +
> > > +	x = __atomic_load_n(&dev->shared->lock, __ATOMIC_RELAXED);
> > > +
> > > +	if ((x != 0) || (__atomic_compare_exchange_n(&dev->shared->lock,
> > > +&x,
> > > 1,
> > > +				1, __ATOMIC_ACQUIRE,
> __ATOMIC_RELAXED)
> > > == 0))
> > > +		return -EBUSY;
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static inline void afu_mf_unlock(struct afu_mf_rawdev *dev) {
> > > +	if (!dev || !dev->shared)
> > > +		return;
> > > +
> > > +	__atomic_store_n(&dev->shared->lock, 0, __ATOMIC_RELEASE); }
> > > +
> > > +static int afu_mf_rawdev_configure(const struct rte_rawdev *rawdev,
> > > +	rte_rawdev_obj_t config, size_t config_size) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return -ENODEV;
> > > +
> > > +	if (dev->ops && dev->ops->config)
> > > +		ret = (*dev->ops->config)(dev, config, config_size);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_start(struct rte_rawdev *rawdev) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return -ENODEV;
> > > +
> > > +	ret = afu_mf_trylock(dev);
> > > +	if (ret) {
> > > +		AFU_MF_PMD_WARN("AFU is busy, please start it later");
> > > +		return ret;
> > > +	}
> > > +
> > > +	if (dev->ops && dev->ops->start)
> > > +		ret = (*dev->ops->start)(dev);
> > > +
> > > +	afu_mf_unlock(dev);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static void afu_mf_rawdev_stop(struct rte_rawdev *rawdev) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return;
> > > +
> > > +	ret = afu_mf_trylock(dev);
> > > +	if (ret) {
> > > +		AFU_MF_PMD_WARN("AFU is busy, please stop it later");
> > > +		return;
> > > +	}
> > > +
> > > +	if (dev->ops && dev->ops->stop)
> > > +		ret = (*dev->ops->stop)(dev);
> > > +
> > > +	afu_mf_unlock(dev);
> > > +}
> > > +
> > > +static int afu_mf_rawdev_close(struct rte_rawdev *rawdev) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return -ENODEV;
> > > +
> > > +	if (dev->ops && dev->ops->close)
> > > +		ret = (*dev->ops->close)(dev);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_reset(struct rte_rawdev *rawdev) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return -ENODEV;
> > > +
> > > +	ret = afu_mf_trylock(dev);
> > > +	if (ret) {
> > > +		AFU_MF_PMD_WARN("AFU is busy, please reset it later");
> > > +		return ret;
> > > +	}
> > > +
> > > +	if (dev->ops && dev->ops->reset)
> > > +		ret = (*dev->ops->reset)(dev);
> > > +
> > > +	afu_mf_unlock(dev);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_selftest(uint16_t dev_id) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	if (!rte_rawdev_pmd_is_valid_dev(dev_id))
> > > +		return -ENODEV;
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(&rte_rawdevs[dev_id]);
> > > +	if (!dev)
> > > +		return -ENOENT;
> > > +
> > > +	ret = afu_mf_trylock(dev);
> > > +	if (ret) {
> > > +		AFU_MF_PMD_WARN("AFU is busy, please test it later");
> > > +		return ret;
> > > +	}
> > > +
> > > +	if (dev->ops && dev->ops->test)
> > > +		ret = (*dev->ops->test)(dev);
> > > +
> > > +	afu_mf_unlock(dev);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) {
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	int ret = 0;
> > > +
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		return -ENODEV;
> > > +
> > > +	if (dev->ops && dev->ops->dump)
> > > +		ret = (*dev->ops->dump)(dev, f);
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static const struct rte_rawdev_ops afu_mf_rawdev_ops = {
> > > +	.dev_info_get = NULL,
> > > +	.dev_configure = afu_mf_rawdev_configure,
> > > +	.dev_start = afu_mf_rawdev_start,
> > > +	.dev_stop = afu_mf_rawdev_stop,
> > > +	.dev_close = afu_mf_rawdev_close,
> > > +	.dev_reset = afu_mf_rawdev_reset,
> > > +
> > > +	.queue_def_conf = NULL,
> > > +	.queue_setup = NULL,
> > > +	.queue_release = NULL,
> > > +	.queue_count = NULL,
> > > +
> > > +	.attr_get = NULL,
> > > +	.attr_set = NULL,
> > > +
> > > +	.enqueue_bufs = NULL,
> > > +	.dequeue_bufs = NULL,
> > > +
> > > +	.dump = afu_mf_rawdev_dump,
> > > +
> > > +	.xstats_get = NULL,
> > > +	.xstats_get_names = NULL,
> > > +	.xstats_get_by_name = NULL,
> > > +	.xstats_reset = NULL,
> > > +
> > > +	.firmware_status_get = NULL,
> > > +	.firmware_version_get = NULL,
> > > +	.firmware_load = NULL,
> > > +	.firmware_unload = NULL,
> > > +
> > > +	.dev_selftest = afu_mf_rawdev_selftest, };
> > > +
> > > +static int
> > > +afu_mf_shared_alloc(const char *name, struct afu_mf_shared **data,
> > > +	int socket_id)
> > > +{
> > > +	const struct rte_memzone *mz;
> > > +	char mz_name[RTE_MEMZONE_NAMESIZE];
> > > +	struct afu_mf_shared *ptr = NULL;
> > > +	int init_mz = 0;
> > > +
> > > +	if (!name || !data)
> > > +		return -EINVAL;
> > > +
> > > +	/* name format is afu_?|??:??.? which is unique */
> > > +	snprintf(mz_name, sizeof(mz_name), "%s", name);
> > > +
> > > +	mz = rte_memzone_lookup(mz_name);
> > > +	if (!mz) {
> > > +		mz = rte_memzone_reserve(mz_name,
> > > +				sizeof(struct afu_mf_shared),
> > > +				socket_id, 0);
> > > +		init_mz = 1;
> > > +	}
> > > +
> > > +	if (!mz) {
> > > +		AFU_MF_PMD_ERR("Allocate memory zone %s failed!",
> > > +			mz_name);
> > > +		return -ENOMEM;
> > > +	}
> > > +
> > > +	ptr = (struct afu_mf_shared *)mz->addr;
> > > +
> > > +	if (init_mz)  /* initialize memory zone on the first time */
> > > +		ptr->lock = 0;
> > > +
> > > +	*data = ptr;
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_name_get(struct rte_afu_device *afu_dev,
> > > +char
> > > *name,
> > > +	size_t size)
> > > +{
> > > +	int n = 0;
> > > +
> > > +	if (!afu_dev || !name || !size)
> > > +		return -EINVAL;
> > > +
> > > +	n = snprintf(name, size, "afu_%s", afu_dev->device.name);
> > > +	if (n >= (int)size) {
> > > +		AFU_MF_PMD_ERR("Name of AFU device is too long!");
> > > +		return -ENAMETOOLONG;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static struct afu_mf_ops *afu_mf_ops_get(struct rte_afu_uuid *afu_id)
> {
> > > +	struct afu_mf_drv *entry = NULL;
> > > +	int i = 0;
> > > +
> > > +	if (!afu_id)
> > > +		return NULL;
> > > +
> > > +	while ((entry = afu_table[i++])) {
> > > +		if ((entry->uuid.uuid_low == afu_id->uuid_low) &&
> > > +			(entry->uuid.uuid_high == afu_id->uuid_high))
> > > +			break;
> > > +	}
> > > +
> > > +	return entry ? entry->ops : NULL;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_create(struct rte_afu_device *afu_dev, int
> > > +socket_id) {
> > > +	struct rte_rawdev *rawdev = NULL;
> > > +	struct afu_mf_rawdev *dev = NULL;
> > > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > > +	int ret = 0;
> > > +
> > > +	if (!afu_dev)
> > > +		return -EINVAL;
> > > +
> > > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > > +	if (ret)
> > > +		return ret;
> > > +
> > > +	AFU_MF_PMD_INFO("Create raw device %s on NUMA node %d",
> > > +		name, socket_id);
> > > +
> > > +	/* Allocate device structure */
> > > +	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> > > afu_mf_rawdev),
> > > +				socket_id);
> > > +	if (!rawdev) {
> > > +		AFU_MF_PMD_ERR("Unable to allocate raw device");
> > > +		return -ENOMEM;
> > > +	}
> > > +
> > > +	rawdev->dev_ops = &afu_mf_rawdev_ops;
> > > +	rawdev->device = &afu_dev->device;
> > > +	rawdev->driver_name = afu_dev->driver->driver.name;
> > > +
> > > +	dev = afu_mf_rawdev_get_priv(rawdev);
> > > +	if (!dev)
> > > +		goto cleanup;
> > > +
> > > +	dev->rawdev = rawdev;
> > > +	dev->port = afu_dev->id.port;
> > > +	dev->addr = afu_dev->mem_resource[0].addr;
> > > +	dev->ops = afu_mf_ops_get(&afu_dev->id.uuid);
> > > +	if (dev->ops == NULL) {
> > > +		AFU_MF_PMD_ERR("Unsupported AFU device");
> > > +		goto cleanup;
> > > +	}
> > > +
> > > +	if (dev->ops->init) {
> > > +		ret = (*dev->ops->init)(dev);
> > > +		if (ret) {
> > > +			AFU_MF_PMD_ERR("Failed to init %s", name);
> > > +			goto cleanup;
> > > +		}
> > > +	}
> > > +
> > > +	ret = afu_mf_shared_alloc(name, &dev->shared, socket_id);
> > > +	if (ret)
> > > +		goto cleanup;
> > > +
> > > +	return ret;
> > > +
> > > +cleanup:
> > > +	rte_rawdev_pmd_release(rawdev);
> > > +	return ret;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_destroy(struct rte_afu_device *afu_dev) {
> > > +	struct rte_rawdev *rawdev = NULL;
> > > +	char name[RTE_RAWDEV_NAME_MAX_LEN] = {0};
> > > +	int ret = 0;
> > > +
> > > +	if (!afu_dev)
> > > +		return -EINVAL;
> > > +
> > > +	ret = afu_mf_rawdev_name_get(afu_dev, name, sizeof(name));
> > > +	if (ret)
> > > +		return ret;
> > > +
> > > +	AFU_MF_PMD_INFO("Destroy raw device %s", name);
> > > +
> > > +	rawdev = rte_rawdev_pmd_get_named_dev(name);
> > > +	if (!rawdev) {
> > > +		AFU_MF_PMD_ERR("Raw device %s not found", name);
> > > +		return -EINVAL;
> > > +	}
> > > +
> > > +	/* rte_rawdev_close is called by pmd_release */
> > > +	ret = rte_rawdev_pmd_release(rawdev);
> > > +	if (ret)
> > > +		AFU_MF_PMD_DEBUG("Device cleanup failed");
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int afu_mf_rawdev_probe(struct rte_afu_device *afu_dev) {
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +	return afu_mf_rawdev_create(afu_dev, rte_socket_id()); }
> > > +
> > > +static int afu_mf_rawdev_remove(struct rte_afu_device *afu_dev) {
> > > +	AFU_MF_PMD_FUNC_TRACE();
> > > +	return afu_mf_rawdev_destroy(afu_dev); }
> > > +
> > > +static struct rte_afu_driver afu_mf_pmd_drv = {
> > > +	.id_table = afu_uuid_map,
> > > +	.probe = afu_mf_rawdev_probe,
> > > +	.remove = afu_mf_rawdev_remove
> > > +};
> > > +
> > > +RTE_PMD_REGISTER_AFU(AFU_MF_PMD_RAWDEV_NAME,
> > afu_mf_pmd_drv);
> > > +RTE_LOG_REGISTER_DEFAULT(afu_mf_pmd_logtype, NOTICE);
> > > diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.h
> > > b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > > new file mode 100644
> > > index 0000000..df6715c
> > > --- /dev/null
> > > +++ b/drivers/raw/afu_mf/afu_mf_rawdev.h
> > > @@ -0,0 +1,71 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright 2022 Intel Corporation  */
> > > +
> > > +#ifndef __AFU_MF_RAWDEV_H__
> > > +#define __AFU_MF_RAWDEV_H__
> > > +
> > > +#include <stdint.h>
> > > +#include <stdio.h>
> > > +#include <unistd.h>
> > > +
> > > +#include <rte_cycles.h>
> > > +#include <rte_bus_ifpga.h>
> > > +#include <rte_rawdev.h>
> > > +
> > > +extern int afu_mf_pmd_logtype;
> > > +
> > > +#define AFU_MF_PMD_LOG(level, fmt, args...) \
> > > +	rte_log(RTE_LOG_ ## level, afu_mf_pmd_logtype, "%s(): " fmt "\n",
> \
> > > +		__func__, ##args)
> > > +
> > > +#define AFU_MF_PMD_FUNC_TRACE() AFU_MF_PMD_LOG(DEBUG,
> ">>")
> > > +
> > > +#define AFU_MF_PMD_DEBUG(fmt, args...) \
> > > +	AFU_MF_PMD_LOG(DEBUG, fmt, ## args) #define
> AFU_MF_PMD_INFO(fmt,
> > > +args...) \
> > > +	AFU_MF_PMD_LOG(INFO, fmt, ## args) #define
> AFU_MF_PMD_ERR(fmt,
> > > +args...) \
> > > +	AFU_MF_PMD_LOG(ERR, fmt, ## args)
> > > +#define AFU_MF_PMD_WARN(fmt, args...) \
> > > +	AFU_MF_PMD_LOG(WARNING, fmt, ## args)
> > > +
> > > +struct afu_mf_rawdev;
> > > +
> > > +struct afu_mf_ops {
> > > +	int (*init)(struct afu_mf_rawdev *dev);
> > > +	int (*config)(struct afu_mf_rawdev *dev, void *config,
> > > +		size_t config_size);
> > > +	int (*start)(struct afu_mf_rawdev *dev);
> > > +	int (*stop)(struct afu_mf_rawdev *dev);
> > > +	int (*test)(struct afu_mf_rawdev *dev);
> > > +	int (*close)(struct afu_mf_rawdev *dev);
> > > +	int (*reset)(struct afu_mf_rawdev *dev);
> > > +	int (*dump)(struct afu_mf_rawdev *dev, FILE *f); };
> > > +
> > > +struct afu_mf_drv {
> > > +	struct rte_afu_uuid uuid;
> > > +	struct afu_mf_ops *ops;
> > > +};
> > > +
> > > +struct afu_mf_shared {
> > > +	int32_t lock;
> > > +};
> > > +
> > > +struct afu_mf_rawdev {
> > > +	struct rte_rawdev *rawdev;  /* point to parent raw device */
> > > +	struct afu_mf_shared *shared;  /* shared data for multi-process */
> > > +	struct afu_mf_ops *ops;  /* device operation functions */
> > > +	int port;  /* index of port the AFU attached */
> > > +	void *addr;  /* base address of AFU registers */
> > > +	void *priv;  /* private driver data */ };
> > > +
> > > +static inline struct afu_mf_rawdev * afu_mf_rawdev_get_priv(const
> > > +struct rte_rawdev *rawdev) {
> > > +	return rawdev ? (struct afu_mf_rawdev *)rawdev->dev_private :
> > > +NULL; }
> > > +
> > > +#endif /* __AFU_MF_RAWDEV_H__ */
> > > diff --git a/drivers/raw/afu_mf/meson.build
> > > b/drivers/raw/afu_mf/meson.build new file mode 100644 index
> > > 0000000..80526a2
> > > --- /dev/null
> > > +++ b/drivers/raw/afu_mf/meson.build
> > > @@ -0,0 +1,5 @@
> > > +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2022 Intel
> > > +Corporation
> > > +
> > > +deps += ['rawdev', 'bus_pci', 'bus_ifpga'] sources =
> > > +files('afu_mf_rawdev.c')
> > > diff --git a/drivers/raw/afu_mf/version.map
> > > b/drivers/raw/afu_mf/version.map new file mode 100644 index
> > > 0000000..c2e0723
> > > --- /dev/null
> > > +++ b/drivers/raw/afu_mf/version.map
> > > @@ -0,0 +1,3 @@
> > > +DPDK_22 {
> > > +	local: *;
> > > +};
> > > diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> > > 05e7de1..c3627f7 100644
> > > --- a/drivers/raw/meson.build
> > > +++ b/drivers/raw/meson.build
> > > @@ -6,6 +6,7 @@ if is_windows
> > >  endif
> > >
> > >  drivers = [
> > > +        'afu_mf',
> > >          'cnxk_bphy',
> > >          'cnxk_gpio',
> > >          'dpaa2_cmdif',
> > > --
> > > 1.8.3.1