[v1,3/3] dma/idxd: add API to create and attach to window

diff --git a/doc/guides/dmadevs/idxd.rst b/doc/guides/dmadevs/idxd.rst
index cb8f1fe729..b0439377f8 100644
--- a/doc/guides/dmadevs/idxd.rst
+++ b/doc/guides/dmadevs/idxd.rst
@@ -225,3 +225,55 @@  which operation failed and kick off the device to continue processing operations
    if (error){
       status_count = rte_dma_completed_status(dev_id, vchan, COMP_BURST_SZ, &idx, status);
    }
+
+Performing Inter-Domain operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Refer to the :ref:`Enqueue / Dequeue APIs <dmadev_enqueue_dequeue>` section of the dmadev library
+documentation for details on operation enqueue, submission and completion API usage.
+
+Refer to the :ref:`Inter-domain operations <dmadev_inter_dom>` section of the dmadev library
+documentation for details on inter-domain operations.
+
+Intel(R) IDXD currently supports the following inter-domain operations:
+
+* Copy operation
+* Fill operation
+
+To use these operations with the IDXD driver, the following program flow should
+be adhered to:
+
+* Process A that wishes to share its memory with others, shall call
+  ``rte_idxd_window_create()``, which will return a file descriptor
+* Process A is to send above mentioned file descriptor to any recipient process
+  (usually over IPC) that wishes to attach to that window
+* Process B, after receiving above mentioned file descriptor from process A over
+  IPC, shall call ``rte_idxd_window_attach()`` and receive an inter-pasid handle
+* Process B shall use this handle as an argument for inter-domain operations
+  using DMA device API
+
+The controller ID parameter for create/attach functions in this case would be
+the controller ID of configured DSA2 devices (located under ``rte_dma_info``
+structure), but which can also be read from ``accel-config`` tool, or from the
+DSA2 work queue name (e.g. work queue ``wq0.3`` would have ``0`` as its
+controller ID).
+
+The ``rte_idxd_window_create()`` call will accept a ``flags`` argument, which
+can contain the following bits:
+
+* ``RTE_IDXD_WIN_FLAGS_PROT_READ`` - allow other process to read from memory
+  region to be shared
+  - In this case, the remote process will be using the resulting inter-pasid
+    handle as source handle for inter-domain DMA operations (and set the
+    ``RTE_DMA_OP_FLAG_SRC_HANDLE`` DMA operation flag)
+* ``RTE_IDXD_WIN_FLAGS_PROT_WRITE`` - allow other process to write into memory
+  region to be shared
+  - In this case, the remote process will be using the resulting inter-pasid
+    handle as destination handle for inter-domain DMA operations (and set the
+    ``RTE_DMA_OP_FLAG_DST_HANDLE`` DMA operation flag)
+* ``RTE_IDXD_WIN_FLAGS_WIN_CHECK`` - if this flag is not set, the remote process
+  will be allowed unrestricted access to entire memory space of the owner
+  process
+* ``RTE_IDXD_WIN_FLAGS_OFFSET_MODE`` - addresses for DMA operations will have to
+  be specified as offsets from base address of the memory region to be shared
+* ``RTE_IDXD_WIN_FLAGS_TYPE_SAMS`` - enable multi-submitter mode.
diff --git a/drivers/dma/idxd/idxd_inter_dom.c b/drivers/dma/idxd/idxd_inter_dom.c
new file mode 100644
index 0000000000..21dcd6980d
--- /dev/null
+++ b/drivers/dma/idxd/idxd_inter_dom.c
@@ -0,0 +1,166 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dirent.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_idxd_inter_dom.h>
+
+#include "idxd_internal.h"
+
+#define IDXD_TYPE       ('d')
+#define IDXD_IOC_BASE   100
+#define IDXD_WIN_BASE   200
+
+enum idxd_win_type {
+	IDXD_WIN_TYPE_SA_SS = 0,
+	IDXD_WIN_TYPE_SA_MS,
+};
+
+#define IDXD_WIN_FLAGS_MASK (RTE_IDXD_WIN_FLAGS_PROT_READ | RTE_IDXD_WIN_FLAGS_PROT_WRITE |\
+		RTE_IDXD_WIN_FLAGS_WIN_CHECK | RTE_IDXD_WIN_FLAGS_OFFSET_MODE|\
+		RTE_IDXD_WIN_FLAGS_TYPE_SAMS)
+
+struct idxd_win_param {
+	uint64_t base;          /* Window base */
+	uint64_t size;          /* Window size */
+	uint32_t type;          /* Window type, see enum idxd_win_type */
+	uint16_t flags;         /* See IDXD windows flags */
+	uint16_t handle;        /* Window handle returned by driver */
+} __attribute__((packed));
+
+struct idxd_win_attach {
+	uint32_t fd;            /* Window file descriptor returned by IDXD_WIN_CREATE */
+	uint16_t handle;        /* Window handle returned by driver */
+} __attribute__((packed));
+
+struct idxd_win_fault {
+	uint64_t offset;        /* Window offset of faulting address */
+	uint64_t len;           /* Faulting range */
+	uint32_t write_fault;   /* Fault generated on write */
+} __attribute__((packed));
+
+#define IDXD_WIN_CREATE         _IOWR(IDXD_TYPE, IDXD_IOC_BASE + 1, struct idxd_win_param)
+#define IDXD_WIN_ATTACH         _IOR(IDXD_TYPE, IDXD_IOC_BASE + 2, struct idxd_win_attach)
+#define IDXD_WIN_FAULT          _IOR(IDXD_TYPE, IDXD_WIN_BASE + 1, struct idxd_win_fault)
+#define DSA_DEV_PATH "/dev/dsa"
+
+static inline const char *
+dsa_get_dev_path(void)
+{
+	const char *path = getenv("DSA_DEV_PATH");
+	return path ? path : DSA_DEV_PATH;
+}
+
+static int
+dsa_find_work_queue(int controller_id)
+{
+	char dev_templ[PATH_MAX], path_templ[PATH_MAX];
+	const char *path = dsa_get_dev_path();
+	struct dirent *wq;
+	DIR *dev_dir;
+	int fd = -1;
+
+	/* construct work queue path template */
+	snprintf(dev_templ, sizeof(dev_templ), "wq%d.", controller_id);
+
+	/* open the DSA device directory */
+	dev_dir = opendir(path);
+	if (dev_dir == NULL)
+		return -1;
+
+	/* find any available work queue */
+	while ((wq = readdir(dev_dir)) != NULL) {
+		/* skip things that aren't work queues */
+		if (strncmp(wq->d_name, dev_templ, strlen(dev_templ)) != 0)
+			continue;
+
+		/* try this work queue */
+		snprintf(path_templ, sizeof(path_templ), "%s/%s", path, wq->d_name);
+
+		fd = open(path_templ, O_RDWR);
+		if (fd < 0)
+			continue;
+
+		break;
+	}
+
+	return fd;
+}
+
+int
+rte_idxd_window_create(int controller_id, void *win_addr,
+	unsigned int win_len, int flags)
+{
+	struct idxd_win_param param = {0};
+	int idpte_fd, fd;
+
+	fd = dsa_find_work_queue(controller_id);
+
+	/* did we find anything? */
+	if (fd < 0) {
+		IDXD_PMD_ERR("%s(): creatomg idpt window failed", __func__);
+		return -1;
+	}
+
+	/* create a wormhole into a parallel reality... */
+	param.base = (uint64_t)win_addr;
+	param.size = win_len;
+	param.flags = flags & IDXD_WIN_FLAGS_MASK;
+	param.type = (flags & RTE_IDXD_WIN_FLAGS_TYPE_SAMS) ?
+		IDXD_WIN_TYPE_SA_MS : IDXD_WIN_TYPE_SA_SS;
+
+	idpte_fd = ioctl(fd, IDXD_WIN_CREATE, &param);
+
+	close(fd);
+
+	if (idpte_fd < 0)
+		rte_errno = idpte_fd;
+
+	return idpte_fd;
+}
+
+int
+rte_idxd_window_attach(int controller_id, int idpte_fd,
+	uint16_t *handle)
+{
+
+	struct idxd_win_attach win_attach = {0};
+	int ret, fd;
+
+	if (handle == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	fd = dsa_find_work_queue(controller_id);
+
+	/* did we find anything? */
+	if (fd < 0) {
+		IDXD_PMD_ERR("%s(): creatomg idpt window failed", __func__);
+		rte_errno = ENOENT;
+		return -1;
+	}
+
+	/* get access to someone else's wormhole */
+	win_attach.fd = idpte_fd;
+
+	ret = ioctl(fd, IDXD_WIN_ATTACH, &win_attach);
+	if (ret != 0) {
+		IDXD_PMD_ERR("%s(): attaching idpt window failed: %s",
+				__func__, strerror(ret));
+		rte_errno = ret;
+		return -1;
+	}
+
+	*handle = win_attach.handle;
+
+	return 0;
+}
diff --git a/drivers/dma/idxd/meson.build b/drivers/dma/idxd/meson.build
index c5403b431c..da73ab340c 100644
--- a/drivers/dma/idxd/meson.build
+++ b/drivers/dma/idxd/meson.build
@@ -22,5 +22,10 @@  sources = files(
 )
 
 if is_linux
-    sources += files('idxd_bus.c')
+    sources += files(
+    'idxd_bus.c',
+    'idxd_inter_dom.c',
+)
 endif
+
+headers = files('rte_idxd_inter_dom.h')
diff --git a/drivers/dma/idxd/rte_idxd_inter_dom.h b/drivers/dma/idxd/rte_idxd_inter_dom.h
new file mode 100644
index 0000000000..c31f3777c9
--- /dev/null
+++ b/drivers/dma/idxd/rte_idxd_inter_dom.h
@@ -0,0 +1,79 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _RTE_IDXD_INTER_DOM_H_
+#define _RTE_IDXD_INTER_DOM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+#include <rte_compat.h>
+
+/** Allow reading from address space. */
+#define RTE_IDXD_WIN_FLAGS_PROT_READ    0x0001
+/** Allow writing to address space. */
+#define RTE_IDXD_WIN_FLAGS_PROT_WRITE   0x0002
+/** If this flag not set, the entire address space will be accessible. */
+#define RTE_IDXD_WIN_FLAGS_WIN_CHECK    0x0004
+/** Destination addresses are offsets from window base address. */
+#define RTE_IDXD_WIN_FLAGS_OFFSET_MODE  0x0008
+/* multiple submitter flag. If not set - single submitter type will be used. */
+#define RTE_IDXD_WIN_FLAGS_TYPE_SAMS    0x0010
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Create an inter-pasid window to allow another process to access this process'
+ * memory. This function returns a file descriptor for the window, that can be
+ * used by another process to access this window.
+ *
+ * @param controller_id
+ *   IDXD controller device ID.
+ * @param win_addr
+ *   Base address of memory chunk being shared (ignored if
+ *   `RTE_IDXD_WIN_FLAGS_WIN_CHECK` is not set).
+ * @param win_len
+ *   Length of memory chunk being shared (ignored if
+ *   `RTE_IDXD_WIN_FLAGS_WIN_CHECK` is not set).
+ * @param flags
+ *   Flags to configure the window.
+ * @return
+ *   Non-negative on success.
+ *   Negative on error.
+ */
+__rte_experimental
+int rte_idxd_window_create(int controller_id, void *win_addr,
+	unsigned int win_len, int flags);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Attach to an inter-pasid window of another process. This function expects a
+ * file descriptor returned by `rte_idxd_window_create()`, and will set the
+ * value pointed to by `handle`. This handle can then be used to perform
+ * inter-domain DMA operations.
+ *
+ * @param controller_id
+ *   IDXD controller device ID.
+ * @param idpte_fd
+ *   File descriptor for another process's window
+ * @param handle
+ *   Pointer to a variable to receive the handle.
+ * @return
+ *   0 on success.
+ *   Negative on error.
+ */
+__rte_experimental
+int rte_idxd_window_attach(int controller_id, int idpte_fd, uint16_t *handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_IDXD_INTER_DOM_H_ */
diff --git a/drivers/dma/idxd/version.map b/drivers/dma/idxd/version.map
new file mode 100644
index 0000000000..e091bb7c09
--- /dev/null
+++ b/drivers/dma/idxd/version.map
@@ -0,0 +1,11 @@ 
+DPDK_23 {
+	local: *;
+};
+
+
+EXPERIMENTAL {
+	global:
+
+	rte_idxd_window_create;
+	rte_idxd_window_attach;
+};