From patchwork Fri May 19 16:39:43 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Anatoly Burakov X-Patchwork-Id: 24407 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 2EBCC3252; Fri, 19 May 2017 18:39:54 +0200 (CEST) Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by dpdk.org (Postfix) with ESMTP id 3BE522C6D for ; Fri, 19 May 2017 18:39:48 +0200 (CEST) Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga104.jf.intel.com with ESMTP; 19 May 2017 09:39:47 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos; i="5.38,364,1491289200"; d="scan'208"; a="1171757718" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by fmsmga002.fm.intel.com with ESMTP; 19 May 2017 09:39:47 -0700 Received: from sivswdev02.ir.intel.com (sivswdev02.ir.intel.com [10.237.217.46]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id v4JGdkGW024642 for ; Fri, 19 May 2017 17:39:46 +0100 Received: from sivswdev02.ir.intel.com (localhost [127.0.0.1]) by sivswdev02.ir.intel.com with ESMTP id v4JGdko9015318 for ; Fri, 19 May 2017 17:39:46 +0100 Received: (from aburakov@localhost) by sivswdev02.ir.intel.com with ? id v4JGdkVW015314 for dev@dpdk.org; Fri, 19 May 2017 17:39:46 +0100 From: Anatoly Burakov To: dev@dpdk.org Date: Fri, 19 May 2017 17:39:43 +0100 Message-Id: <1495211986-15177-2-git-send-email-anatoly.burakov@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1495211986-15177-1-git-send-email-anatoly.burakov@intel.com> References: <1495211986-15177-1-git-send-email-anatoly.burakov@intel.com> Subject: [dpdk-dev] [RFC 1/4] vfio: refactor sockets into separate files X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Signed-off-by: Anatoly Burakov --- lib/librte_eal/linuxapp/eal/Makefile | 1 + lib/librte_eal/linuxapp/eal/eal_mp_socket.c | 301 +++++++++++++++++++++++++ lib/librte_eal/linuxapp/eal/eal_mp_socket.h | 54 +++++ lib/librte_eal/linuxapp/eal/eal_vfio.c | 20 +- lib/librte_eal/linuxapp/eal/eal_vfio.h | 24 +- lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c | 243 +++----------------- 6 files changed, 410 insertions(+), 233 deletions(-) create mode 100755 lib/librte_eal/linuxapp/eal/eal_mp_socket.c create mode 100755 lib/librte_eal/linuxapp/eal/eal_mp_socket.h diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile index 640afd0..24aab8d 100644 --- a/lib/librte_eal/linuxapp/eal/Makefile +++ b/lib/librte_eal/linuxapp/eal/Makefile @@ -60,6 +60,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_xen_memory.c endif SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_thread.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_log.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_mp_socket.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_vfio.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_vfio_mp_sync.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_pci.c diff --git a/lib/librte_eal/linuxapp/eal/eal_mp_socket.c b/lib/librte_eal/linuxapp/eal/eal_mp_socket.c new file mode 100755 index 0000000..18c5a72 --- /dev/null +++ b/lib/librte_eal/linuxapp/eal/eal_mp_socket.c @@ -0,0 +1,301 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +/* sys/un.h with __USE_MISC uses strlen, which is unsafe */ +#ifdef __USE_MISC +#define REMOVED_USE_MISC +#undef __USE_MISC +#endif +#include +/* make sure we redefine __USE_MISC only if it was previously undefined */ +#ifdef REMOVED_USE_MISC +#define __USE_MISC +#undef REMOVED_USE_MISC +#endif + +#include + +#include "eal_mp_socket.h" + +/** + * @file + * Sockets for communication between primary and secondary processes. + */ + +#define CMSGLEN (CMSG_LEN(sizeof(int))) +#define FD_TO_CMSGHDR(fd, chdr) \ + do {\ + (chdr).cmsg_len = CMSGLEN;\ + (chdr).cmsg_level = SOL_SOCKET;\ + (chdr).cmsg_type = SCM_RIGHTS;\ + memcpy((chdr).__cmsg_data, &(fd), sizeof(fd));\ + } while (0) +#define CMSGHDR_TO_FD(chdr, fd) \ + memcpy(&(fd), (chdr).__cmsg_data, sizeof(fd)) + +/* send a request, return -1 on error */ +int +eal_mp_sync_send_request(int socket, int req) +{ + struct msghdr hdr; + struct iovec iov; + int buf; + int ret; + + memset(&hdr, 0, sizeof(hdr)); + + buf = req; + + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = (char *) &buf; + iov.iov_len = sizeof(buf); + + ret = sendmsg(socket, &hdr, 0); + if (ret < 0) + return -1; + return 0; +} + +/* receive a request and return it */ +int +eal_mp_sync_receive_request(int socket) +{ + int buf; + struct msghdr hdr; + struct iovec iov; + int ret, req; + + memset(&hdr, 0, sizeof(hdr)); + + buf = SOCKET_ERR; + + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = (char *) &buf; + iov.iov_len = sizeof(buf); + + ret = recvmsg(socket, &hdr, 0); + if (ret < 0) + return -1; + + req = buf; + + return req; +} + +/* send OK in message, fd in control message */ +int +eal_mp_sync_send_fd(int socket, int fd) +{ + int buf; + struct msghdr hdr; + struct cmsghdr *chdr; + char chdr_buf[CMSGLEN]; + struct iovec iov; + int ret; + + chdr = (struct cmsghdr *) chdr_buf; + memset(chdr, 0, sizeof(chdr_buf)); + memset(&hdr, 0, sizeof(hdr)); + + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = (char *) &buf; + iov.iov_len = sizeof(buf); + hdr.msg_control = chdr; + hdr.msg_controllen = CMSGLEN; + + buf = SOCKET_FD; + FD_TO_CMSGHDR(fd, *chdr); + + ret = sendmsg(socket, &hdr, 0); + if (ret < 0) + return -1; + return 0; +} + +/* receive OK in message, fd in control message */ +int +eal_mp_sync_receive_fd(int socket) +{ + int buf; + struct msghdr hdr; + struct cmsghdr *chdr; + char chdr_buf[CMSGLEN]; + struct iovec iov; + int ret, req, fd = -1; + + buf = SOCKET_ERR; + + chdr = (struct cmsghdr *) chdr_buf; + memset(chdr, 0, sizeof(chdr_buf)); + memset(&hdr, 0, sizeof(hdr)); + + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = (char *) &buf; + iov.iov_len = sizeof(buf); + hdr.msg_control = chdr; + hdr.msg_controllen = CMSGLEN; + + ret = recvmsg(socket, &hdr, 0); + if (ret < 0) + return -1; + + req = buf; + + if (req != SOCKET_FD) + return -1; + + CMSGHDR_TO_FD(*chdr, fd); + + return fd; +} + +/* send path, return -1 on error */ +int eal_mp_sync_send_data(int socket, void *data, int len) +{ + struct msghdr hdr; + struct iovec iov; + int ret; + + memset(&hdr, 0, sizeof(hdr)); + + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = data; + iov.iov_len = (size_t) len; + + ret = sendmsg(socket, &hdr, 0); + if (ret < 0) + return -1; + return 0; +} + +/* receive a path into buffer of specified sz */ +int eal_mp_sync_receive_data(int socket, void *data, int sz) +{ + struct msghdr hdr; + struct iovec iov; + int ret; + + memset(&hdr, 0, sizeof(hdr)); + + /* receive path */ + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + iov.iov_base = data; + iov.iov_len = (size_t) sz; + + ret = recvmsg(socket, &hdr, 0); + if (ret < 0 || (hdr.msg_flags & MSG_TRUNC)) + return -1; + + /* path received */ + + return 0; +} + + +/* connect socket_fd in secondary process to the primary process's socket */ +int +eal_mp_sync_connect_to_primary(const char *path) +{ + struct sockaddr_un addr; + socklen_t sockaddr_len; + int socket_fd; + + /* set up a socket */ + socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0); + if (socket_fd < 0) { + RTE_LOG(ERR, EAL, "Failed to create socket!\n"); + return -1; + } + + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); + addr.sun_family = AF_UNIX; + + sockaddr_len = sizeof(struct sockaddr_un); + + if (connect(socket_fd, (struct sockaddr *) &addr, sockaddr_len) == 0) + return socket_fd; + + /* if connect failed */ + close(socket_fd); + return -1; +} + +int +eal_mp_sync_socket_setup(const char *path) +{ + int ret, socket_fd; + struct sockaddr_un addr; + socklen_t sockaddr_len; + + /* set up a socket */ + socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0); + if (socket_fd < 0) { + RTE_LOG(ERR, EAL, "Failed to create socket!\n"); + return -1; + } + + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); + addr.sun_family = AF_UNIX; + + sockaddr_len = sizeof(struct sockaddr_un); + + unlink(addr.sun_path); + + ret = bind(socket_fd, (struct sockaddr *) &addr, sockaddr_len); + if (ret) { + RTE_LOG(ERR, EAL, "Failed to bind socket: %s!\n", strerror(errno)); + close(socket_fd); + return -1; + } + + ret = listen(socket_fd, 50); + if (ret) { + RTE_LOG(ERR, EAL, "Failed to listen: %s!\n", strerror(errno)); + close(socket_fd); + return -1; + } + + return socket_fd; +} diff --git a/lib/librte_eal/linuxapp/eal/eal_mp_socket.h b/lib/librte_eal/linuxapp/eal/eal_mp_socket.h new file mode 100755 index 0000000..2c46969 --- /dev/null +++ b/lib/librte_eal/linuxapp/eal/eal_mp_socket.h @@ -0,0 +1,54 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef EAL_MP_SOCKET_H +#define EAL_MP_SOCKET_H + +/* + * Function prototypes for multiprocess sync functions + */ +int eal_mp_sync_send_request(int socket, int req); +int eal_mp_sync_receive_request(int socket); +int eal_mp_sync_send_fd(int socket, int fd); +int eal_mp_sync_receive_fd(int socket); +int eal_mp_sync_send_data(int socket, void *data, int len); +int eal_mp_sync_receive_data(int socket, void *data, int sz); +int eal_mp_sync_connect_to_primary(const char *path); +int eal_mp_sync_socket_setup(const char *path); + +#define SOCKET_REQ_USER 0x100 +#define SOCKET_OK 0 +#define SOCKET_FD 1 +#define SOCKET_ERR -1 + +#endif // EAL_MP_SOCKET_H diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c index 53ac725..485fbbe 100644 --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -140,23 +140,23 @@ vfio_get_group_fd(int iommu_group_no) RTE_LOG(ERR, EAL, " cannot connect to primary process!\n"); return -1; } - if (vfio_mp_sync_send_request(socket_fd, SOCKET_REQ_GROUP) < 0) { + if (eal_mp_sync_send_request(socket_fd, SOCKET_REQ_GROUP) < 0) { RTE_LOG(ERR, EAL, " cannot request container fd!\n"); close(socket_fd); return -1; } - if (vfio_mp_sync_send_request(socket_fd, iommu_group_no) < 0) { + if (eal_mp_sync_send_request(socket_fd, iommu_group_no) < 0) { RTE_LOG(ERR, EAL, " cannot send group number!\n"); close(socket_fd); return -1; } - ret = vfio_mp_sync_receive_request(socket_fd); + ret = eal_mp_sync_receive_request(socket_fd); switch (ret) { case SOCKET_NO_FD: close(socket_fd); return 0; case SOCKET_OK: - vfio_group_fd = vfio_mp_sync_receive_fd(socket_fd); + vfio_group_fd = eal_mp_sync_receive_fd(socket_fd); /* if we got the fd, return it */ if (vfio_group_fd > 0) { close(socket_fd); @@ -247,19 +247,19 @@ clear_group(int vfio_group_fd) return -1; } - if (vfio_mp_sync_send_request(socket_fd, SOCKET_CLR_GROUP) < 0) { + if (eal_mp_sync_send_request(socket_fd, SOCKET_CLR_GROUP) < 0) { RTE_LOG(ERR, EAL, " cannot request container fd!\n"); close(socket_fd); return -1; } - if (vfio_mp_sync_send_request(socket_fd, vfio_group_fd) < 0) { + if (eal_mp_sync_send_request(socket_fd, vfio_group_fd) < 0) { RTE_LOG(ERR, EAL, " cannot send group fd!\n"); close(socket_fd); return -1; } - ret = vfio_mp_sync_receive_request(socket_fd); + ret = eal_mp_sync_receive_request(socket_fd); switch (ret) { case SOCKET_NO_FD: RTE_LOG(ERR, EAL, " BAD VFIO group fd!\n"); @@ -628,12 +628,12 @@ vfio_get_container_fd(void) RTE_LOG(ERR, EAL, " cannot connect to primary process!\n"); return -1; } - if (vfio_mp_sync_send_request(socket_fd, SOCKET_REQ_CONTAINER) < 0) { + if (eal_mp_sync_send_request(socket_fd, SOCKET_REQ_CONTAINER) < 0) { RTE_LOG(ERR, EAL, " cannot request container fd!\n"); close(socket_fd); return -1; } - vfio_container_fd = vfio_mp_sync_receive_fd(socket_fd); + vfio_container_fd = eal_mp_sync_receive_fd(socket_fd); if (vfio_container_fd < 0) { RTE_LOG(ERR, EAL, " cannot get container fd!\n"); close(socket_fd); diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h index 5ff63e5..66e7139 100644 --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,6 +42,8 @@ #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) #include +#include "eal_mp_socket.h" + #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) #define RTE_PCI_MSIX_TABLE_BIR 0x7 #define RTE_PCI_MSIX_TABLE_OFFSET 0xfffffff8 @@ -119,15 +121,6 @@ struct vfio_iommu_spapr_tce_info { #define VFIO_MAX_GROUPS 64 /* - * Function prototypes for VFIO multiprocess sync functions - */ -int vfio_mp_sync_send_request(int socket, int req); -int vfio_mp_sync_receive_request(int socket); -int vfio_mp_sync_send_fd(int socket, int fd); -int vfio_mp_sync_receive_fd(int socket); -int vfio_mp_sync_connect_to_primary(void); - -/* * we don't need to store device fd's anywhere since they can be obtained from * the group fd via an ioctl() call. */ @@ -209,13 +202,12 @@ int pci_vfio_enable(void); int pci_vfio_is_enabled(void); int vfio_mp_sync_setup(void); +int vfio_mp_sync_connect_to_primary(void); -#define SOCKET_REQ_CONTAINER 0x100 -#define SOCKET_REQ_GROUP 0x200 -#define SOCKET_CLR_GROUP 0x300 -#define SOCKET_OK 0x0 -#define SOCKET_NO_FD 0x1 -#define SOCKET_ERR 0xFF +#define SOCKET_REQ_CONTAINER SOCKET_REQ_USER + 0 +#define SOCKET_REQ_GROUP SOCKET_REQ_USER + 1 +#define SOCKET_CLR_GROUP SOCKET_REQ_USER + 2 +#define SOCKET_NO_FD SOCKET_REQ_USER + 3 #define VFIO_PRESENT #endif /* kernel version */ diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c index 7e8095c..17b0539 100644 --- a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c +++ b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -66,21 +66,10 @@ #ifdef VFIO_PRESENT -#define SOCKET_PATH_FMT "%s/.%s_mp_socket" -#define CMSGLEN (CMSG_LEN(sizeof(int))) -#define FD_TO_CMSGHDR(fd, chdr) \ - do {\ - (chdr).cmsg_len = CMSGLEN;\ - (chdr).cmsg_level = SOL_SOCKET;\ - (chdr).cmsg_type = SCM_RIGHTS;\ - memcpy((chdr).__cmsg_data, &(fd), sizeof(fd));\ - } while (0) -#define CMSGHDR_TO_FD(chdr, fd) \ - memcpy(&(fd), (chdr).__cmsg_data, sizeof(fd)) - -static pthread_t socket_thread; -static int mp_socket_fd; +#define SOCKET_PATH_FMT "%s/.%s_mp_vfio_socket" +static pthread_t vfio_socket_thread; +static int mp_vfio_socket_fd; /* get socket path (/var/run if root, $HOME otherwise) */ static void @@ -111,156 +100,6 @@ get_socket_path(char *buffer, int bufsz) * in case of any error, socket is closed. */ -/* send a request, return -1 on error */ -int -vfio_mp_sync_send_request(int socket, int req) -{ - struct msghdr hdr; - struct iovec iov; - int buf; - int ret; - - memset(&hdr, 0, sizeof(hdr)); - - buf = req; - - hdr.msg_iov = &iov; - hdr.msg_iovlen = 1; - iov.iov_base = (char *) &buf; - iov.iov_len = sizeof(buf); - - ret = sendmsg(socket, &hdr, 0); - if (ret < 0) - return -1; - return 0; -} - -/* receive a request and return it */ -int -vfio_mp_sync_receive_request(int socket) -{ - int buf; - struct msghdr hdr; - struct iovec iov; - int ret, req; - - memset(&hdr, 0, sizeof(hdr)); - - buf = SOCKET_ERR; - - hdr.msg_iov = &iov; - hdr.msg_iovlen = 1; - iov.iov_base = (char *) &buf; - iov.iov_len = sizeof(buf); - - ret = recvmsg(socket, &hdr, 0); - if (ret < 0) - return -1; - - req = buf; - - return req; -} - -/* send OK in message, fd in control message */ -int -vfio_mp_sync_send_fd(int socket, int fd) -{ - int buf; - struct msghdr hdr; - struct cmsghdr *chdr; - char chdr_buf[CMSGLEN]; - struct iovec iov; - int ret; - - chdr = (struct cmsghdr *) chdr_buf; - memset(chdr, 0, sizeof(chdr_buf)); - memset(&hdr, 0, sizeof(hdr)); - - hdr.msg_iov = &iov; - hdr.msg_iovlen = 1; - iov.iov_base = (char *) &buf; - iov.iov_len = sizeof(buf); - hdr.msg_control = chdr; - hdr.msg_controllen = CMSGLEN; - - buf = SOCKET_OK; - FD_TO_CMSGHDR(fd, *chdr); - - ret = sendmsg(socket, &hdr, 0); - if (ret < 0) - return -1; - return 0; -} - -/* receive OK in message, fd in control message */ -int -vfio_mp_sync_receive_fd(int socket) -{ - int buf; - struct msghdr hdr; - struct cmsghdr *chdr; - char chdr_buf[CMSGLEN]; - struct iovec iov; - int ret, req, fd; - - buf = SOCKET_ERR; - - chdr = (struct cmsghdr *) chdr_buf; - memset(chdr, 0, sizeof(chdr_buf)); - memset(&hdr, 0, sizeof(hdr)); - - hdr.msg_iov = &iov; - hdr.msg_iovlen = 1; - iov.iov_base = (char *) &buf; - iov.iov_len = sizeof(buf); - hdr.msg_control = chdr; - hdr.msg_controllen = CMSGLEN; - - ret = recvmsg(socket, &hdr, 0); - if (ret < 0) - return -1; - - req = buf; - - if (req != SOCKET_OK) - return -1; - - CMSGHDR_TO_FD(*chdr, fd); - - return fd; -} - -/* connect socket_fd in secondary process to the primary process's socket */ -int -vfio_mp_sync_connect_to_primary(void) -{ - struct sockaddr_un addr; - socklen_t sockaddr_len; - int socket_fd; - - /* set up a socket */ - socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0); - if (socket_fd < 0) { - RTE_LOG(ERR, EAL, "Failed to create socket!\n"); - return -1; - } - - get_socket_path(addr.sun_path, sizeof(addr.sun_path)); - addr.sun_family = AF_UNIX; - - sockaddr_len = sizeof(struct sockaddr_un); - - if (connect(socket_fd, (struct sockaddr *) &addr, sockaddr_len) == 0) - return socket_fd; - - /* if connect failed */ - close(socket_fd); - return -1; -} - - - /* * socket listening thread for primary process */ @@ -276,7 +115,7 @@ vfio_mp_sync_thread(void __rte_unused * arg) socklen_t sockaddr_len = sizeof(addr); /* this is a blocking call */ - conn_sock = accept(mp_socket_fd, (struct sockaddr *) &addr, + conn_sock = accept(mp_vfio_socket_fd, (struct sockaddr *) &addr, &sockaddr_len); /* just restart on error */ @@ -292,20 +131,20 @@ vfio_mp_sync_thread(void __rte_unused * arg) RTE_LOG(WARNING, EAL, "Cannot set SO_LINGER option " "on listen socket (%s)\n", strerror(errno)); - ret = vfio_mp_sync_receive_request(conn_sock); + ret = eal_mp_sync_receive_request(conn_sock); switch (ret) { case SOCKET_REQ_CONTAINER: fd = vfio_get_container_fd(); if (fd < 0) - vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); + eal_mp_sync_send_request(conn_sock, SOCKET_ERR); else - vfio_mp_sync_send_fd(conn_sock, fd); + eal_mp_sync_send_fd(conn_sock, fd); close(fd); break; case SOCKET_REQ_GROUP: /* wait for group number */ - vfio_data = vfio_mp_sync_receive_request(conn_sock); + vfio_data = eal_mp_sync_receive_request(conn_sock); if (vfio_data < 0) { close(conn_sock); continue; @@ -314,19 +153,19 @@ vfio_mp_sync_thread(void __rte_unused * arg) fd = vfio_get_group_fd(vfio_data); if (fd < 0) - vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); + eal_mp_sync_send_request(conn_sock, SOCKET_ERR); /* if VFIO group exists but isn't bound to VFIO driver */ else if (fd == 0) - vfio_mp_sync_send_request(conn_sock, SOCKET_NO_FD); + eal_mp_sync_send_request(conn_sock, SOCKET_NO_FD); /* if group exists and is bound to VFIO driver */ else { - vfio_mp_sync_send_request(conn_sock, SOCKET_OK); - vfio_mp_sync_send_fd(conn_sock, fd); + eal_mp_sync_send_request(conn_sock, SOCKET_OK); + eal_mp_sync_send_fd(conn_sock, fd); } break; case SOCKET_CLR_GROUP: /* wait for group fd */ - vfio_data = vfio_mp_sync_receive_request(conn_sock); + vfio_data = eal_mp_sync_receive_request(conn_sock); if (vfio_data < 0) { close(conn_sock); continue; @@ -335,12 +174,12 @@ vfio_mp_sync_thread(void __rte_unused * arg) ret = clear_group(vfio_data); if (ret < 0) - vfio_mp_sync_send_request(conn_sock, SOCKET_NO_FD); + eal_mp_sync_send_request(conn_sock, SOCKET_NO_FD); else - vfio_mp_sync_send_request(conn_sock, SOCKET_OK); + eal_mp_sync_send_request(conn_sock, SOCKET_OK); break; default: - vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); + eal_mp_sync_send_request(conn_sock, SOCKET_ERR); break; } close(conn_sock); @@ -350,42 +189,32 @@ vfio_mp_sync_thread(void __rte_unused * arg) static int vfio_mp_sync_socket_setup(void) { - int ret, socket_fd; - struct sockaddr_un addr; - socklen_t sockaddr_len; + int socket_fd; + char path[PATH_MAX]; + + get_socket_path(path, sizeof(path)); - /* set up a socket */ - socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0); + socket_fd = eal_mp_sync_socket_setup(path); if (socket_fd < 0) { RTE_LOG(ERR, EAL, "Failed to create socket!\n"); return -1; } - get_socket_path(addr.sun_path, sizeof(addr.sun_path)); - addr.sun_family = AF_UNIX; - - sockaddr_len = sizeof(struct sockaddr_un); - - unlink(addr.sun_path); + /* save the socket in local configuration */ + mp_vfio_socket_fd = socket_fd; - ret = bind(socket_fd, (struct sockaddr *) &addr, sockaddr_len); - if (ret) { - RTE_LOG(ERR, EAL, "Failed to bind socket: %s!\n", strerror(errno)); - close(socket_fd); - return -1; - } + return 0; +} - ret = listen(socket_fd, 50); - if (ret) { - RTE_LOG(ERR, EAL, "Failed to listen: %s!\n", strerror(errno)); - close(socket_fd); - return -1; - } +/* connect socket_fd in secondary process to the primary process's socket */ +int +vfio_mp_sync_connect_to_primary(void) +{ + char path[PATH_MAX]; - /* save the socket in local configuration */ - mp_socket_fd = socket_fd; + get_socket_path(path, sizeof(path)); - return 0; + return eal_mp_sync_connect_to_primary(path); } /* @@ -402,18 +231,18 @@ vfio_mp_sync_setup(void) return -1; } - ret = pthread_create(&socket_thread, NULL, + ret = pthread_create(&vfio_socket_thread, NULL, vfio_mp_sync_thread, NULL); if (ret) { RTE_LOG(ERR, EAL, "Failed to create thread for communication with secondary processes!\n"); - close(mp_socket_fd); + close(mp_vfio_socket_fd); return -1; } /* Set thread_name for aid in debugging. */ snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "vfio-sync"); - ret = rte_thread_setname(socket_thread, thread_name); + ret = rte_thread_setname(vfio_socket_thread, thread_name); if (ret) RTE_LOG(DEBUG, EAL, "Failed to set thread name for secondary processes!\n");