From patchwork Wed Dec 10 21:37:55 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Huawei Xie X-Patchwork-Id: 1933 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 5B8D2803E; Wed, 10 Dec 2014 22:38:57 +0100 (CET) Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by dpdk.org (Postfix) with ESMTP id C34878053 for ; Wed, 10 Dec 2014 22:38:54 +0100 (CET) Received: from orsmga001.jf.intel.com ([10.7.209.18]) by orsmga103.jf.intel.com with ESMTP; 10 Dec 2014 13:36:55 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.07,553,1413270000"; d="scan'208";a="621818774" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by orsmga001.jf.intel.com with ESMTP; 10 Dec 2014 13:38:52 -0800 Received: from shecgisg003.sh.intel.com (shecgisg003.sh.intel.com [10.239.29.90]) by shvmail01.sh.intel.com with ESMTP id sBALcpTA000908; Thu, 11 Dec 2014 05:38:51 +0800 Received: from shecgisg003.sh.intel.com (localhost [127.0.0.1]) by shecgisg003.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id sBALcmon014024; Thu, 11 Dec 2014 05:38:50 +0800 Received: (from hxie5@localhost) by shecgisg003.sh.intel.com (8.13.6/8.13.6/Submit) id sBALcm5v014020; Thu, 11 Dec 2014 05:38:48 +0800 From: Huawei Xie To: dev@dpdk.org Date: Thu, 11 Dec 2014 05:37:55 +0800 Message-Id: <1418247477-13920-11-git-send-email-huawei.xie@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1418247477-13920-1-git-send-email-huawei.xie@intel.com> References: <1418247477-13920-1-git-send-email-huawei.xie@intel.com> Cc: haifeng.lin@intel.com Subject: [dpdk-dev] [PATCH RFC v2 10/12] lib/librte_vhost: vhost-user memory region map X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" deals with vhost user memory map/unmap alignment Signed-off-by: Huawei Xie --- lib/librte_vhost/rte_virtio_net.h | 2 + lib/librte_vhost/vhost-net.h | 2 - lib/librte_vhost/vhost_user/vhost-net-user.h | 3 +- lib/librte_vhost/vhost_user/virtio-net-user.c | 105 ++++++++++++++++++++++++-- 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h index 00b1328..77db80b 100644 --- a/lib/librte_vhost/rte_virtio_net.h +++ b/lib/librte_vhost/rte_virtio_net.h @@ -48,6 +48,8 @@ #include #include +#define VHOST_MEMORY_MAX_NREGIONS 8 + /* Used to indicate that the device is running on a data core */ #define VIRTIO_DEV_RUNNING 1 diff --git a/lib/librte_vhost/vhost-net.h b/lib/librte_vhost/vhost-net.h index f9ec40b..ec2584f 100644 --- a/lib/librte_vhost/vhost-net.h +++ b/lib/librte_vhost/vhost-net.h @@ -43,8 +43,6 @@ #include "rte_virtio_net.h" -#define VHOST_MEMORY_MAX_NREGIONS 8 - extern struct vhost_net_device_ops const *ops; /* Macros for printing using RTE_LOG */ diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.h b/lib/librte_vhost/vhost_user/vhost-net-user.h index c138844..f4c9d01 100644 --- a/lib/librte_vhost/vhost_user/vhost-net-user.h +++ b/lib/librte_vhost/vhost_user/vhost-net-user.h @@ -37,6 +37,7 @@ #include #include +#include "rte_virtio_net.h" #include "fd_man.h" struct vhost_server { @@ -47,8 +48,6 @@ struct vhost_server { /* refer to hw/virtio/vhost-user.c */ -#define VHOST_MEMORY_MAX_NREGIONS 8 - typedef enum VhostUserRequest { VHOST_USER_NONE = 0, VHOST_USER_GET_FEATURES = 1, diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c b/lib/librte_vhost/vhost_user/virtio-net-user.c index ad59fcc..3aecb17 100644 --- a/lib/librte_vhost/vhost_user/virtio-net-user.c +++ b/lib/librte_vhost/vhost_user/virtio-net-user.c @@ -36,7 +36,11 @@ #include #include #include +#include +#include +#include +#include #include #include "virtio-net.h" @@ -44,13 +48,56 @@ #include "vhost-net-user.h" #include "vhost-net.h" +struct orig_region_map { + int fd; + uint64_t mapped_address; + uint64_t mapped_size; + uint64_t blksz; +}; + +#define orig_region(ptr, nregions) (struct orig_region_map *)RTE_PTR_ADD(ptr, sizeof(struct virtio_memory) + sizeof(struct virtio_memory_regions) * (nregions)) + +static uint64_t +get_blk_size(int fd) +{ + struct stat stat; + fstat(fd, &stat); + return (uint64_t)stat.st_blksize; +} + int user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg) { - unsigned int idx; struct VhostUserMemory memory = pmsg->payload.memory; struct virtio_memory_regions regions[VHOST_MEMORY_MAX_NREGIONS]; - uint64_t mapped_address, base_address = 0; + uint64_t mapped_address, mapped_size, base_address = 0; + struct virtio_net *dev; + unsigned int idx = 0; + struct orig_region_map tmp[VHOST_MEMORY_MAX_NREGIONS] = + { [0 ... VHOST_MEMORY_MAX_NREGIONS - 1] = { 0 } }; + struct orig_region_map *region; + uint64_t alignment; + int ret; + + /* unmap old memory regions one by one*/ + dev = get_device(ctx); + if (dev->mem) { + region = orig_region(dev->mem, dev->mem->nregions); + for (idx = 0; idx < dev->mem->nregions; idx++) { + if (region[idx].mapped_address) { + alignment = region[idx].blksz; + printf("Freeing %p\n", + (void *)(uintptr_t)region[idx].mapped_address); + ret = munmap((void *)RTE_ALIGN_FLOOR(region[idx].mapped_address, alignment), + RTE_ALIGN_CEIL(region[idx].mapped_size, alignment)); + printf("munmap ret= %d\n", ret); + printf("close file %d\n", region[idx].fd); + close(region[idx].fd); + } + } + free(dev->mem); + dev->mem = NULL; + } for (idx = 0; idx < memory.nregions; idx++) { if (memory.regions[idx].guest_phys_addr == 0) @@ -73,22 +120,30 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg) memory.regions[idx].userspace_addr; /* This is ugly */ + mapped_size = regions[idx].memory_size + + memory.regions[idx].mmap_offset; mapped_address = (uint64_t)(uintptr_t)mmap(NULL, - regions[idx].memory_size + - memory.regions[idx].mmap_offset, + mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, pmsg->fds[idx], 0); + RTE_LOG(INFO, VHOST_CONFIG, - "mapped region %d to %p\n", - idx, (void *)mapped_address); + "mapped region %d fd:%d to %p sz:0x%"PRIx64" off:0x%"PRIx64"\n", + idx, pmsg->fds[idx], (void *)mapped_address, + mapped_size, memory.regions[idx].mmap_offset); if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) { RTE_LOG(ERR, VHOST_CONFIG, "mmap qemu guest failed.\n"); - return -1; + goto err; } + tmp[idx].mapped_address = mapped_address; + tmp[idx].mapped_size = mapped_size; + tmp[idx].blksz = get_blk_size(pmsg->fds[idx]); + tmp[idx].fd = pmsg->fds[idx]; + mapped_address += memory.regions[idx].mmap_offset; regions[idx].address_offset = mapped_address - @@ -100,10 +155,44 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg) (void *)(uintptr_t)regions[idx].userspace_address, regions[idx].memory_size); } + ops->set_mem_table(ctx, regions, memory.nregions); + + if (dev->mem) { + void *tmp_mem; + tmp_mem = realloc(dev->mem, + sizeof(struct virtio_memory) + + sizeof(struct virtio_memory_regions) * memory.nregions + + sizeof(struct orig_region_map) * memory.nregions); + if (tmp_mem == NULL) { + goto err_realloc; + } + dev->mem = tmp_mem; + region = orig_region(dev->mem, memory.nregions); + for (idx = 0; idx < memory.nregions; idx++) { + region[idx].mapped_address = tmp[idx].mapped_address; + region[idx].mapped_size = tmp[idx].mapped_size; + region[idx].blksz = tmp[idx].blksz; + region[idx].fd = tmp[idx].fd; + } + } else + goto err_set_mem_table; + return 0; -} +err_realloc: + free(dev->mem); +err_set_mem_table: +err: + while (idx--) { + alignment = tmp[idx].blksz; + munmap((void *)RTE_ALIGN_FLOOR(tmp[idx].mapped_address, alignment), + RTE_ALIGN_CEIL(tmp[idx].mapped_size, alignment)); + close(tmp[idx].fd); + } + dev->mem = NULL; + return -1; +} static int virtio_is_ready(struct virtio_net *dev)