[v3,06/15] example/vdpa:add vdpa blk support in example

  Add virtio blk device support to vdpa example.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 examples/vdpa/Makefile           |   2 +-
 examples/vdpa/main.c             |   8 ++
 examples/vdpa/meson.build        |   1 +
 examples/vdpa/vdpa_blk_compact.c | 150 +++++++++++++++++++++++++++++++
 examples/vdpa/vdpa_blk_compact.h | 117 ++++++++++++++++++++++++
 examples/vdpa/vhost_user.h       | 189 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 466 insertions(+), 1 deletion(-)
 create mode 100644 examples/vdpa/vdpa_blk_compact.c
 create mode 100644 examples/vdpa/vdpa_blk_compact.h
 create mode 100644 examples/vdpa/vhost_user.h
  

On 1/29/22 04:03, Andy Pei wrote:
> Add virtio blk device support to vdpa example.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   examples/vdpa/Makefile           |   2 +-
>   examples/vdpa/main.c             |   8 ++
>   examples/vdpa/meson.build        |   1 +
>   examples/vdpa/vdpa_blk_compact.c | 150 +++++++++++++++++++++++++++++++
>   examples/vdpa/vdpa_blk_compact.h | 117 ++++++++++++++++++++++++
>   examples/vdpa/vhost_user.h       | 189 +++++++++++++++++++++++++++++++++++++++
>   6 files changed, 466 insertions(+), 1 deletion(-)
>   create mode 100644 examples/vdpa/vdpa_blk_compact.c
>   create mode 100644 examples/vdpa/vdpa_blk_compact.h
>   create mode 100644 examples/vdpa/vhost_user.h
> 
> diff --git a/examples/vdpa/Makefile b/examples/vdpa/Makefile
> index d974db4..9d0479b 100644
> --- a/examples/vdpa/Makefile
> +++ b/examples/vdpa/Makefile
> @@ -5,7 +5,7 @@
>   APP = vdpa
>   
>   # all source are stored in SRCS-y
> -SRCS-y := main.c
> +SRCS-y := main.c vdpa_blk_compact.c
>   CFLAGS += -DALLOW_EXPERIMENTAL_API
>   
>   PKGCONF ?= pkg-config
> diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c
> index 5ab0765..924ad7b 100644
> --- a/examples/vdpa/main.c
> +++ b/examples/vdpa/main.c
> @@ -20,6 +20,7 @@
>   #include <cmdline_parse_string.h>
>   #include <cmdline_parse_num.h>
>   #include <cmdline.h>
> +#include "vdpa_blk_compact.h"
>   
>   #define MAX_PATH_LEN 128
>   #define MAX_VDPA_SAMPLE_PORTS 1024
> @@ -156,6 +157,7 @@ struct vdpa_port {
>   static const struct rte_vhost_device_ops vdpa_sample_devops = {
>   	.new_device = new_device,
>   	.destroy_device = destroy_device,
> +	.new_connection = rte_vhost_blk_session_install_rte_compat_hooks,
>   };
>   
>   static int
> @@ -192,6 +194,12 @@ struct vdpa_port {
>   			"attach vdpa device failed: %s\n",
>   			socket_path);
>   
> +	if (vdpa_blk_device_set_features_and_protocol(socket_path, vport->dev)
> +		< 0)
> +		rte_exit(EXIT_FAILURE,
> +			"set vhost blk driver features and protocol features failed: %s\n",
> +			socket_path);
> +

That does not look right, blk devices specitic functions shuold be
called only for block devices.

>   	if (rte_vhost_driver_start(socket_path) < 0)
>   		rte_exit(EXIT_FAILURE,
>   			"start vhost driver failed: %s\n",
> diff --git a/examples/vdpa/meson.build b/examples/vdpa/meson.build
> index bd08605..f0d111c 100644
> --- a/examples/vdpa/meson.build
> +++ b/examples/vdpa/meson.build
> @@ -15,4 +15,5 @@ deps += 'vhost'
>   allow_experimental_apis = true
>   sources = files(
>           'main.c',
> +	'vdpa_blk_compact.c',
>   )
> diff --git a/examples/vdpa/vdpa_blk_compact.c b/examples/vdpa/vdpa_blk_compact.c
> new file mode 100644
> index 0000000..0c4d3ee
> --- /dev/null
> +++ b/examples/vdpa/vdpa_blk_compact.c
> @@ -0,0 +1,150 @@
> +/*    INTEL CONFIDENTIAL
> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +/* @file
> + *
> + * Block device specific vhost lib
> + */
> +
> +#include <stdbool.h>
> +
> +#include <rte_malloc.h>
> +#include <vdpa_driver.h>

That's wrong, the application is not supposed to include the driver
APIs.

> +#include <rte_vhost.h>
> +#include "vdpa_blk_compact.h"
> +#include "vhost_user.h"
> +
> +#define VHOST_USER_GET_CONFIG	24
> +#define VHOST_USER_SET_CONFIG	25
> +
> +#ifndef VHOST_USER_PROTOCOL_F_CONFIG
> +#define VHOST_USER_PROTOCOL_F_CONFIG   9
> +#endif
> +
> +/*
> + * Function to handle vhost user blk message
> + */
> +static enum rte_vhost_msg_result
> +rte_vhost_blk_extern_vhost_pre_msg_handler(int vid, void *_msg)
> +{
> +	struct VhostUserMsg *msg = _msg;
> +	struct rte_vdpa_device *vdev = NULL;
> +
> +	vdev = rte_vhost_get_vdpa_device(vid);
> +	if (vdev == NULL)
> +		return RTE_VHOST_MSG_RESULT_ERR;
> +
> +	fprintf(stderr, "msg is %d\n", msg->request.master);
> +	switch (msg->request.master) {
> +	case VHOST_USER_GET_CONFIG: {
> +		int rc = 0;
> +
> +		fprintf(stdout, "read message VHOST_USER_GET_CONFIG\n");
> +
> +		if (vdev->ops->get_config) {
> +			fprintf(stdout, "get_config() function is valid!\n");
> +			rc = vdev->ops->get_config(vid,
> +						   msg->payload.cfg.region,
> +						   msg->payload.cfg.size);
> +			if (rc != 0) {
> +				msg->size = 0;
> +				fprintf(stdout, "get_config() return error!\n");
> +			}
> +		} else {
> +			fprintf(stdout, "get_config() function is invalid!\n");
> +		}
> +
> +		return RTE_VHOST_MSG_RESULT_REPLY;
> +	}
> +	case VHOST_USER_SET_CONFIG: {
> +		int rc = 0;
> +
> +		fprintf(stdout,
> +			"read message VHOST_USER_SET_CONFIG\n");
> +
> +		if (vdev->ops->set_config) {
> +			rc = vdev->ops->set_config(vid,
> +				msg->payload.cfg.region,
> +				msg->payload.cfg.offset,
> +				msg->payload.cfg.size,
> +				msg->payload.cfg.flags);
> +		}
> +
> +		return rc == 0 ? RTE_VHOST_MSG_RESULT_OK : RTE_VHOST_MSG_RESULT_ERR;
> +	}
> +	default:
> +		break;
> +	}
> +
> +	return RTE_VHOST_MSG_RESULT_NOT_HANDLED;
> +}

I think above message handling should be done in the Vhost library
directly. VHOST_USER_SET_CONFIG and VHOST_USER_GET_CONFIG are not
specific to blk backends, these are generic messages.

> +
> +struct rte_vhost_user_extern_ops g_blk_extern_vhost_ops = {
> +	.pre_msg_handle = rte_vhost_blk_extern_vhost_pre_msg_handler,
> +	.post_msg_handle = NULL,
> +};
> +
> +int
> +rte_vhost_blk_session_install_rte_compat_hooks(int vid)
> +{
> +	int rc;
> +
> +	rc = rte_vhost_extern_callback_register(vid,
> +						&g_blk_extern_vhost_ops,
> +						NULL);
> +	if (rc != 0) {
> +		fprintf(stderr, "%s() failed for vid = %d\n",  __func__, vid);
> +		return -1;
> +	}
> +	fprintf(stdout, "register extern vhost ops on vid = %d\n", vid);
> +	return 0;
> +}
> +
> +
> +int
> +vdpa_blk_device_set_features_and_protocol(const char *path,
> +	struct rte_vdpa_device *vdev)
> +{
> +	uint64_t protocol_features = 0;
> +
> +	if (!vdev) {
> +		fprintf(stdout, "vdev is NULL.\n");
> +		return -EINVAL;
> +	}
> +
> +	/* vdpa net does not have the get_config */
> +	if (!vdev->ops->get_config)
> +		return 0;

That's not good, as I said earlier, the drivers callback should not be
visible to the application. Maybe the VDPA API should be extended to
return the device type, I'm not sure, but accessing the drivers ops is
prohibited.

> +	rte_vhost_driver_set_features(path, SPDK_VHOST_BLK_FEATURES_BASE);
> +	rte_vhost_driver_disable_features(path,
> +		SPDK_VHOST_BLK_DISABLED_FEATURES);
> +
> +	rte_vhost_driver_get_protocol_features(path, &protocol_features);
> +	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
> +	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
> +	rte_vhost_driver_set_protocol_features(path, protocol_features);
> +
> +	return 0;
> +}
> diff --git a/examples/vdpa/vdpa_blk_compact.h b/examples/vdpa/vdpa_blk_compact.h
> new file mode 100644
> index 0000000..420d48e
> --- /dev/null
> +++ b/examples/vdpa/vdpa_blk_compact.h
> @@ -0,0 +1,117 @@
> +/*    INTEL CONFIDENTIAL

I hope it was not supposed to be confidential :)

> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +#ifndef _VDPA_BLK_COMPACT_H_
> +#define _VDPA_BLK_COMPACT_H_
> +
> +/**
> + * @file
> + *
> + * Device specific vhost lib
> + */
> +/vdpa
> +#include <stdbool.h>
> +
> +#include <rte_pci.h>
> +#include <rte_vhost.h>
> +
> +/* Feature bits */
> +#define VIRTIO_BLK_F_SIZE_MAX     1    /* Indicates maximum segment size */
> +#define VIRTIO_BLK_F_SEG_MAX      2    /* Indicates maximum # of segments */
> +#define VIRTIO_BLK_F_GEOMETRY     4    /* Legacy geometry available  */
> +#define VIRTIO_BLK_F_RO           5    /* Disk is read-only */
> +#define VIRTIO_BLK_F_BLK_SIZE     6    /* Block size of disk is available */
> +#define VIRTIO_BLK_F_TOPOLOGY     10   /* Topology information is available */
> +#define VIRTIO_BLK_F_MQ           12   /* support more than one vq */
> +#define VIRTIO_BLK_F_DISCARD      13   /* DISCARD is supported */
> +#define VIRTIO_BLK_F_WRITE_ZEROES 14   /* WRITE ZEROES is supported */
> +
> +/* Legacy feature bits */
> +#ifndef VIRTIO_BLK_NO_LEGACY
> +#define VIRTIO_BLK_F_BARRIER      0    /* Does host support barriers? */
> +#define VIRTIO_BLK_F_SCSI         7    /* Supports scsi command passthru */
> +#define VIRTIO_BLK_F_FLUSH        9    /* Flush command supported */
> +#define VIRTIO_BLK_F_CONFIG_WCE   11   /* Writeback mode available in config */
> +
> +/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
> +#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
> +#endif /* !VIRTIO_BLK_NO_LEGACY */
> +
> +#ifndef VHOST_USER_F_PROTOCOL_FEATURES
> +#define VHOST_USER_F_PROTOCOL_FEATURES 30
> +#endif
> +
> +#define SPDK_VHOST_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \

Why these references to SPDK?

> +	(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
> +	(1ULL << VIRTIO_F_VERSION_1) | \
> +	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \
> +	(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
> +	(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
> +
> +#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \
> +	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY))
> +
> +#define SPDK_VHOST_BLK_FEATURES_BASE (SPDK_VHOST_FEATURES | \
> +	(1ULL << VIRTIO_BLK_F_SIZE_MAX) | (1ULL << VIRTIO_BLK_F_SEG_MAX) | \
> +	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \
> +	(1ULL << VIRTIO_BLK_F_TOPOLOGY) | (1ULL << VIRTIO_BLK_F_BARRIER)  | \
> +	(1ULL << VIRTIO_BLK_F_SCSI)     | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
> +	(1ULL << VIRTIO_BLK_F_MQ))
> +
> +/* Not supported features */
> +#define SPDK_VHOST_BLK_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \
> +	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
> +	(1ULL << VIRTIO_BLK_F_BARRIER)  | (1ULL << VIRTIO_BLK_F_SCSI))
> +
> +/* Vhost-blk support protocol features */
> +#define SPDK_VHOST_BLK_PROTOCOL_FEATURES \
> +	((1ULL << VHOST_USER_PROTOCOL_F_CONFIG) | \
> +	(1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * This function will set vhost user block
> + *
> + * @param path
> + *  socket path
> + */
> +int
> +vdpa_blk_device_set_features_and_protocol(const char *path,
> +	struct rte_vdpa_device *vdev);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Install external hook to handle vhost user block message
> + *
> + * @param vid
> + *  vhost device id
> + */
> +int
> +rte_vhost_blk_session_install_rte_compat_hooks(int vid);
> +
> +#endif /* _VDPA_BLK_COMPACT_H_ */
> diff --git a/examples/vdpa/vhost_user.h b/examples/vdpa/vhost_user.h
> new file mode 100644
> index 0000000..8b747d0
> --- /dev/null
> +++ b/examples/vdpa/vhost_user.h
> @@ -0,0 +1,189 @@
> +/*    INTEL CONFIDENTIAL
> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +#ifndef _VHOST_NET_USER_H
> +#define _VHOST_NET_USER_H
> +
> +#include <stdint.h>
> +#include <linux/vhost.h>
> +
> +#include "rte_vhost.h"
> +
> +/* refer to hw/virtio/vhost-user.c */
> +
> +#define VHOST_MEMORY_MAX_NREGIONS 8
> +
> +#ifndef VHOST_USER_MAX_CONFIG_SIZE
> +#define VHOST_USER_MAX_CONFIG_SIZE		256
> +#endif
> +
> +#define VHOST_USER_PROTOCOL_FEATURES	((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\
> +			(1ULL << VHOST_USER_PROTOCOL_F_RARP) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_CRYPTO_SESSION) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT))
> +
> +typedef enum VhostUserRequest {
> +	VHOST_USER_NONE = 0,
> +	VHOST_USER_GET_FEATURES = 1,
> +	VHOST_USER_SET_FEATURES = 2,
> +	VHOST_USER_SET_OWNER = 3,
> +	VHOST_USER_RESET_OWNER = 4,
> +	VHOST_USER_SET_MEM_TABLE = 5,
> +	VHOST_USER_SET_LOG_BASE = 6,
> +	VHOST_USER_SET_LOG_FD = 7,
> +	VHOST_USER_SET_VRING_NUM = 8,
> +	VHOST_USER_SET_VRING_ADDR = 9,
> +	VHOST_USER_SET_VRING_BASE = 10,
> +	VHOST_USER_GET_VRING_BASE = 11,
> +	VHOST_USER_SET_VRING_KICK = 12,
> +	VHOST_USER_SET_VRING_CALL = 13,
> +	VHOST_USER_SET_VRING_ERR = 14,
> +	VHOST_USER_GET_PROTOCOL_FEATURES = 15,
> +	VHOST_USER_SET_PROTOCOL_FEATURES = 16,
> +	VHOST_USER_GET_QUEUE_NUM = 17,
> +	VHOST_USER_SET_VRING_ENABLE = 18,
> +	VHOST_USER_SEND_RARP = 19,
> +	VHOST_USER_NET_SET_MTU = 20,
> +	VHOST_USER_SET_SLAVE_REQ_FD = 21,
> +	VHOST_USER_IOTLB_MSG = 22,
> +	VHOST_USER_CRYPTO_CREATE_SESS = 26,
> +	VHOST_USER_CRYPTO_CLOSE_SESS = 27,
> +	VHOST_USER_POSTCOPY_ADVISE = 28,
> +	VHOST_USER_POSTCOPY_LISTEN = 29,
> +	VHOST_USER_POSTCOPY_END = 30,
> +	VHOST_USER_GET_INFLIGHT_FD = 31,
> +	VHOST_USER_SET_INFLIGHT_FD = 32,
> +	VHOST_USER_MAX = 33
> +} VhostUserRequest;
> +
> +typedef enum VhostUserSlaveRequest {
> +	VHOST_USER_SLAVE_NONE = 0,
> +	VHOST_USER_SLAVE_IOTLB_MSG = 1,
> +	VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
> +	VHOST_USER_SLAVE_MAX
> +} VhostUserSlaveRequest;
> +
> +typedef struct VhostUserMemoryRegion {
> +	uint64_t guest_phys_addr;
> +	uint64_t memory_size;
> +	uint64_t userspace_addr;
> +	uint64_t mmap_offset;
> +} VhostUserMemoryRegion;
> +
> +typedef struct VhostUserMemory {
> +	uint32_t nregions;
> +	uint32_t padding;
> +	VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
> +} VhostUserMemory;
> +
> +typedef struct VhostUserLog {
> +	uint64_t mmap_size;
> +	uint64_t mmap_offset;
> +} VhostUserLog;
> +
> +/* Comply with Cryptodev-Linux */
> +#define VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH	512
> +#define VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH	64
> +
> +/* Same structure as vhost-user backend session info */
> +typedef struct VhostUserCryptoSessionParam {
> +	int64_t session_id;
> +	uint32_t op_code;
> +	uint32_t cipher_algo;
> +	uint32_t cipher_key_len;
> +	uint32_t hash_algo;
> +	uint32_t digest_len;
> +	uint32_t auth_key_len;
> +	uint32_t aad_len;
> +	uint8_t op_type;
> +	uint8_t dir;
> +	uint8_t hash_mode;
> +	uint8_t chaining_dir;
> +	uint8_t *ciphe_key;
> +	uint8_t *auth_key;
> +	uint8_t cipher_key_buf[VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH];
> +	uint8_t auth_key_buf[VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH];
> +} VhostUserCryptoSessionParam;
> +
> +typedef struct VhostUserVringArea {
> +	uint64_t u64;
> +	uint64_t size;
> +	uint64_t offset;
> +} VhostUserVringArea;
> +
> +typedef struct VhostUserInflight {
> +	uint64_t mmap_size;
> +	uint64_t mmap_offset;
> +	uint16_t num_queues;
> +	uint16_t queue_size;
> +} VhostUserInflight;
> +
> +/** Get/set config msg payload */
> +struct vhost_user_config {
> +	uint32_t offset;
> +	uint32_t size;
> +	uint32_t flags;
> +	uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
> +};
> +
> +typedef struct VhostUserMsg {
> +	union {
> +		uint32_t master; /* a VhostUserRequest value */
> +		uint32_t slave;  /* a VhostUserSlaveRequest value*/
> +	} request;
> +
> +#define VHOST_USER_VERSION_MASK     0x3
> +#define VHOST_USER_REPLY_MASK       (0x1 << 2)
> +#define VHOST_USER_NEED_REPLY		(0x1 << 3)
> +	uint32_t flags;
> +	uint32_t size; /* the following payload size */
> +	union {
> +#define VHOST_USER_VRING_IDX_MASK   0xff
> +#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
> +		uint64_t u64;
> +		struct vhost_vring_state state;
> +		struct vhost_vring_addr addr;
> +		VhostUserMemory memory;
> +		VhostUserLog    log;
> +		struct vhost_iotlb_msg iotlb;
> +		VhostUserCryptoSessionParam crypto_session;
> +		VhostUserVringArea area;
> +		VhostUserInflight inflight;
> +		struct vhost_user_config cfg;
> +	} payload;
> +	int fds[VHOST_MEMORY_MAX_NREGIONS];
> +	int fd_num;
> +} __attribute((packed)) VhostUserMsg;
> +
> +#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
> +
> +/* The version of the protocol we support */
> +#define VHOST_USER_VERSION    0x1
> +#endif
  

Hi Maxime,

I seems a lot of problem with the example.
I think I will re-work the example according to your comments.
Thanks for your comments.

-----Original Message-----
From: Maxime Coquelin <maxime.coquelin@redhat.com> 
Sent: Tuesday, March 22, 2022 7:30 PM
To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
Cc: Xia, Chenbo <chenbo.xia@intel.com>; Cao, Gang <gang.cao@intel.com>; Liu, Changpeng <changpeng.liu@intel.com>
Subject: Re: [PATCH v3 06/15] example/vdpa:add vdpa blk support in example



On 1/29/22 04:03, Andy Pei wrote:
> Add virtio blk device support to vdpa example.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   examples/vdpa/Makefile           |   2 +-
>   examples/vdpa/main.c             |   8 ++
>   examples/vdpa/meson.build        |   1 +
>   examples/vdpa/vdpa_blk_compact.c | 150 +++++++++++++++++++++++++++++++
>   examples/vdpa/vdpa_blk_compact.h | 117 ++++++++++++++++++++++++
>   examples/vdpa/vhost_user.h       | 189 +++++++++++++++++++++++++++++++++++++++
>   6 files changed, 466 insertions(+), 1 deletion(-)
>   create mode 100644 examples/vdpa/vdpa_blk_compact.c
>   create mode 100644 examples/vdpa/vdpa_blk_compact.h
>   create mode 100644 examples/vdpa/vhost_user.h
> 
> diff --git a/examples/vdpa/Makefile b/examples/vdpa/Makefile index 
> d974db4..9d0479b 100644
> --- a/examples/vdpa/Makefile
> +++ b/examples/vdpa/Makefile
> @@ -5,7 +5,7 @@
>   APP = vdpa
>   
>   # all source are stored in SRCS-y
> -SRCS-y := main.c
> +SRCS-y := main.c vdpa_blk_compact.c
>   CFLAGS += -DALLOW_EXPERIMENTAL_API
>   
>   PKGCONF ?= pkg-config
> diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c index 
> 5ab0765..924ad7b 100644
> --- a/examples/vdpa/main.c
> +++ b/examples/vdpa/main.c
> @@ -20,6 +20,7 @@
>   #include <cmdline_parse_string.h>
>   #include <cmdline_parse_num.h>
>   #include <cmdline.h>
> +#include "vdpa_blk_compact.h"
>   
>   #define MAX_PATH_LEN 128
>   #define MAX_VDPA_SAMPLE_PORTS 1024
> @@ -156,6 +157,7 @@ struct vdpa_port {
>   static const struct rte_vhost_device_ops vdpa_sample_devops = {
>   	.new_device = new_device,
>   	.destroy_device = destroy_device,
> +	.new_connection = rte_vhost_blk_session_install_rte_compat_hooks,
>   };
>   
>   static int
> @@ -192,6 +194,12 @@ struct vdpa_port {
>   			"attach vdpa device failed: %s\n",
>   			socket_path);
>   
> +	if (vdpa_blk_device_set_features_and_protocol(socket_path, vport->dev)
> +		< 0)
> +		rte_exit(EXIT_FAILURE,
> +			"set vhost blk driver features and protocol features failed: %s\n",
> +			socket_path);
> +

That does not look right, blk devices specitic functions shuold be called only for block devices.

>   	if (rte_vhost_driver_start(socket_path) < 0)
>   		rte_exit(EXIT_FAILURE,
>   			"start vhost driver failed: %s\n", diff --git 
> a/examples/vdpa/meson.build b/examples/vdpa/meson.build index 
> bd08605..f0d111c 100644
> --- a/examples/vdpa/meson.build
> +++ b/examples/vdpa/meson.build
> @@ -15,4 +15,5 @@ deps += 'vhost'
>   allow_experimental_apis = true
>   sources = files(
>           'main.c',
> +	'vdpa_blk_compact.c',
>   )
> diff --git a/examples/vdpa/vdpa_blk_compact.c 
> b/examples/vdpa/vdpa_blk_compact.c
> new file mode 100644
> index 0000000..0c4d3ee
> --- /dev/null
> +++ b/examples/vdpa/vdpa_blk_compact.c
> @@ -0,0 +1,150 @@
> +/*    INTEL CONFIDENTIAL
> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +/* @file
> + *
> + * Block device specific vhost lib
> + */
> +
> +#include <stdbool.h>
> +
> +#include <rte_malloc.h>
> +#include <vdpa_driver.h>

That's wrong, the application is not supposed to include the driver APIs.

> +#include <rte_vhost.h>
> +#include "vdpa_blk_compact.h"
> +#include "vhost_user.h"
> +
> +#define VHOST_USER_GET_CONFIG	24
> +#define VHOST_USER_SET_CONFIG	25
> +
> +#ifndef VHOST_USER_PROTOCOL_F_CONFIG
> +#define VHOST_USER_PROTOCOL_F_CONFIG   9
> +#endif
> +
> +/*
> + * Function to handle vhost user blk message  */ static enum 
> +rte_vhost_msg_result rte_vhost_blk_extern_vhost_pre_msg_handler(int 
> +vid, void *_msg) {
> +	struct VhostUserMsg *msg = _msg;
> +	struct rte_vdpa_device *vdev = NULL;
> +
> +	vdev = rte_vhost_get_vdpa_device(vid);
> +	if (vdev == NULL)
> +		return RTE_VHOST_MSG_RESULT_ERR;
> +
> +	fprintf(stderr, "msg is %d\n", msg->request.master);
> +	switch (msg->request.master) {
> +	case VHOST_USER_GET_CONFIG: {
> +		int rc = 0;
> +
> +		fprintf(stdout, "read message VHOST_USER_GET_CONFIG\n");
> +
> +		if (vdev->ops->get_config) {
> +			fprintf(stdout, "get_config() function is valid!\n");
> +			rc = vdev->ops->get_config(vid,
> +						   msg->payload.cfg.region,
> +						   msg->payload.cfg.size);
> +			if (rc != 0) {
> +				msg->size = 0;
> +				fprintf(stdout, "get_config() return error!\n");
> +			}
> +		} else {
> +			fprintf(stdout, "get_config() function is invalid!\n");
> +		}
> +
> +		return RTE_VHOST_MSG_RESULT_REPLY;
> +	}
> +	case VHOST_USER_SET_CONFIG: {
> +		int rc = 0;
> +
> +		fprintf(stdout,
> +			"read message VHOST_USER_SET_CONFIG\n");
> +
> +		if (vdev->ops->set_config) {
> +			rc = vdev->ops->set_config(vid,
> +				msg->payload.cfg.region,
> +				msg->payload.cfg.offset,
> +				msg->payload.cfg.size,
> +				msg->payload.cfg.flags);
> +		}
> +
> +		return rc == 0 ? RTE_VHOST_MSG_RESULT_OK : RTE_VHOST_MSG_RESULT_ERR;
> +	}
> +	default:
> +		break;
> +	}
> +
> +	return RTE_VHOST_MSG_RESULT_NOT_HANDLED; }

I think above message handling should be done in the Vhost library directly. VHOST_USER_SET_CONFIG and VHOST_USER_GET_CONFIG are not specific to blk backends, these are generic messages.

> +
> +struct rte_vhost_user_extern_ops g_blk_extern_vhost_ops = {
> +	.pre_msg_handle = rte_vhost_blk_extern_vhost_pre_msg_handler,
> +	.post_msg_handle = NULL,
> +};
> +
> +int
> +rte_vhost_blk_session_install_rte_compat_hooks(int vid) {
> +	int rc;
> +
> +	rc = rte_vhost_extern_callback_register(vid,
> +						&g_blk_extern_vhost_ops,
> +						NULL);
> +	if (rc != 0) {
> +		fprintf(stderr, "%s() failed for vid = %d\n",  __func__, vid);
> +		return -1;
> +	}
> +	fprintf(stdout, "register extern vhost ops on vid = %d\n", vid);
> +	return 0;
> +}
> +
> +
> +int
> +vdpa_blk_device_set_features_and_protocol(const char *path,
> +	struct rte_vdpa_device *vdev)
> +{
> +	uint64_t protocol_features = 0;
> +
> +	if (!vdev) {
> +		fprintf(stdout, "vdev is NULL.\n");
> +		return -EINVAL;
> +	}
> +
> +	/* vdpa net does not have the get_config */
> +	if (!vdev->ops->get_config)
> +		return 0;

That's not good, as I said earlier, the drivers callback should not be visible to the application. Maybe the VDPA API should be extended to return the device type, I'm not sure, but accessing the drivers ops is prohibited.

> +	rte_vhost_driver_set_features(path, SPDK_VHOST_BLK_FEATURES_BASE);
> +	rte_vhost_driver_disable_features(path,
> +		SPDK_VHOST_BLK_DISABLED_FEATURES);
> +
> +	rte_vhost_driver_get_protocol_features(path, &protocol_features);
> +	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
> +	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
> +	rte_vhost_driver_set_protocol_features(path, protocol_features);
> +
> +	return 0;
> +}
> diff --git a/examples/vdpa/vdpa_blk_compact.h 
> b/examples/vdpa/vdpa_blk_compact.h
> new file mode 100644
> index 0000000..420d48e
> --- /dev/null
> +++ b/examples/vdpa/vdpa_blk_compact.h
> @@ -0,0 +1,117 @@
> +/*    INTEL CONFIDENTIAL

I hope it was not supposed to be confidential :)

> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +#ifndef _VDPA_BLK_COMPACT_H_
> +#define _VDPA_BLK_COMPACT_H_
> +
> +/**
> + * @file
> + *
> + * Device specific vhost lib
> + */
> +/vdpa
> +#include <stdbool.h>
> +
> +#include <rte_pci.h>
> +#include <rte_vhost.h>
> +
> +/* Feature bits */
> +#define VIRTIO_BLK_F_SIZE_MAX     1    /* Indicates maximum segment size */
> +#define VIRTIO_BLK_F_SEG_MAX      2    /* Indicates maximum # of segments */
> +#define VIRTIO_BLK_F_GEOMETRY     4    /* Legacy geometry available  */
> +#define VIRTIO_BLK_F_RO           5    /* Disk is read-only */
> +#define VIRTIO_BLK_F_BLK_SIZE     6    /* Block size of disk is available */
> +#define VIRTIO_BLK_F_TOPOLOGY     10   /* Topology information is available */
> +#define VIRTIO_BLK_F_MQ           12   /* support more than one vq */
> +#define VIRTIO_BLK_F_DISCARD      13   /* DISCARD is supported */
> +#define VIRTIO_BLK_F_WRITE_ZEROES 14   /* WRITE ZEROES is supported */
> +
> +/* Legacy feature bits */
> +#ifndef VIRTIO_BLK_NO_LEGACY
> +#define VIRTIO_BLK_F_BARRIER      0    /* Does host support barriers? */
> +#define VIRTIO_BLK_F_SCSI         7    /* Supports scsi command passthru */
> +#define VIRTIO_BLK_F_FLUSH        9    /* Flush command supported */
> +#define VIRTIO_BLK_F_CONFIG_WCE   11   /* Writeback mode available in config */
> +
> +/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */ #define 
> +VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH #endif /* !VIRTIO_BLK_NO_LEGACY 
> +*/
> +
> +#ifndef VHOST_USER_F_PROTOCOL_FEATURES #define 
> +VHOST_USER_F_PROTOCOL_FEATURES 30 #endif
> +
> +#define SPDK_VHOST_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \

Why these references to SPDK?

> +	(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
> +	(1ULL << VIRTIO_F_VERSION_1) | \
> +	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \
> +	(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
> +	(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
> +
> +#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \
> +	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY))
> +
> +#define SPDK_VHOST_BLK_FEATURES_BASE (SPDK_VHOST_FEATURES | \
> +	(1ULL << VIRTIO_BLK_F_SIZE_MAX) | (1ULL << VIRTIO_BLK_F_SEG_MAX) | \
> +	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \
> +	(1ULL << VIRTIO_BLK_F_TOPOLOGY) | (1ULL << VIRTIO_BLK_F_BARRIER)  | \
> +	(1ULL << VIRTIO_BLK_F_SCSI)     | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
> +	(1ULL << VIRTIO_BLK_F_MQ))
> +
> +/* Not supported features */
> +#define SPDK_VHOST_BLK_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \
> +	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
> +	(1ULL << VIRTIO_BLK_F_BARRIER)  | (1ULL << VIRTIO_BLK_F_SCSI))
> +
> +/* Vhost-blk support protocol features */ #define 
> +SPDK_VHOST_BLK_PROTOCOL_FEATURES \
> +	((1ULL << VHOST_USER_PROTOCOL_F_CONFIG) | \
> +	(1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * This function will set vhost user block
> + *
> + * @param path
> + *  socket path
> + */
> +int
> +vdpa_blk_device_set_features_and_protocol(const char *path,
> +	struct rte_vdpa_device *vdev);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Install external hook to handle vhost user block message
> + *
> + * @param vid
> + *  vhost device id
> + */
> +int
> +rte_vhost_blk_session_install_rte_compat_hooks(int vid);
> +
> +#endif /* _VDPA_BLK_COMPACT_H_ */
> diff --git a/examples/vdpa/vhost_user.h b/examples/vdpa/vhost_user.h 
> new file mode 100644 index 0000000..8b747d0
> --- /dev/null
> +++ b/examples/vdpa/vhost_user.h
> @@ -0,0 +1,189 @@
> +/*    INTEL CONFIDENTIAL
> + *
> + *    Copyright (c) Intel Corporation.
> + *    All rights reserved.
> + *
> + *    The source code contained or described herein and all documents related
> + *    to the source code ("Material") are owned by Intel Corporation or its
> + *    suppliers or licensors.  Title to the Material remains with Intel
> + *    Corporation or its suppliers and licensors.  The Material contains trade
> + *    secrets and proprietary and confidential information of Intel or its
> + *    suppliers and licensors.  The Material is protected by worldwide
> + *    copyright and trade secret laws and treaty provisions.  No part of the
> + *    Material may be used, copied, reproduced, modified, published, uploaded,
> + *    posted, transmitted, distributed, or disclosed in any way without Intel's
> + *    prior express written permission.
> + *
> + *    No license under any patent, copyright, trade secret or other
> + *    intellectual property right is granted to or conferred upon you by
> + *    disclosure or delivery of the Materials, either expressly, by
> + *    implication, inducement, estoppel or otherwise.  Any license under such
> + *    intellectual property rights must be express and approved by Intel in
> + *    writing.
> + */
> +
> +#ifndef _VHOST_NET_USER_H
> +#define _VHOST_NET_USER_H
> +
> +#include <stdint.h>
> +#include <linux/vhost.h>
> +
> +#include "rte_vhost.h"
> +
> +/* refer to hw/virtio/vhost-user.c */
> +
> +#define VHOST_MEMORY_MAX_NREGIONS 8
> +
> +#ifndef VHOST_USER_MAX_CONFIG_SIZE
> +#define VHOST_USER_MAX_CONFIG_SIZE		256
> +#endif
> +
> +#define VHOST_USER_PROTOCOL_FEATURES	((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\
> +			(1ULL << VHOST_USER_PROTOCOL_F_RARP) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_CRYPTO_SESSION) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
> +			(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT))
> +
> +typedef enum VhostUserRequest {
> +	VHOST_USER_NONE = 0,
> +	VHOST_USER_GET_FEATURES = 1,
> +	VHOST_USER_SET_FEATURES = 2,
> +	VHOST_USER_SET_OWNER = 3,
> +	VHOST_USER_RESET_OWNER = 4,
> +	VHOST_USER_SET_MEM_TABLE = 5,
> +	VHOST_USER_SET_LOG_BASE = 6,
> +	VHOST_USER_SET_LOG_FD = 7,
> +	VHOST_USER_SET_VRING_NUM = 8,
> +	VHOST_USER_SET_VRING_ADDR = 9,
> +	VHOST_USER_SET_VRING_BASE = 10,
> +	VHOST_USER_GET_VRING_BASE = 11,
> +	VHOST_USER_SET_VRING_KICK = 12,
> +	VHOST_USER_SET_VRING_CALL = 13,
> +	VHOST_USER_SET_VRING_ERR = 14,
> +	VHOST_USER_GET_PROTOCOL_FEATURES = 15,
> +	VHOST_USER_SET_PROTOCOL_FEATURES = 16,
> +	VHOST_USER_GET_QUEUE_NUM = 17,
> +	VHOST_USER_SET_VRING_ENABLE = 18,
> +	VHOST_USER_SEND_RARP = 19,
> +	VHOST_USER_NET_SET_MTU = 20,
> +	VHOST_USER_SET_SLAVE_REQ_FD = 21,
> +	VHOST_USER_IOTLB_MSG = 22,
> +	VHOST_USER_CRYPTO_CREATE_SESS = 26,
> +	VHOST_USER_CRYPTO_CLOSE_SESS = 27,
> +	VHOST_USER_POSTCOPY_ADVISE = 28,
> +	VHOST_USER_POSTCOPY_LISTEN = 29,
> +	VHOST_USER_POSTCOPY_END = 30,
> +	VHOST_USER_GET_INFLIGHT_FD = 31,
> +	VHOST_USER_SET_INFLIGHT_FD = 32,
> +	VHOST_USER_MAX = 33
> +} VhostUserRequest;
> +
> +typedef enum VhostUserSlaveRequest {
> +	VHOST_USER_SLAVE_NONE = 0,
> +	VHOST_USER_SLAVE_IOTLB_MSG = 1,
> +	VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
> +	VHOST_USER_SLAVE_MAX
> +} VhostUserSlaveRequest;
> +
> +typedef struct VhostUserMemoryRegion {
> +	uint64_t guest_phys_addr;
> +	uint64_t memory_size;
> +	uint64_t userspace_addr;
> +	uint64_t mmap_offset;
> +} VhostUserMemoryRegion;
> +
> +typedef struct VhostUserMemory {
> +	uint32_t nregions;
> +	uint32_t padding;
> +	VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
> +} VhostUserMemory;
> +
> +typedef struct VhostUserLog {
> +	uint64_t mmap_size;
> +	uint64_t mmap_offset;
> +} VhostUserLog;
> +
> +/* Comply with Cryptodev-Linux */
> +#define VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH	512
> +#define VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH	64
> +
> +/* Same structure as vhost-user backend session info */ typedef 
> +struct VhostUserCryptoSessionParam {
> +	int64_t session_id;
> +	uint32_t op_code;
> +	uint32_t cipher_algo;
> +	uint32_t cipher_key_len;
> +	uint32_t hash_algo;
> +	uint32_t digest_len;
> +	uint32_t auth_key_len;
> +	uint32_t aad_len;
> +	uint8_t op_type;
> +	uint8_t dir;
> +	uint8_t hash_mode;
> +	uint8_t chaining_dir;
> +	uint8_t *ciphe_key;
> +	uint8_t *auth_key;
> +	uint8_t cipher_key_buf[VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH];
> +	uint8_t auth_key_buf[VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH];
> +} VhostUserCryptoSessionParam;
> +
> +typedef struct VhostUserVringArea {
> +	uint64_t u64;
> +	uint64_t size;
> +	uint64_t offset;
> +} VhostUserVringArea;
> +
> +typedef struct VhostUserInflight {
> +	uint64_t mmap_size;
> +	uint64_t mmap_offset;
> +	uint16_t num_queues;
> +	uint16_t queue_size;
> +} VhostUserInflight;
> +
> +/** Get/set config msg payload */
> +struct vhost_user_config {
> +	uint32_t offset;
> +	uint32_t size;
> +	uint32_t flags;
> +	uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
> +};
> +
> +typedef struct VhostUserMsg {
> +	union {
> +		uint32_t master; /* a VhostUserRequest value */
> +		uint32_t slave;  /* a VhostUserSlaveRequest value*/
> +	} request;
> +
> +#define VHOST_USER_VERSION_MASK     0x3
> +#define VHOST_USER_REPLY_MASK       (0x1 << 2)
> +#define VHOST_USER_NEED_REPLY		(0x1 << 3)
> +	uint32_t flags;
> +	uint32_t size; /* the following payload size */
> +	union {
> +#define VHOST_USER_VRING_IDX_MASK   0xff
> +#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
> +		uint64_t u64;
> +		struct vhost_vring_state state;
> +		struct vhost_vring_addr addr;
> +		VhostUserMemory memory;
> +		VhostUserLog    log;
> +		struct vhost_iotlb_msg iotlb;
> +		VhostUserCryptoSessionParam crypto_session;
> +		VhostUserVringArea area;
> +		VhostUserInflight inflight;
> +		struct vhost_user_config cfg;
> +	} payload;
> +	int fds[VHOST_MEMORY_MAX_NREGIONS];
> +	int fd_num;
> +} __attribute((packed)) VhostUserMsg;
> +
> +#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
> +
> +/* The version of the protocol we support */
> +#define VHOST_USER_VERSION    0x1
> +#endif