@@ -27,6 +27,7 @@ sources = files(
'vhost_crypto.c',
'vhost_user.c',
'virtio_net.c',
+ 'virtio_net_ctrl.c',
)
headers = files(
'rte_vdpa.h',
@@ -527,6 +527,8 @@ struct virtio_net {
int postcopy_ufd;
int postcopy_listening;
+ struct vhost_virtqueue *cvq;
+
struct rte_vdpa_device *vdpa_dev;
/* context data for the external message handlers */
new file mode 100644
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Red Hat, Inc.
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include "iotlb.h"
+#include "vhost.h"
+#include "virtio_net_ctrl.h"
+
+struct virtio_net_ctrl {
+ uint8_t class;
+ uint8_t command;
+ uint8_t command_data[];
+};
+
+struct virtio_net_ctrl_elem {
+ struct virtio_net_ctrl *ctrl_req;
+ uint16_t head_idx;
+ uint16_t n_descs;
+ uint8_t *desc_ack;
+};
+
+static int
+virtio_net_ctrl_pop(struct virtio_net *dev, struct vhost_virtqueue *cvq,
+ struct virtio_net_ctrl_elem *ctrl_elem)
+ __rte_shared_locks_required(&cvq->iotlb_lock)
+{
+ uint16_t avail_idx, desc_idx, n_descs = 0;
+ uint64_t desc_len, desc_addr, desc_iova, data_len = 0;
+ uint8_t *ctrl_req;
+ struct vring_desc *descs;
+
+ avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE);
+ if (avail_idx == cvq->last_avail_idx) {
+ VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n");
+ return 0;
+ }
+
+ desc_idx = cvq->avail->ring[cvq->last_avail_idx];
+ if (desc_idx >= cvq->size) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Out of range desc index, dropping\n");
+ goto err;
+ }
+
+ ctrl_elem->head_idx = desc_idx;
+
+ if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
+ desc_len = cvq->desc[desc_idx].len;
+ desc_iova = cvq->desc[desc_idx].addr;
+
+ descs = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+ desc_iova, &desc_len, VHOST_ACCESS_RO);
+ if (!descs || desc_len != cvq->desc[desc_idx].len) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl indirect descs\n");
+ goto err;
+ }
+
+ desc_idx = 0;
+ } else {
+ descs = cvq->desc;
+ }
+
+ while (1) {
+ desc_len = descs[desc_idx].len;
+ desc_iova = descs[desc_idx].addr;
+
+ n_descs++;
+
+ if (descs[desc_idx].flags & VRING_DESC_F_WRITE) {
+ if (ctrl_elem->desc_ack) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR,
+ "Unexpected ctrl chain layout\n");
+ goto err;
+ }
+
+ if (desc_len != sizeof(uint8_t)) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR,
+ "Invalid ack size for ctrl req, dropping\n");
+ goto err;
+ }
+
+ ctrl_elem->desc_ack = (uint8_t *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+ desc_iova, &desc_len, VHOST_ACCESS_WO);
+ if (!ctrl_elem->desc_ack || desc_len != sizeof(uint8_t)) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR,
+ "Failed to map ctrl ack descriptor\n");
+ goto err;
+ }
+ } else {
+ if (ctrl_elem->desc_ack) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR,
+ "Unexpected ctrl chain layout\n");
+ goto err;
+ }
+
+ data_len += desc_len;
+ }
+
+ if (!(descs[desc_idx].flags & VRING_DESC_F_NEXT))
+ break;
+
+ desc_idx = descs[desc_idx].next;
+ }
+
+ desc_idx = ctrl_elem->head_idx;
+
+ if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT)
+ ctrl_elem->n_descs = 1;
+ else
+ ctrl_elem->n_descs = n_descs;
+
+ if (!ctrl_elem->desc_ack) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Missing ctrl ack descriptor\n");
+ goto err;
+ }
+
+ if (data_len < sizeof(ctrl_elem->ctrl_req->class) + sizeof(ctrl_elem->ctrl_req->command)) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Invalid control header size\n");
+ goto err;
+ }
+
+ ctrl_elem->ctrl_req = malloc(data_len);
+ if (!ctrl_elem->ctrl_req) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to alloc ctrl request\n");
+ goto err;
+ }
+
+ ctrl_req = (uint8_t *)ctrl_elem->ctrl_req;
+
+ if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
+ desc_len = cvq->desc[desc_idx].len;
+ desc_iova = cvq->desc[desc_idx].addr;
+
+ descs = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+ desc_iova, &desc_len, VHOST_ACCESS_RO);
+ if (!descs || desc_len != cvq->desc[desc_idx].len) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl indirect descs\n");
+ goto free_err;
+ }
+
+ desc_idx = 0;
+ } else {
+ descs = cvq->desc;
+ }
+
+ while (!(descs[desc_idx].flags & VRING_DESC_F_WRITE)) {
+ desc_len = descs[desc_idx].len;
+ desc_iova = descs[desc_idx].addr;
+
+ desc_addr = vhost_iova_to_vva(dev, cvq, desc_iova, &desc_len, VHOST_ACCESS_RO);
+ if (!desc_addr || desc_len < descs[desc_idx].len) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl descriptor\n");
+ goto free_err;
+ }
+
+ memcpy(ctrl_req, (void *)(uintptr_t)desc_addr, desc_len);
+ ctrl_req += desc_len;
+
+ if (!(descs[desc_idx].flags & VRING_DESC_F_NEXT))
+ break;
+
+ desc_idx = descs[desc_idx].next;
+ }
+
+ cvq->last_avail_idx++;
+ if (cvq->last_avail_idx >= cvq->size)
+ cvq->last_avail_idx -= cvq->size;
+
+ if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
+ vhost_avail_event(cvq) = cvq->last_avail_idx;
+
+ return 1;
+
+free_err:
+ free(ctrl_elem->ctrl_req);
+err:
+ cvq->last_avail_idx++;
+ if (cvq->last_avail_idx >= cvq->size)
+ cvq->last_avail_idx -= cvq->size;
+
+ if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
+ vhost_avail_event(cvq) = cvq->last_avail_idx;
+
+ return -1;
+}
+
+static uint8_t
+virtio_net_ctrl_handle_req(struct virtio_net *dev, struct virtio_net_ctrl *ctrl_req)
+{
+ uint8_t ret = VIRTIO_NET_ERR;
+
+ if (ctrl_req->class == VIRTIO_NET_CTRL_MQ &&
+ ctrl_req->command == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
+ uint16_t queue_pairs;
+ uint32_t i;
+
+ queue_pairs = *(uint16_t *)(uintptr_t)ctrl_req->command_data;
+ VHOST_LOG_CONFIG(dev->ifname, INFO, "Ctrl req: MQ %u queue pairs\n", queue_pairs);
+ ret = VIRTIO_NET_OK;
+
+ for (i = 0; i < dev->nr_vring; i++) {
+ struct vhost_virtqueue *vq = dev->virtqueue[i];
+ bool enable;
+
+ if (vq == dev->cvq)
+ continue;
+
+ if (i < queue_pairs * 2)
+ enable = true;
+ else
+ enable = false;
+
+ vq->enabled = enable;
+ if (dev->notify_ops->vring_state_changed)
+ dev->notify_ops->vring_state_changed(dev->vid, i, enable);
+ }
+ }
+
+ return ret;
+}
+
+static int
+virtio_net_ctrl_push(struct virtio_net *dev, struct virtio_net_ctrl_elem *ctrl_elem)
+{
+ struct vhost_virtqueue *cvq = dev->cvq;
+ struct vring_used_elem *used_elem;
+
+ used_elem = &cvq->used->ring[cvq->last_used_idx];
+ used_elem->id = ctrl_elem->head_idx;
+ used_elem->len = ctrl_elem->n_descs;
+
+ cvq->last_used_idx++;
+ if (cvq->last_used_idx >= cvq->size)
+ cvq->last_used_idx -= cvq->size;
+
+ __atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE);
+
+ vhost_vring_call_split(dev, dev->cvq);
+
+ free(ctrl_elem->ctrl_req);
+
+ return 0;
+}
+
+int
+virtio_net_ctrl_handle(struct virtio_net *dev)
+{
+ int ret = 0;
+
+ if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "Packed ring not supported yet\n");
+ return -1;
+ }
+
+ if (!dev->cvq) {
+ VHOST_LOG_CONFIG(dev->ifname, ERR, "missing control queue\n");
+ return -1;
+ }
+
+ rte_rwlock_read_lock(&dev->cvq->access_lock);
+ vhost_user_iotlb_rd_lock(dev->cvq);
+
+ while (1) {
+ struct virtio_net_ctrl_elem ctrl_elem;
+
+ memset(&ctrl_elem, 0, sizeof(struct virtio_net_ctrl_elem));
+
+ ret = virtio_net_ctrl_pop(dev, dev->cvq, &ctrl_elem);
+ if (ret <= 0)
+ break;
+
+ *ctrl_elem.desc_ack = virtio_net_ctrl_handle_req(dev, ctrl_elem.ctrl_req);
+
+ ret = virtio_net_ctrl_push(dev, &ctrl_elem);
+ if (ret < 0)
+ break;
+ }
+
+ vhost_user_iotlb_rd_unlock(dev->cvq);
+ rte_rwlock_read_unlock(&dev->cvq->access_lock);
+
+ return ret;
+}
new file mode 100644
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Red Hat, Inc.
+ */
+
+#ifndef _VIRTIO_NET_CTRL_H
+#define _VIRTIO_NET_CTRL_H
+
+int virtio_net_ctrl_handle(struct virtio_net *dev);
+
+#endif