From patchwork Tue Jul 18 15:40:17 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 27034 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 0928656A1; Tue, 18 Jul 2017 17:41:55 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by dpdk.org (Postfix) with ESMTP id 1D582376C for ; Tue, 18 Jul 2017 17:41:44 +0200 (CEST) Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 18 Jul 2017 08:41:43 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.40,378,1496127600"; d="scan'208";a="994343814" Received: from dpdk25.sh.intel.com ([10.67.111.96]) by orsmga003.jf.intel.com with ESMTP; 18 Jul 2017 08:41:43 -0700 From: Tiwei Bie To: dev@dpdk.org, jfreiman@redhat.com Date: Tue, 18 Jul 2017 23:40:17 +0800 Message-Id: <1500392421-76672-3-git-send-email-tiwei.bie@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1500392421-76672-1-git-send-email-tiwei.bie@intel.com> References: <1500392421-76672-1-git-send-email-tiwei.bie@intel.com> Subject: [dpdk-dev] [RFC 2/6] vhost: optimize enqueue path X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Signed-off-by: Tiwei Bie --- lib/librte_vhost/virtio_net.c | 185 ++++++++++++++++++++---------------------- 1 file changed, 88 insertions(+), 97 deletions(-) diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index c0b4dde..0888d2b 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -583,129 +583,120 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, return pkt_idx; } -static inline int __attribute__((always_inline)) -enqueue_pkt(struct virtio_net *dev, struct vring_desc_1_1 *descs, - uint16_t desc_idx, struct rte_mbuf *m) -{ - uint32_t desc_avail, desc_offset; - uint32_t mbuf_avail, mbuf_offset; - uint32_t cpy_len; - struct vring_desc_1_1 *desc; - uint64_t desc_addr; - struct virtio_net_hdr_mrg_rxbuf *hdr; - - desc = &descs[desc_idx]; - desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr); - /* - * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid - * performance issue with some versions of gcc (4.8.4 and 5.3.0) which - * otherwise stores offset on the stack instead of in a register. - */ - if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) - return -1; - - rte_prefetch0((void *)(uintptr_t)desc_addr); - - hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr; - virtio_enqueue_offload(m, &hdr->hdr); - vhost_log_write(dev, desc->addr, dev->vhost_hlen); - PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); - - desc_offset = dev->vhost_hlen; - desc_avail = desc->len - dev->vhost_hlen; - - mbuf_avail = rte_pktmbuf_data_len(m); - mbuf_offset = 0; - while (mbuf_avail != 0 || m->next != NULL) { - /* done with current mbuf, fetch next */ - if (mbuf_avail == 0) { - m = m->next; - - mbuf_offset = 0; - mbuf_avail = rte_pktmbuf_data_len(m); - } - - /* done with current desc buf, fetch next */ - if (desc_avail == 0) { - if ((desc->flags & VRING_DESC_F_NEXT) == 0) { - /* Room in vring buffer is not enough */ - return -1; - } - - rte_panic("Shouldn't reach here\n"); - /** NOTE: we should not come here with current - virtio-user implementation **/ - desc_idx = (desc_idx + 1); // & (vq->size - 1); - desc = &descs[desc_idx]; - if (unlikely(!(desc->flags & DESC_HW))) - return -1; - - desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr); - if (unlikely(!desc_addr)) - return -1; - - desc_offset = 0; - desc_avail = desc->len; - } - - cpy_len = RTE_MIN(desc_avail, mbuf_avail); - rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)), - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), - cpy_len); - vhost_log_write(dev, desc->addr + desc_offset, cpy_len); - PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), - cpy_len, 0); - - mbuf_avail -= cpy_len; - mbuf_offset += cpy_len; - desc_avail -= cpy_len; - desc_offset += cpy_len; - } - - return 0; -} - static inline uint32_t __attribute__((always_inline)) vhost_enqueue_burst_1_1(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; + struct vring_desc_1_1 *descs; + uint16_t head_idx, idx; + uint16_t mask; uint16_t i; - uint16_t idx; - struct vring_desc_1_1 *desc; - uint16_t head_idx; vq = dev->virtqueue[queue_id]; if (unlikely(vq->enabled == 0)) return 0; - head_idx = vq->last_used_idx; - desc = vq->desc_1_1; - count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); + descs = vq->desc_1_1; + mask = vq->size - 1; + head_idx = vq->last_used_idx & mask; for (i = 0; i < count; i++) { + uint32_t desc_avail, desc_offset; + uint32_t mbuf_avail, mbuf_offset; + uint32_t cpy_len; + struct vring_desc_1_1 *desc; + uint64_t desc_addr; + struct virtio_net_hdr_mrg_rxbuf *hdr; + struct rte_mbuf *m = pkts[i]; + /* XXX: there is an assumption that no desc will be chained */ - idx = vq->last_used_idx & (vq->size - 1); - if (!(desc[idx].flags & DESC_HW)) + idx = vq->last_used_idx & mask; + desc = &descs[idx]; + + if (!(desc->flags & DESC_HW)) break; - if (enqueue_pkt(dev, desc, idx, pkts[i]) < 0) + desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr); + /* + * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid + * performance issue with some versions of gcc (4.8.4 and 5.3.0) which + * otherwise stores offset on the stack instead of in a register. + */ + if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) break; + hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr; + virtio_enqueue_offload(m, &hdr->hdr); + vhost_log_write(dev, desc->addr, dev->vhost_hlen); + PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); + + desc_offset = dev->vhost_hlen; + desc_avail = desc->len - dev->vhost_hlen; + + mbuf_avail = rte_pktmbuf_data_len(m); + mbuf_offset = 0; + while (mbuf_avail != 0 || m->next != NULL) { + /* done with current mbuf, fetch next */ + if (mbuf_avail == 0) { + m = m->next; + + mbuf_offset = 0; + mbuf_avail = rte_pktmbuf_data_len(m); + } + + /* done with current desc buf, fetch next */ + if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) { + /* Room in vring buffer is not enough */ + goto end_of_tx; + } + + rte_panic("Shouldn't reach here\n"); + /** NOTE: we should not come here with current + virtio-user implementation **/ + idx = (idx + 1); // & (vq->size - 1); + desc = &descs[idx]; + if (unlikely(!(desc->flags & DESC_HW))) + goto end_of_tx; + + desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr); + if (unlikely(!desc_addr)) + goto end_of_tx; + + desc_offset = 0; + desc_avail = desc->len; + } + + cpy_len = RTE_MIN(desc_avail, mbuf_avail); + rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)), + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + cpy_len); + vhost_log_write(dev, desc->addr + desc_offset, cpy_len); + PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), + cpy_len, 0); + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + } + vq->last_used_idx++; } + +end_of_tx: count = i; if (count) { for (i = 1; i < count; i++) { - idx = (head_idx + i) & (vq->size - 1); - desc[idx].len = pkts[i]->pkt_len + dev->vhost_hlen; - desc[idx].flags &= ~DESC_HW; + idx = (head_idx + i) & mask; + descs[idx].len = pkts[i]->pkt_len + dev->vhost_hlen; + descs[idx].flags &= ~DESC_HW; } - desc[head_idx & (vq->size - 1)].len = - pkts[0]->pkt_len + dev->vhost_hlen; + descs[head_idx].len = pkts[0]->pkt_len + dev->vhost_hlen; rte_smp_wmb(); - desc[head_idx & (vq->size - 1)].flags &= ~DESC_HW; + descs[head_idx].flags &= ~DESC_HW; } return count;