From patchwork Fri Dec 17 05:36:00 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joyce Kong X-Patchwork-Id: 105197 X-Patchwork-Delegate: qi.z.zhang@intel.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 699E6A04A6; Fri, 17 Dec 2021 06:36:21 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EBB5C40143; Fri, 17 Dec 2021 06:36:20 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 085DD4013F for ; Fri, 17 Dec 2021 06:36:18 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 35E3212FC; Thu, 16 Dec 2021 21:36:18 -0800 (PST) Received: from net-arm-n1amp-02.shanghai.arm.com (net-arm-n1amp-02.shanghai.arm.com [10.169.210.110]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 5DF523F5A1; Thu, 16 Dec 2021 21:36:16 -0800 (PST) From: Joyce Kong To: Ruifeng Wang , Beilei Xing Cc: dev@dpdk.org, nd@arm.com, Joyce Kong Subject: [PATCH v1] net/i40e: add flow mark capability to NEON vector routine Date: Fri, 17 Dec 2021 05:36:00 +0000 Message-Id: <20211217053601.776086-1-joyce.kong@arm.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org This commit adds a flow director support to i40e NEON vector RX path. I40e can have 16 and 32 byte descriptors, and the Flow Director ID data and indication-bit are in different locations for each size descriptor. The support is implemented in two separate functions as they require vastly different operations. The 16B descriptor re-purposes the "filter-status" u32 field to indicate FDIR ID when the FLM bit is set. No extra loads are required, however we do have to store to mbuf->fdir.hi, which is not stored to in the RX path before this patch. The 32B descriptor requires loading the 2nd 16 bytes of each descriptor, to get the FLEXBH_STAT and FD Filter ID from qword3. The resulting data must also be stored to mbuf->fdir.hi, same as the 16B code path. Signed-off-by: Joyce Kong Reviewed-by: Ruifeng Wang --- drivers/net/i40e/i40e_rxtx_vec_neon.c | 145 ++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 6 deletions(-) diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c index b951ea2dc3..a7ca2c18e5 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_neon.c +++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c @@ -77,9 +77,130 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id); } +#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC +/* NEON version of FDIR mark extraction for 4 32B descriptors at a time */ +static inline uint32x4_t +descs_to_fdir_32b(volatile union i40e_rx_desc *rxdp, struct rte_mbuf **rx_pkt) +{ + /* 32B descriptors: Load 2nd half of descriptors for FDIR ID data */ + uint64x2_t desc0_qw23, desc1_qw23, desc2_qw23, desc3_qw23; + desc0_qw23 = vld1q_u64((uint64_t *)&(rxdp + 0)->wb.qword2); + desc1_qw23 = vld1q_u64((uint64_t *)&(rxdp + 1)->wb.qword2); + desc2_qw23 = vld1q_u64((uint64_t *)&(rxdp + 2)->wb.qword2); + desc3_qw23 = vld1q_u64((uint64_t *)&(rxdp + 3)->wb.qword2); + + /* FDIR ID data: move last u32 of each desc to 4 u32 lanes */ + uint32x4_t v_unpack_02, v_unpack_13; + v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23), + vreinterpretq_u32_u64(desc2_qw23)).val[1]; + v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23), + vreinterpretq_u32_u64(desc3_qw23)).val[1]; + uint32x4_t v_fdir_ids = vzipq_u32(v_unpack_02, v_unpack_13).val[1]; + + /* Extended Status: extract from each lower 32 bits, to u32 lanes */ + v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23), + vreinterpretq_u32_u64(desc2_qw23)).val[0]; + v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23), + vreinterpretq_u32_u64(desc3_qw23)).val[0]; + uint32x4_t v_flt_status = vzipq_u32(v_unpack_02, v_unpack_13).val[0]; + + /* Shift u32 left and right to "mask away" bits not required. + * Data required is 4:5 (zero based), so left shift by 26 (32-6) + * and then right shift by 30 (32 - 2 bits required). + */ + v_flt_status = vshlq_n_u32(v_flt_status, 26); + v_flt_status = vshrq_n_u32(v_flt_status, 30); + + /* Generate constant 1 in all u32 lanes */ + RTE_BUILD_BUG_ON(I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID != 1); + uint32x4_t v_u32_one = vdupq_n_u32(1); + + /* Per desc mask, bits set if FDIR ID is valid */ + uint32x4_t v_fd_id_mask = vceqq_u32(v_flt_status, v_u32_one); + + /* Mask ID data to zero if the FD_ID bit not set in desc */ + v_fdir_ids = vandq_u32(v_fdir_ids, v_fd_id_mask); + + /* Store data to fdir.hi in mbuf */ + rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0); + rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1); + rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2); + rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3); + + /* Convert fdir_id_mask into a single bit, then shift as required for + * correct location in the mbuf->olflags + */ + RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13)); + v_fd_id_mask = vshrq_n_u32(v_fd_id_mask, 31); + v_fd_id_mask = vshlq_n_u32(v_fd_id_mask, 13); + + /* The returned value must be combined into each mbuf. This is already + * being done for RSS and VLAN mbuf olflags, so return bits to OR in. + */ + return v_fd_id_mask; +} + +#else /* 32 or 16B FDIR ID handling */ + +/* Handle 16B descriptor FDIR ID flag setting based on FLM(bit11). See scalar driver + * for scalar implementation of the same functionality. + */ +static inline uint32x4_t +descs_to_fdir_16b(uint32x4_t fltstat, uint64x2_t descs[4], struct rte_mbuf **rx_pkt) +{ + /* Unpack filter-status data from descriptors */ + uint32x4_t v_tmp_02 = vzipq_u32(vreinterpretq_u32_u64(descs[0]), + vreinterpretq_u32_u64(descs[2])).val[0]; + uint32x4_t v_tmp_13 = vzipq_u32(vreinterpretq_u32_u64(descs[1]), + vreinterpretq_u32_u64(descs[3])).val[0]; + uint32x4_t v_fdir_ids = vzipq_u32(v_tmp_02, v_tmp_13).val[1]; + + /* Generate 111 and 11 in each u32 lane */ + uint32x4_t v_111_mask = vdupq_n_u32(7); + uint32x4_t v_11_mask = vdupq_n_u32(3); + + /* Compare and mask away FDIR ID data if bit not set */ + uint32x4_t v_u32_bits = vandq_u32(v_111_mask, fltstat); + uint32x4_t v_fdir_id_mask = vceqq_u32(v_u32_bits, v_11_mask); + v_fdir_ids = vandq_u32(v_fdir_id_mask, v_fdir_ids); + + /* Store data to fdir.hi in mbuf */ + rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0); + rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1); + rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2); + rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3); + + /* Top lane ones mask for FDIR isolation */ + uint32x4_t v_desc_fdir_mask = {0, UINT32_MAX, 0, 0}; + + /* Move fdir_id_mask to correct lane, zero RSS in mbuf if fdir hits */ + uint32x4_t v_zeros = {0, 0, 0, 0}; + uint32x4_t v_desc3_shift = vextq_u32(v_fdir_id_mask, v_zeros, 2); + uint32x4_t v_desc3_mask = vandq_u32(v_desc_fdir_mask, v_desc3_shift); + descs[3] = vbslq_u32(v_desc3_mask, v_zeros, vreinterpretq_u32_u64(descs[3])); + + uint32x4_t v_desc2_shift = vextq_u32(v_fdir_id_mask, v_zeros, 1); + uint32x4_t v_desc2_mask = vandq_u32(v_desc_fdir_mask, v_desc2_shift); + descs[2] = vbslq_u32(v_desc2_mask, v_zeros, vreinterpretq_u32_u64(descs[2])); + + uint32x4_t v_desc1_shift = v_fdir_id_mask; + uint32x4_t v_desc1_mask = vandq_u32(v_desc_fdir_mask, v_desc1_shift); + descs[1] = vbslq_u32(v_desc1_mask, v_zeros, vreinterpretq_u32_u64(descs[1])); + + uint32x4_t v_desc0_shift = vextq_u32(v_zeros, v_fdir_id_mask, 3); + uint32x4_t v_desc0_mask = vandq_u32(v_desc_fdir_mask, v_desc0_shift); + descs[0] = vbslq_u32(v_desc0_mask, v_zeros, vreinterpretq_u32_u64(descs[0])); + + /* Shift to 1 or 0 bit per u32 lane, then to RTE_MBUF_F_RX_FDIR_ID offset */ + RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13)); + uint32x4_t v_mask_one_bit = vshrq_n_u32(v_fdir_id_mask, 31); + return vshlq_n_u32(v_mask_one_bit, 13); +} +#endif + static inline void -desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4], - struct rte_mbuf **rx_pkts) +desc_to_olflags_v(struct i40e_rx_queue *rxq, volatile union i40e_rx_desc *rxdp, + uint64x2_t descs[4], struct rte_mbuf **rx_pkts) { uint32x4_t vlan0, vlan1, rss, l3_l4e; const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; @@ -142,9 +263,9 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4], vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags, vreinterpretq_u8_u32(vlan1))); - rss = vshrq_n_u32(vlan1, 11); + const uint32x4_t desc_fltstat = vshrq_n_u32(vlan1, 11); rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags, - vreinterpretq_u8_u32(rss))); + vreinterpretq_u8_u32(desc_fltstat))); l3_l4e = vshrq_n_u32(vlan1, 22); l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags, @@ -157,6 +278,18 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4], vlan0 = vorrq_u32(vlan0, rss); vlan0 = vorrq_u32(vlan0, l3_l4e); + /* Extract FDIR ID only if FDIR is enabled to avoid useless work */ + if (rxq->fdir_enabled) { +#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC + uint32x4_t v_fdir_ol_flags = descs_to_fdir_32b(rxdp, rx_pkts); +#else + (void)rxdp; /* rxdp not required for 16B desc mode */ + uint32x4_t v_fdir_ol_flags = descs_to_fdir_16b(desc_fltstat, descs, rx_pkts); +#endif + /* OR in ol_flag bits after descriptor specific extraction */ + vlan0 = vorrq_u32(vlan0, v_fdir_ol_flags); + } + rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1); rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1); rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1); @@ -335,6 +468,8 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq, vreinterpretq_u16_u64(descs[0]), 7)); + desc_to_olflags_v(rxq, rxdp, descs, &rx_pkts[pos]); + /* D.1 pkts convert format from desc to pktmbuf */ pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk); pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk); @@ -363,8 +498,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq, desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); - desc_to_olflags_v(rxq, descs, &rx_pkts[pos]); - if (likely(pos + RTE_I40E_DESCS_PER_LOOP < nb_pkts)) { rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP); }